1 /**************************************************************************** 2 * Driver for Solarflare network controllers and boards 3 * Copyright 2005-2006 Fen Systems Ltd. 4 * Copyright 2005-2013 Solarflare Communications Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation, incorporated herein by reference. 9 */ 10 11 #include <linux/module.h> 12 #include <linux/pci.h> 13 #include <linux/netdevice.h> 14 #include <linux/etherdevice.h> 15 #include <linux/delay.h> 16 #include <linux/notifier.h> 17 #include <linux/ip.h> 18 #include <linux/tcp.h> 19 #include <linux/in.h> 20 #include <linux/ethtool.h> 21 #include <linux/topology.h> 22 #include <linux/gfp.h> 23 #include <linux/aer.h> 24 #include <linux/interrupt.h> 25 #include "net_driver.h" 26 #include "efx.h" 27 #include "nic.h" 28 #include "selftest.h" 29 30 #include "workarounds.h" 31 32 /************************************************************************** 33 * 34 * Type name strings 35 * 36 ************************************************************************** 37 */ 38 39 /* Loopback mode names (see LOOPBACK_MODE()) */ 40 const unsigned int ef4_loopback_mode_max = LOOPBACK_MAX; 41 const char *const ef4_loopback_mode_names[] = { 42 [LOOPBACK_NONE] = "NONE", 43 [LOOPBACK_DATA] = "DATAPATH", 44 [LOOPBACK_GMAC] = "GMAC", 45 [LOOPBACK_XGMII] = "XGMII", 46 [LOOPBACK_XGXS] = "XGXS", 47 [LOOPBACK_XAUI] = "XAUI", 48 [LOOPBACK_GMII] = "GMII", 49 [LOOPBACK_SGMII] = "SGMII", 50 [LOOPBACK_XGBR] = "XGBR", 51 [LOOPBACK_XFI] = "XFI", 52 [LOOPBACK_XAUI_FAR] = "XAUI_FAR", 53 [LOOPBACK_GMII_FAR] = "GMII_FAR", 54 [LOOPBACK_SGMII_FAR] = "SGMII_FAR", 55 [LOOPBACK_XFI_FAR] = "XFI_FAR", 56 [LOOPBACK_GPHY] = "GPHY", 57 [LOOPBACK_PHYXS] = "PHYXS", 58 [LOOPBACK_PCS] = "PCS", 59 [LOOPBACK_PMAPMD] = "PMA/PMD", 60 [LOOPBACK_XPORT] = "XPORT", 61 [LOOPBACK_XGMII_WS] = "XGMII_WS", 62 [LOOPBACK_XAUI_WS] = "XAUI_WS", 63 [LOOPBACK_XAUI_WS_FAR] = "XAUI_WS_FAR", 64 [LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR", 65 [LOOPBACK_GMII_WS] = "GMII_WS", 66 [LOOPBACK_XFI_WS] = "XFI_WS", 67 [LOOPBACK_XFI_WS_FAR] = "XFI_WS_FAR", 68 [LOOPBACK_PHYXS_WS] = "PHYXS_WS", 69 }; 70 71 const unsigned int ef4_reset_type_max = RESET_TYPE_MAX; 72 const char *const ef4_reset_type_names[] = { 73 [RESET_TYPE_INVISIBLE] = "INVISIBLE", 74 [RESET_TYPE_ALL] = "ALL", 75 [RESET_TYPE_RECOVER_OR_ALL] = "RECOVER_OR_ALL", 76 [RESET_TYPE_WORLD] = "WORLD", 77 [RESET_TYPE_RECOVER_OR_DISABLE] = "RECOVER_OR_DISABLE", 78 [RESET_TYPE_DATAPATH] = "DATAPATH", 79 [RESET_TYPE_DISABLE] = "DISABLE", 80 [RESET_TYPE_TX_WATCHDOG] = "TX_WATCHDOG", 81 [RESET_TYPE_INT_ERROR] = "INT_ERROR", 82 [RESET_TYPE_RX_RECOVERY] = "RX_RECOVERY", 83 [RESET_TYPE_DMA_ERROR] = "DMA_ERROR", 84 [RESET_TYPE_TX_SKIP] = "TX_SKIP", 85 }; 86 87 /* Reset workqueue. If any NIC has a hardware failure then a reset will be 88 * queued onto this work queue. This is not a per-nic work queue, because 89 * ef4_reset_work() acquires the rtnl lock, so resets are naturally serialised. 90 */ 91 static struct workqueue_struct *reset_workqueue; 92 93 /* How often and how many times to poll for a reset while waiting for a 94 * BIST that another function started to complete. 95 */ 96 #define BIST_WAIT_DELAY_MS 100 97 #define BIST_WAIT_DELAY_COUNT 100 98 99 /************************************************************************** 100 * 101 * Configurable values 102 * 103 *************************************************************************/ 104 105 /* 106 * Use separate channels for TX and RX events 107 * 108 * Set this to 1 to use separate channels for TX and RX. It allows us 109 * to control interrupt affinity separately for TX and RX. 110 * 111 * This is only used in MSI-X interrupt mode 112 */ 113 bool ef4_separate_tx_channels; 114 module_param(ef4_separate_tx_channels, bool, 0444); 115 MODULE_PARM_DESC(ef4_separate_tx_channels, 116 "Use separate channels for TX and RX"); 117 118 /* This is the weight assigned to each of the (per-channel) virtual 119 * NAPI devices. 120 */ 121 static int napi_weight = 64; 122 123 /* This is the time (in jiffies) between invocations of the hardware 124 * monitor. 125 * On Falcon-based NICs, this will: 126 * - Check the on-board hardware monitor; 127 * - Poll the link state and reconfigure the hardware as necessary. 128 * On Siena-based NICs for power systems with EEH support, this will give EEH a 129 * chance to start. 130 */ 131 static unsigned int ef4_monitor_interval = 1 * HZ; 132 133 /* Initial interrupt moderation settings. They can be modified after 134 * module load with ethtool. 135 * 136 * The default for RX should strike a balance between increasing the 137 * round-trip latency and reducing overhead. 138 */ 139 static unsigned int rx_irq_mod_usec = 60; 140 141 /* Initial interrupt moderation settings. They can be modified after 142 * module load with ethtool. 143 * 144 * This default is chosen to ensure that a 10G link does not go idle 145 * while a TX queue is stopped after it has become full. A queue is 146 * restarted when it drops below half full. The time this takes (assuming 147 * worst case 3 descriptors per packet and 1024 descriptors) is 148 * 512 / 3 * 1.2 = 205 usec. 149 */ 150 static unsigned int tx_irq_mod_usec = 150; 151 152 /* This is the first interrupt mode to try out of: 153 * 0 => MSI-X 154 * 1 => MSI 155 * 2 => legacy 156 */ 157 static unsigned int interrupt_mode; 158 159 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS), 160 * i.e. the number of CPUs among which we may distribute simultaneous 161 * interrupt handling. 162 * 163 * Cards without MSI-X will only target one CPU via legacy or MSI interrupt. 164 * The default (0) means to assign an interrupt to each core. 165 */ 166 static unsigned int rss_cpus; 167 module_param(rss_cpus, uint, 0444); 168 MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling"); 169 170 static bool phy_flash_cfg; 171 module_param(phy_flash_cfg, bool, 0644); 172 MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially"); 173 174 static unsigned irq_adapt_low_thresh = 8000; 175 module_param(irq_adapt_low_thresh, uint, 0644); 176 MODULE_PARM_DESC(irq_adapt_low_thresh, 177 "Threshold score for reducing IRQ moderation"); 178 179 static unsigned irq_adapt_high_thresh = 16000; 180 module_param(irq_adapt_high_thresh, uint, 0644); 181 MODULE_PARM_DESC(irq_adapt_high_thresh, 182 "Threshold score for increasing IRQ moderation"); 183 184 static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 185 NETIF_MSG_LINK | NETIF_MSG_IFDOWN | 186 NETIF_MSG_IFUP | NETIF_MSG_RX_ERR | 187 NETIF_MSG_TX_ERR | NETIF_MSG_HW); 188 module_param(debug, uint, 0); 189 MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value"); 190 191 /************************************************************************** 192 * 193 * Utility functions and prototypes 194 * 195 *************************************************************************/ 196 197 static int ef4_soft_enable_interrupts(struct ef4_nic *efx); 198 static void ef4_soft_disable_interrupts(struct ef4_nic *efx); 199 static void ef4_remove_channel(struct ef4_channel *channel); 200 static void ef4_remove_channels(struct ef4_nic *efx); 201 static const struct ef4_channel_type ef4_default_channel_type; 202 static void ef4_remove_port(struct ef4_nic *efx); 203 static void ef4_init_napi_channel(struct ef4_channel *channel); 204 static void ef4_fini_napi(struct ef4_nic *efx); 205 static void ef4_fini_napi_channel(struct ef4_channel *channel); 206 static void ef4_fini_struct(struct ef4_nic *efx); 207 static void ef4_start_all(struct ef4_nic *efx); 208 static void ef4_stop_all(struct ef4_nic *efx); 209 210 #define EF4_ASSERT_RESET_SERIALISED(efx) \ 211 do { \ 212 if ((efx->state == STATE_READY) || \ 213 (efx->state == STATE_RECOVERY) || \ 214 (efx->state == STATE_DISABLED)) \ 215 ASSERT_RTNL(); \ 216 } while (0) 217 218 static int ef4_check_disabled(struct ef4_nic *efx) 219 { 220 if (efx->state == STATE_DISABLED || efx->state == STATE_RECOVERY) { 221 netif_err(efx, drv, efx->net_dev, 222 "device is disabled due to earlier errors\n"); 223 return -EIO; 224 } 225 return 0; 226 } 227 228 /************************************************************************** 229 * 230 * Event queue processing 231 * 232 *************************************************************************/ 233 234 /* Process channel's event queue 235 * 236 * This function is responsible for processing the event queue of a 237 * single channel. The caller must guarantee that this function will 238 * never be concurrently called more than once on the same channel, 239 * though different channels may be being processed concurrently. 240 */ 241 static int ef4_process_channel(struct ef4_channel *channel, int budget) 242 { 243 struct ef4_tx_queue *tx_queue; 244 int spent; 245 246 if (unlikely(!channel->enabled)) 247 return 0; 248 249 ef4_for_each_channel_tx_queue(tx_queue, channel) { 250 tx_queue->pkts_compl = 0; 251 tx_queue->bytes_compl = 0; 252 } 253 254 spent = ef4_nic_process_eventq(channel, budget); 255 if (spent && ef4_channel_has_rx_queue(channel)) { 256 struct ef4_rx_queue *rx_queue = 257 ef4_channel_get_rx_queue(channel); 258 259 ef4_rx_flush_packet(channel); 260 ef4_fast_push_rx_descriptors(rx_queue, true); 261 } 262 263 /* Update BQL */ 264 ef4_for_each_channel_tx_queue(tx_queue, channel) { 265 if (tx_queue->bytes_compl) { 266 netdev_tx_completed_queue(tx_queue->core_txq, 267 tx_queue->pkts_compl, tx_queue->bytes_compl); 268 } 269 } 270 271 return spent; 272 } 273 274 /* NAPI poll handler 275 * 276 * NAPI guarantees serialisation of polls of the same device, which 277 * provides the guarantee required by ef4_process_channel(). 278 */ 279 static void ef4_update_irq_mod(struct ef4_nic *efx, struct ef4_channel *channel) 280 { 281 int step = efx->irq_mod_step_us; 282 283 if (channel->irq_mod_score < irq_adapt_low_thresh) { 284 if (channel->irq_moderation_us > step) { 285 channel->irq_moderation_us -= step; 286 efx->type->push_irq_moderation(channel); 287 } 288 } else if (channel->irq_mod_score > irq_adapt_high_thresh) { 289 if (channel->irq_moderation_us < 290 efx->irq_rx_moderation_us) { 291 channel->irq_moderation_us += step; 292 efx->type->push_irq_moderation(channel); 293 } 294 } 295 296 channel->irq_count = 0; 297 channel->irq_mod_score = 0; 298 } 299 300 static int ef4_poll(struct napi_struct *napi, int budget) 301 { 302 struct ef4_channel *channel = 303 container_of(napi, struct ef4_channel, napi_str); 304 struct ef4_nic *efx = channel->efx; 305 int spent; 306 307 netif_vdbg(efx, intr, efx->net_dev, 308 "channel %d NAPI poll executing on CPU %d\n", 309 channel->channel, raw_smp_processor_id()); 310 311 spent = ef4_process_channel(channel, budget); 312 313 if (spent < budget) { 314 if (ef4_channel_has_rx_queue(channel) && 315 efx->irq_rx_adaptive && 316 unlikely(++channel->irq_count == 1000)) { 317 ef4_update_irq_mod(efx, channel); 318 } 319 320 ef4_filter_rfs_expire(channel); 321 322 /* There is no race here; although napi_disable() will 323 * only wait for napi_complete(), this isn't a problem 324 * since ef4_nic_eventq_read_ack() will have no effect if 325 * interrupts have already been disabled. 326 */ 327 napi_complete_done(napi, spent); 328 ef4_nic_eventq_read_ack(channel); 329 } 330 331 return spent; 332 } 333 334 /* Create event queue 335 * Event queue memory allocations are done only once. If the channel 336 * is reset, the memory buffer will be reused; this guards against 337 * errors during channel reset and also simplifies interrupt handling. 338 */ 339 static int ef4_probe_eventq(struct ef4_channel *channel) 340 { 341 struct ef4_nic *efx = channel->efx; 342 unsigned long entries; 343 344 netif_dbg(efx, probe, efx->net_dev, 345 "chan %d create event queue\n", channel->channel); 346 347 /* Build an event queue with room for one event per tx and rx buffer, 348 * plus some extra for link state events and MCDI completions. */ 349 entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128); 350 EF4_BUG_ON_PARANOID(entries > EF4_MAX_EVQ_SIZE); 351 channel->eventq_mask = max(entries, EF4_MIN_EVQ_SIZE) - 1; 352 353 return ef4_nic_probe_eventq(channel); 354 } 355 356 /* Prepare channel's event queue */ 357 static int ef4_init_eventq(struct ef4_channel *channel) 358 { 359 struct ef4_nic *efx = channel->efx; 360 int rc; 361 362 EF4_WARN_ON_PARANOID(channel->eventq_init); 363 364 netif_dbg(efx, drv, efx->net_dev, 365 "chan %d init event queue\n", channel->channel); 366 367 rc = ef4_nic_init_eventq(channel); 368 if (rc == 0) { 369 efx->type->push_irq_moderation(channel); 370 channel->eventq_read_ptr = 0; 371 channel->eventq_init = true; 372 } 373 return rc; 374 } 375 376 /* Enable event queue processing and NAPI */ 377 void ef4_start_eventq(struct ef4_channel *channel) 378 { 379 netif_dbg(channel->efx, ifup, channel->efx->net_dev, 380 "chan %d start event queue\n", channel->channel); 381 382 /* Make sure the NAPI handler sees the enabled flag set */ 383 channel->enabled = true; 384 smp_wmb(); 385 386 napi_enable(&channel->napi_str); 387 ef4_nic_eventq_read_ack(channel); 388 } 389 390 /* Disable event queue processing and NAPI */ 391 void ef4_stop_eventq(struct ef4_channel *channel) 392 { 393 if (!channel->enabled) 394 return; 395 396 napi_disable(&channel->napi_str); 397 channel->enabled = false; 398 } 399 400 static void ef4_fini_eventq(struct ef4_channel *channel) 401 { 402 if (!channel->eventq_init) 403 return; 404 405 netif_dbg(channel->efx, drv, channel->efx->net_dev, 406 "chan %d fini event queue\n", channel->channel); 407 408 ef4_nic_fini_eventq(channel); 409 channel->eventq_init = false; 410 } 411 412 static void ef4_remove_eventq(struct ef4_channel *channel) 413 { 414 netif_dbg(channel->efx, drv, channel->efx->net_dev, 415 "chan %d remove event queue\n", channel->channel); 416 417 ef4_nic_remove_eventq(channel); 418 } 419 420 /************************************************************************** 421 * 422 * Channel handling 423 * 424 *************************************************************************/ 425 426 /* Allocate and initialise a channel structure. */ 427 static struct ef4_channel * 428 ef4_alloc_channel(struct ef4_nic *efx, int i, struct ef4_channel *old_channel) 429 { 430 struct ef4_channel *channel; 431 struct ef4_rx_queue *rx_queue; 432 struct ef4_tx_queue *tx_queue; 433 int j; 434 435 channel = kzalloc(sizeof(*channel), GFP_KERNEL); 436 if (!channel) 437 return NULL; 438 439 channel->efx = efx; 440 channel->channel = i; 441 channel->type = &ef4_default_channel_type; 442 443 for (j = 0; j < EF4_TXQ_TYPES; j++) { 444 tx_queue = &channel->tx_queue[j]; 445 tx_queue->efx = efx; 446 tx_queue->queue = i * EF4_TXQ_TYPES + j; 447 tx_queue->channel = channel; 448 } 449 450 rx_queue = &channel->rx_queue; 451 rx_queue->efx = efx; 452 setup_timer(&rx_queue->slow_fill, ef4_rx_slow_fill, 453 (unsigned long)rx_queue); 454 455 return channel; 456 } 457 458 /* Allocate and initialise a channel structure, copying parameters 459 * (but not resources) from an old channel structure. 460 */ 461 static struct ef4_channel * 462 ef4_copy_channel(const struct ef4_channel *old_channel) 463 { 464 struct ef4_channel *channel; 465 struct ef4_rx_queue *rx_queue; 466 struct ef4_tx_queue *tx_queue; 467 int j; 468 469 channel = kmalloc(sizeof(*channel), GFP_KERNEL); 470 if (!channel) 471 return NULL; 472 473 *channel = *old_channel; 474 475 channel->napi_dev = NULL; 476 INIT_HLIST_NODE(&channel->napi_str.napi_hash_node); 477 channel->napi_str.napi_id = 0; 478 channel->napi_str.state = 0; 479 memset(&channel->eventq, 0, sizeof(channel->eventq)); 480 481 for (j = 0; j < EF4_TXQ_TYPES; j++) { 482 tx_queue = &channel->tx_queue[j]; 483 if (tx_queue->channel) 484 tx_queue->channel = channel; 485 tx_queue->buffer = NULL; 486 memset(&tx_queue->txd, 0, sizeof(tx_queue->txd)); 487 } 488 489 rx_queue = &channel->rx_queue; 490 rx_queue->buffer = NULL; 491 memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd)); 492 setup_timer(&rx_queue->slow_fill, ef4_rx_slow_fill, 493 (unsigned long)rx_queue); 494 495 return channel; 496 } 497 498 static int ef4_probe_channel(struct ef4_channel *channel) 499 { 500 struct ef4_tx_queue *tx_queue; 501 struct ef4_rx_queue *rx_queue; 502 int rc; 503 504 netif_dbg(channel->efx, probe, channel->efx->net_dev, 505 "creating channel %d\n", channel->channel); 506 507 rc = channel->type->pre_probe(channel); 508 if (rc) 509 goto fail; 510 511 rc = ef4_probe_eventq(channel); 512 if (rc) 513 goto fail; 514 515 ef4_for_each_channel_tx_queue(tx_queue, channel) { 516 rc = ef4_probe_tx_queue(tx_queue); 517 if (rc) 518 goto fail; 519 } 520 521 ef4_for_each_channel_rx_queue(rx_queue, channel) { 522 rc = ef4_probe_rx_queue(rx_queue); 523 if (rc) 524 goto fail; 525 } 526 527 return 0; 528 529 fail: 530 ef4_remove_channel(channel); 531 return rc; 532 } 533 534 static void 535 ef4_get_channel_name(struct ef4_channel *channel, char *buf, size_t len) 536 { 537 struct ef4_nic *efx = channel->efx; 538 const char *type; 539 int number; 540 541 number = channel->channel; 542 if (efx->tx_channel_offset == 0) { 543 type = ""; 544 } else if (channel->channel < efx->tx_channel_offset) { 545 type = "-rx"; 546 } else { 547 type = "-tx"; 548 number -= efx->tx_channel_offset; 549 } 550 snprintf(buf, len, "%s%s-%d", efx->name, type, number); 551 } 552 553 static void ef4_set_channel_names(struct ef4_nic *efx) 554 { 555 struct ef4_channel *channel; 556 557 ef4_for_each_channel(channel, efx) 558 channel->type->get_name(channel, 559 efx->msi_context[channel->channel].name, 560 sizeof(efx->msi_context[0].name)); 561 } 562 563 static int ef4_probe_channels(struct ef4_nic *efx) 564 { 565 struct ef4_channel *channel; 566 int rc; 567 568 /* Restart special buffer allocation */ 569 efx->next_buffer_table = 0; 570 571 /* Probe channels in reverse, so that any 'extra' channels 572 * use the start of the buffer table. This allows the traffic 573 * channels to be resized without moving them or wasting the 574 * entries before them. 575 */ 576 ef4_for_each_channel_rev(channel, efx) { 577 rc = ef4_probe_channel(channel); 578 if (rc) { 579 netif_err(efx, probe, efx->net_dev, 580 "failed to create channel %d\n", 581 channel->channel); 582 goto fail; 583 } 584 } 585 ef4_set_channel_names(efx); 586 587 return 0; 588 589 fail: 590 ef4_remove_channels(efx); 591 return rc; 592 } 593 594 /* Channels are shutdown and reinitialised whilst the NIC is running 595 * to propagate configuration changes (mtu, checksum offload), or 596 * to clear hardware error conditions 597 */ 598 static void ef4_start_datapath(struct ef4_nic *efx) 599 { 600 netdev_features_t old_features = efx->net_dev->features; 601 bool old_rx_scatter = efx->rx_scatter; 602 struct ef4_tx_queue *tx_queue; 603 struct ef4_rx_queue *rx_queue; 604 struct ef4_channel *channel; 605 size_t rx_buf_len; 606 607 /* Calculate the rx buffer allocation parameters required to 608 * support the current MTU, including padding for header 609 * alignment and overruns. 610 */ 611 efx->rx_dma_len = (efx->rx_prefix_size + 612 EF4_MAX_FRAME_LEN(efx->net_dev->mtu) + 613 efx->type->rx_buffer_padding); 614 rx_buf_len = (sizeof(struct ef4_rx_page_state) + 615 efx->rx_ip_align + efx->rx_dma_len); 616 if (rx_buf_len <= PAGE_SIZE) { 617 efx->rx_scatter = efx->type->always_rx_scatter; 618 efx->rx_buffer_order = 0; 619 } else if (efx->type->can_rx_scatter) { 620 BUILD_BUG_ON(EF4_RX_USR_BUF_SIZE % L1_CACHE_BYTES); 621 BUILD_BUG_ON(sizeof(struct ef4_rx_page_state) + 622 2 * ALIGN(NET_IP_ALIGN + EF4_RX_USR_BUF_SIZE, 623 EF4_RX_BUF_ALIGNMENT) > 624 PAGE_SIZE); 625 efx->rx_scatter = true; 626 efx->rx_dma_len = EF4_RX_USR_BUF_SIZE; 627 efx->rx_buffer_order = 0; 628 } else { 629 efx->rx_scatter = false; 630 efx->rx_buffer_order = get_order(rx_buf_len); 631 } 632 633 ef4_rx_config_page_split(efx); 634 if (efx->rx_buffer_order) 635 netif_dbg(efx, drv, efx->net_dev, 636 "RX buf len=%u; page order=%u batch=%u\n", 637 efx->rx_dma_len, efx->rx_buffer_order, 638 efx->rx_pages_per_batch); 639 else 640 netif_dbg(efx, drv, efx->net_dev, 641 "RX buf len=%u step=%u bpp=%u; page batch=%u\n", 642 efx->rx_dma_len, efx->rx_page_buf_step, 643 efx->rx_bufs_per_page, efx->rx_pages_per_batch); 644 645 /* Restore previously fixed features in hw_features and remove 646 * features which are fixed now 647 */ 648 efx->net_dev->hw_features |= efx->net_dev->features; 649 efx->net_dev->hw_features &= ~efx->fixed_features; 650 efx->net_dev->features |= efx->fixed_features; 651 if (efx->net_dev->features != old_features) 652 netdev_features_change(efx->net_dev); 653 654 /* RX filters may also have scatter-enabled flags */ 655 if (efx->rx_scatter != old_rx_scatter) 656 efx->type->filter_update_rx_scatter(efx); 657 658 /* We must keep at least one descriptor in a TX ring empty. 659 * We could avoid this when the queue size does not exactly 660 * match the hardware ring size, but it's not that important. 661 * Therefore we stop the queue when one more skb might fill 662 * the ring completely. We wake it when half way back to 663 * empty. 664 */ 665 efx->txq_stop_thresh = efx->txq_entries - ef4_tx_max_skb_descs(efx); 666 efx->txq_wake_thresh = efx->txq_stop_thresh / 2; 667 668 /* Initialise the channels */ 669 ef4_for_each_channel(channel, efx) { 670 ef4_for_each_channel_tx_queue(tx_queue, channel) { 671 ef4_init_tx_queue(tx_queue); 672 atomic_inc(&efx->active_queues); 673 } 674 675 ef4_for_each_channel_rx_queue(rx_queue, channel) { 676 ef4_init_rx_queue(rx_queue); 677 atomic_inc(&efx->active_queues); 678 ef4_stop_eventq(channel); 679 ef4_fast_push_rx_descriptors(rx_queue, false); 680 ef4_start_eventq(channel); 681 } 682 683 WARN_ON(channel->rx_pkt_n_frags); 684 } 685 686 if (netif_device_present(efx->net_dev)) 687 netif_tx_wake_all_queues(efx->net_dev); 688 } 689 690 static void ef4_stop_datapath(struct ef4_nic *efx) 691 { 692 struct ef4_channel *channel; 693 struct ef4_tx_queue *tx_queue; 694 struct ef4_rx_queue *rx_queue; 695 int rc; 696 697 EF4_ASSERT_RESET_SERIALISED(efx); 698 BUG_ON(efx->port_enabled); 699 700 /* Stop RX refill */ 701 ef4_for_each_channel(channel, efx) { 702 ef4_for_each_channel_rx_queue(rx_queue, channel) 703 rx_queue->refill_enabled = false; 704 } 705 706 ef4_for_each_channel(channel, efx) { 707 /* RX packet processing is pipelined, so wait for the 708 * NAPI handler to complete. At least event queue 0 709 * might be kept active by non-data events, so don't 710 * use napi_synchronize() but actually disable NAPI 711 * temporarily. 712 */ 713 if (ef4_channel_has_rx_queue(channel)) { 714 ef4_stop_eventq(channel); 715 ef4_start_eventq(channel); 716 } 717 } 718 719 rc = efx->type->fini_dmaq(efx); 720 if (rc && EF4_WORKAROUND_7803(efx)) { 721 /* Schedule a reset to recover from the flush failure. The 722 * descriptor caches reference memory we're about to free, 723 * but falcon_reconfigure_mac_wrapper() won't reconnect 724 * the MACs because of the pending reset. 725 */ 726 netif_err(efx, drv, efx->net_dev, 727 "Resetting to recover from flush failure\n"); 728 ef4_schedule_reset(efx, RESET_TYPE_ALL); 729 } else if (rc) { 730 netif_err(efx, drv, efx->net_dev, "failed to flush queues\n"); 731 } else { 732 netif_dbg(efx, drv, efx->net_dev, 733 "successfully flushed all queues\n"); 734 } 735 736 ef4_for_each_channel(channel, efx) { 737 ef4_for_each_channel_rx_queue(rx_queue, channel) 738 ef4_fini_rx_queue(rx_queue); 739 ef4_for_each_possible_channel_tx_queue(tx_queue, channel) 740 ef4_fini_tx_queue(tx_queue); 741 } 742 } 743 744 static void ef4_remove_channel(struct ef4_channel *channel) 745 { 746 struct ef4_tx_queue *tx_queue; 747 struct ef4_rx_queue *rx_queue; 748 749 netif_dbg(channel->efx, drv, channel->efx->net_dev, 750 "destroy chan %d\n", channel->channel); 751 752 ef4_for_each_channel_rx_queue(rx_queue, channel) 753 ef4_remove_rx_queue(rx_queue); 754 ef4_for_each_possible_channel_tx_queue(tx_queue, channel) 755 ef4_remove_tx_queue(tx_queue); 756 ef4_remove_eventq(channel); 757 channel->type->post_remove(channel); 758 } 759 760 static void ef4_remove_channels(struct ef4_nic *efx) 761 { 762 struct ef4_channel *channel; 763 764 ef4_for_each_channel(channel, efx) 765 ef4_remove_channel(channel); 766 } 767 768 int 769 ef4_realloc_channels(struct ef4_nic *efx, u32 rxq_entries, u32 txq_entries) 770 { 771 struct ef4_channel *other_channel[EF4_MAX_CHANNELS], *channel; 772 u32 old_rxq_entries, old_txq_entries; 773 unsigned i, next_buffer_table = 0; 774 int rc, rc2; 775 776 rc = ef4_check_disabled(efx); 777 if (rc) 778 return rc; 779 780 /* Not all channels should be reallocated. We must avoid 781 * reallocating their buffer table entries. 782 */ 783 ef4_for_each_channel(channel, efx) { 784 struct ef4_rx_queue *rx_queue; 785 struct ef4_tx_queue *tx_queue; 786 787 if (channel->type->copy) 788 continue; 789 next_buffer_table = max(next_buffer_table, 790 channel->eventq.index + 791 channel->eventq.entries); 792 ef4_for_each_channel_rx_queue(rx_queue, channel) 793 next_buffer_table = max(next_buffer_table, 794 rx_queue->rxd.index + 795 rx_queue->rxd.entries); 796 ef4_for_each_channel_tx_queue(tx_queue, channel) 797 next_buffer_table = max(next_buffer_table, 798 tx_queue->txd.index + 799 tx_queue->txd.entries); 800 } 801 802 ef4_device_detach_sync(efx); 803 ef4_stop_all(efx); 804 ef4_soft_disable_interrupts(efx); 805 806 /* Clone channels (where possible) */ 807 memset(other_channel, 0, sizeof(other_channel)); 808 for (i = 0; i < efx->n_channels; i++) { 809 channel = efx->channel[i]; 810 if (channel->type->copy) 811 channel = channel->type->copy(channel); 812 if (!channel) { 813 rc = -ENOMEM; 814 goto out; 815 } 816 other_channel[i] = channel; 817 } 818 819 /* Swap entry counts and channel pointers */ 820 old_rxq_entries = efx->rxq_entries; 821 old_txq_entries = efx->txq_entries; 822 efx->rxq_entries = rxq_entries; 823 efx->txq_entries = txq_entries; 824 for (i = 0; i < efx->n_channels; i++) { 825 channel = efx->channel[i]; 826 efx->channel[i] = other_channel[i]; 827 other_channel[i] = channel; 828 } 829 830 /* Restart buffer table allocation */ 831 efx->next_buffer_table = next_buffer_table; 832 833 for (i = 0; i < efx->n_channels; i++) { 834 channel = efx->channel[i]; 835 if (!channel->type->copy) 836 continue; 837 rc = ef4_probe_channel(channel); 838 if (rc) 839 goto rollback; 840 ef4_init_napi_channel(efx->channel[i]); 841 } 842 843 out: 844 /* Destroy unused channel structures */ 845 for (i = 0; i < efx->n_channels; i++) { 846 channel = other_channel[i]; 847 if (channel && channel->type->copy) { 848 ef4_fini_napi_channel(channel); 849 ef4_remove_channel(channel); 850 kfree(channel); 851 } 852 } 853 854 rc2 = ef4_soft_enable_interrupts(efx); 855 if (rc2) { 856 rc = rc ? rc : rc2; 857 netif_err(efx, drv, efx->net_dev, 858 "unable to restart interrupts on channel reallocation\n"); 859 ef4_schedule_reset(efx, RESET_TYPE_DISABLE); 860 } else { 861 ef4_start_all(efx); 862 netif_device_attach(efx->net_dev); 863 } 864 return rc; 865 866 rollback: 867 /* Swap back */ 868 efx->rxq_entries = old_rxq_entries; 869 efx->txq_entries = old_txq_entries; 870 for (i = 0; i < efx->n_channels; i++) { 871 channel = efx->channel[i]; 872 efx->channel[i] = other_channel[i]; 873 other_channel[i] = channel; 874 } 875 goto out; 876 } 877 878 void ef4_schedule_slow_fill(struct ef4_rx_queue *rx_queue) 879 { 880 mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(100)); 881 } 882 883 static const struct ef4_channel_type ef4_default_channel_type = { 884 .pre_probe = ef4_channel_dummy_op_int, 885 .post_remove = ef4_channel_dummy_op_void, 886 .get_name = ef4_get_channel_name, 887 .copy = ef4_copy_channel, 888 .keep_eventq = false, 889 }; 890 891 int ef4_channel_dummy_op_int(struct ef4_channel *channel) 892 { 893 return 0; 894 } 895 896 void ef4_channel_dummy_op_void(struct ef4_channel *channel) 897 { 898 } 899 900 /************************************************************************** 901 * 902 * Port handling 903 * 904 **************************************************************************/ 905 906 /* This ensures that the kernel is kept informed (via 907 * netif_carrier_on/off) of the link status, and also maintains the 908 * link status's stop on the port's TX queue. 909 */ 910 void ef4_link_status_changed(struct ef4_nic *efx) 911 { 912 struct ef4_link_state *link_state = &efx->link_state; 913 914 /* SFC Bug 5356: A net_dev notifier is registered, so we must ensure 915 * that no events are triggered between unregister_netdev() and the 916 * driver unloading. A more general condition is that NETDEV_CHANGE 917 * can only be generated between NETDEV_UP and NETDEV_DOWN */ 918 if (!netif_running(efx->net_dev)) 919 return; 920 921 if (link_state->up != netif_carrier_ok(efx->net_dev)) { 922 efx->n_link_state_changes++; 923 924 if (link_state->up) 925 netif_carrier_on(efx->net_dev); 926 else 927 netif_carrier_off(efx->net_dev); 928 } 929 930 /* Status message for kernel log */ 931 if (link_state->up) 932 netif_info(efx, link, efx->net_dev, 933 "link up at %uMbps %s-duplex (MTU %d)\n", 934 link_state->speed, link_state->fd ? "full" : "half", 935 efx->net_dev->mtu); 936 else 937 netif_info(efx, link, efx->net_dev, "link down\n"); 938 } 939 940 void ef4_link_set_advertising(struct ef4_nic *efx, u32 advertising) 941 { 942 efx->link_advertising = advertising; 943 if (advertising) { 944 if (advertising & ADVERTISED_Pause) 945 efx->wanted_fc |= (EF4_FC_TX | EF4_FC_RX); 946 else 947 efx->wanted_fc &= ~(EF4_FC_TX | EF4_FC_RX); 948 if (advertising & ADVERTISED_Asym_Pause) 949 efx->wanted_fc ^= EF4_FC_TX; 950 } 951 } 952 953 void ef4_link_set_wanted_fc(struct ef4_nic *efx, u8 wanted_fc) 954 { 955 efx->wanted_fc = wanted_fc; 956 if (efx->link_advertising) { 957 if (wanted_fc & EF4_FC_RX) 958 efx->link_advertising |= (ADVERTISED_Pause | 959 ADVERTISED_Asym_Pause); 960 else 961 efx->link_advertising &= ~(ADVERTISED_Pause | 962 ADVERTISED_Asym_Pause); 963 if (wanted_fc & EF4_FC_TX) 964 efx->link_advertising ^= ADVERTISED_Asym_Pause; 965 } 966 } 967 968 static void ef4_fini_port(struct ef4_nic *efx); 969 970 /* We assume that efx->type->reconfigure_mac will always try to sync RX 971 * filters and therefore needs to read-lock the filter table against freeing 972 */ 973 void ef4_mac_reconfigure(struct ef4_nic *efx) 974 { 975 down_read(&efx->filter_sem); 976 efx->type->reconfigure_mac(efx); 977 up_read(&efx->filter_sem); 978 } 979 980 /* Push loopback/power/transmit disable settings to the PHY, and reconfigure 981 * the MAC appropriately. All other PHY configuration changes are pushed 982 * through phy_op->set_link_ksettings(), and pushed asynchronously to the MAC 983 * through ef4_monitor(). 984 * 985 * Callers must hold the mac_lock 986 */ 987 int __ef4_reconfigure_port(struct ef4_nic *efx) 988 { 989 enum ef4_phy_mode phy_mode; 990 int rc; 991 992 WARN_ON(!mutex_is_locked(&efx->mac_lock)); 993 994 /* Disable PHY transmit in mac level loopbacks */ 995 phy_mode = efx->phy_mode; 996 if (LOOPBACK_INTERNAL(efx)) 997 efx->phy_mode |= PHY_MODE_TX_DISABLED; 998 else 999 efx->phy_mode &= ~PHY_MODE_TX_DISABLED; 1000 1001 rc = efx->type->reconfigure_port(efx); 1002 1003 if (rc) 1004 efx->phy_mode = phy_mode; 1005 1006 return rc; 1007 } 1008 1009 /* Reinitialise the MAC to pick up new PHY settings, even if the port is 1010 * disabled. */ 1011 int ef4_reconfigure_port(struct ef4_nic *efx) 1012 { 1013 int rc; 1014 1015 EF4_ASSERT_RESET_SERIALISED(efx); 1016 1017 mutex_lock(&efx->mac_lock); 1018 rc = __ef4_reconfigure_port(efx); 1019 mutex_unlock(&efx->mac_lock); 1020 1021 return rc; 1022 } 1023 1024 /* Asynchronous work item for changing MAC promiscuity and multicast 1025 * hash. Avoid a drain/rx_ingress enable by reconfiguring the current 1026 * MAC directly. */ 1027 static void ef4_mac_work(struct work_struct *data) 1028 { 1029 struct ef4_nic *efx = container_of(data, struct ef4_nic, mac_work); 1030 1031 mutex_lock(&efx->mac_lock); 1032 if (efx->port_enabled) 1033 ef4_mac_reconfigure(efx); 1034 mutex_unlock(&efx->mac_lock); 1035 } 1036 1037 static int ef4_probe_port(struct ef4_nic *efx) 1038 { 1039 int rc; 1040 1041 netif_dbg(efx, probe, efx->net_dev, "create port\n"); 1042 1043 if (phy_flash_cfg) 1044 efx->phy_mode = PHY_MODE_SPECIAL; 1045 1046 /* Connect up MAC/PHY operations table */ 1047 rc = efx->type->probe_port(efx); 1048 if (rc) 1049 return rc; 1050 1051 /* Initialise MAC address to permanent address */ 1052 ether_addr_copy(efx->net_dev->dev_addr, efx->net_dev->perm_addr); 1053 1054 return 0; 1055 } 1056 1057 static int ef4_init_port(struct ef4_nic *efx) 1058 { 1059 int rc; 1060 1061 netif_dbg(efx, drv, efx->net_dev, "init port\n"); 1062 1063 mutex_lock(&efx->mac_lock); 1064 1065 rc = efx->phy_op->init(efx); 1066 if (rc) 1067 goto fail1; 1068 1069 efx->port_initialized = true; 1070 1071 /* Reconfigure the MAC before creating dma queues (required for 1072 * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */ 1073 ef4_mac_reconfigure(efx); 1074 1075 /* Ensure the PHY advertises the correct flow control settings */ 1076 rc = efx->phy_op->reconfigure(efx); 1077 if (rc && rc != -EPERM) 1078 goto fail2; 1079 1080 mutex_unlock(&efx->mac_lock); 1081 return 0; 1082 1083 fail2: 1084 efx->phy_op->fini(efx); 1085 fail1: 1086 mutex_unlock(&efx->mac_lock); 1087 return rc; 1088 } 1089 1090 static void ef4_start_port(struct ef4_nic *efx) 1091 { 1092 netif_dbg(efx, ifup, efx->net_dev, "start port\n"); 1093 BUG_ON(efx->port_enabled); 1094 1095 mutex_lock(&efx->mac_lock); 1096 efx->port_enabled = true; 1097 1098 /* Ensure MAC ingress/egress is enabled */ 1099 ef4_mac_reconfigure(efx); 1100 1101 mutex_unlock(&efx->mac_lock); 1102 } 1103 1104 /* Cancel work for MAC reconfiguration, periodic hardware monitoring 1105 * and the async self-test, wait for them to finish and prevent them 1106 * being scheduled again. This doesn't cover online resets, which 1107 * should only be cancelled when removing the device. 1108 */ 1109 static void ef4_stop_port(struct ef4_nic *efx) 1110 { 1111 netif_dbg(efx, ifdown, efx->net_dev, "stop port\n"); 1112 1113 EF4_ASSERT_RESET_SERIALISED(efx); 1114 1115 mutex_lock(&efx->mac_lock); 1116 efx->port_enabled = false; 1117 mutex_unlock(&efx->mac_lock); 1118 1119 /* Serialise against ef4_set_multicast_list() */ 1120 netif_addr_lock_bh(efx->net_dev); 1121 netif_addr_unlock_bh(efx->net_dev); 1122 1123 cancel_delayed_work_sync(&efx->monitor_work); 1124 ef4_selftest_async_cancel(efx); 1125 cancel_work_sync(&efx->mac_work); 1126 } 1127 1128 static void ef4_fini_port(struct ef4_nic *efx) 1129 { 1130 netif_dbg(efx, drv, efx->net_dev, "shut down port\n"); 1131 1132 if (!efx->port_initialized) 1133 return; 1134 1135 efx->phy_op->fini(efx); 1136 efx->port_initialized = false; 1137 1138 efx->link_state.up = false; 1139 ef4_link_status_changed(efx); 1140 } 1141 1142 static void ef4_remove_port(struct ef4_nic *efx) 1143 { 1144 netif_dbg(efx, drv, efx->net_dev, "destroying port\n"); 1145 1146 efx->type->remove_port(efx); 1147 } 1148 1149 /************************************************************************** 1150 * 1151 * NIC handling 1152 * 1153 **************************************************************************/ 1154 1155 static LIST_HEAD(ef4_primary_list); 1156 static LIST_HEAD(ef4_unassociated_list); 1157 1158 static bool ef4_same_controller(struct ef4_nic *left, struct ef4_nic *right) 1159 { 1160 return left->type == right->type && 1161 left->vpd_sn && right->vpd_sn && 1162 !strcmp(left->vpd_sn, right->vpd_sn); 1163 } 1164 1165 static void ef4_associate(struct ef4_nic *efx) 1166 { 1167 struct ef4_nic *other, *next; 1168 1169 if (efx->primary == efx) { 1170 /* Adding primary function; look for secondaries */ 1171 1172 netif_dbg(efx, probe, efx->net_dev, "adding to primary list\n"); 1173 list_add_tail(&efx->node, &ef4_primary_list); 1174 1175 list_for_each_entry_safe(other, next, &ef4_unassociated_list, 1176 node) { 1177 if (ef4_same_controller(efx, other)) { 1178 list_del(&other->node); 1179 netif_dbg(other, probe, other->net_dev, 1180 "moving to secondary list of %s %s\n", 1181 pci_name(efx->pci_dev), 1182 efx->net_dev->name); 1183 list_add_tail(&other->node, 1184 &efx->secondary_list); 1185 other->primary = efx; 1186 } 1187 } 1188 } else { 1189 /* Adding secondary function; look for primary */ 1190 1191 list_for_each_entry(other, &ef4_primary_list, node) { 1192 if (ef4_same_controller(efx, other)) { 1193 netif_dbg(efx, probe, efx->net_dev, 1194 "adding to secondary list of %s %s\n", 1195 pci_name(other->pci_dev), 1196 other->net_dev->name); 1197 list_add_tail(&efx->node, 1198 &other->secondary_list); 1199 efx->primary = other; 1200 return; 1201 } 1202 } 1203 1204 netif_dbg(efx, probe, efx->net_dev, 1205 "adding to unassociated list\n"); 1206 list_add_tail(&efx->node, &ef4_unassociated_list); 1207 } 1208 } 1209 1210 static void ef4_dissociate(struct ef4_nic *efx) 1211 { 1212 struct ef4_nic *other, *next; 1213 1214 list_del(&efx->node); 1215 efx->primary = NULL; 1216 1217 list_for_each_entry_safe(other, next, &efx->secondary_list, node) { 1218 list_del(&other->node); 1219 netif_dbg(other, probe, other->net_dev, 1220 "moving to unassociated list\n"); 1221 list_add_tail(&other->node, &ef4_unassociated_list); 1222 other->primary = NULL; 1223 } 1224 } 1225 1226 /* This configures the PCI device to enable I/O and DMA. */ 1227 static int ef4_init_io(struct ef4_nic *efx) 1228 { 1229 struct pci_dev *pci_dev = efx->pci_dev; 1230 dma_addr_t dma_mask = efx->type->max_dma_mask; 1231 unsigned int mem_map_size = efx->type->mem_map_size(efx); 1232 int rc, bar; 1233 1234 netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n"); 1235 1236 bar = efx->type->mem_bar; 1237 1238 rc = pci_enable_device(pci_dev); 1239 if (rc) { 1240 netif_err(efx, probe, efx->net_dev, 1241 "failed to enable PCI device\n"); 1242 goto fail1; 1243 } 1244 1245 pci_set_master(pci_dev); 1246 1247 /* Set the PCI DMA mask. Try all possibilities from our 1248 * genuine mask down to 32 bits, because some architectures 1249 * (e.g. x86_64 with iommu_sac_force set) will allow 40 bit 1250 * masks event though they reject 46 bit masks. 1251 */ 1252 while (dma_mask > 0x7fffffffUL) { 1253 rc = dma_set_mask_and_coherent(&pci_dev->dev, dma_mask); 1254 if (rc == 0) 1255 break; 1256 dma_mask >>= 1; 1257 } 1258 if (rc) { 1259 netif_err(efx, probe, efx->net_dev, 1260 "could not find a suitable DMA mask\n"); 1261 goto fail2; 1262 } 1263 netif_dbg(efx, probe, efx->net_dev, 1264 "using DMA mask %llx\n", (unsigned long long) dma_mask); 1265 1266 efx->membase_phys = pci_resource_start(efx->pci_dev, bar); 1267 rc = pci_request_region(pci_dev, bar, "sfc"); 1268 if (rc) { 1269 netif_err(efx, probe, efx->net_dev, 1270 "request for memory BAR failed\n"); 1271 rc = -EIO; 1272 goto fail3; 1273 } 1274 efx->membase = ioremap_nocache(efx->membase_phys, mem_map_size); 1275 if (!efx->membase) { 1276 netif_err(efx, probe, efx->net_dev, 1277 "could not map memory BAR at %llx+%x\n", 1278 (unsigned long long)efx->membase_phys, mem_map_size); 1279 rc = -ENOMEM; 1280 goto fail4; 1281 } 1282 netif_dbg(efx, probe, efx->net_dev, 1283 "memory BAR at %llx+%x (virtual %p)\n", 1284 (unsigned long long)efx->membase_phys, mem_map_size, 1285 efx->membase); 1286 1287 return 0; 1288 1289 fail4: 1290 pci_release_region(efx->pci_dev, bar); 1291 fail3: 1292 efx->membase_phys = 0; 1293 fail2: 1294 pci_disable_device(efx->pci_dev); 1295 fail1: 1296 return rc; 1297 } 1298 1299 static void ef4_fini_io(struct ef4_nic *efx) 1300 { 1301 int bar; 1302 1303 netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n"); 1304 1305 if (efx->membase) { 1306 iounmap(efx->membase); 1307 efx->membase = NULL; 1308 } 1309 1310 if (efx->membase_phys) { 1311 bar = efx->type->mem_bar; 1312 pci_release_region(efx->pci_dev, bar); 1313 efx->membase_phys = 0; 1314 } 1315 1316 /* Don't disable bus-mastering if VFs are assigned */ 1317 if (!pci_vfs_assigned(efx->pci_dev)) 1318 pci_disable_device(efx->pci_dev); 1319 } 1320 1321 void ef4_set_default_rx_indir_table(struct ef4_nic *efx) 1322 { 1323 size_t i; 1324 1325 for (i = 0; i < ARRAY_SIZE(efx->rx_indir_table); i++) 1326 efx->rx_indir_table[i] = 1327 ethtool_rxfh_indir_default(i, efx->rss_spread); 1328 } 1329 1330 static unsigned int ef4_wanted_parallelism(struct ef4_nic *efx) 1331 { 1332 cpumask_var_t thread_mask; 1333 unsigned int count; 1334 int cpu; 1335 1336 if (rss_cpus) { 1337 count = rss_cpus; 1338 } else { 1339 if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) { 1340 netif_warn(efx, probe, efx->net_dev, 1341 "RSS disabled due to allocation failure\n"); 1342 return 1; 1343 } 1344 1345 count = 0; 1346 for_each_online_cpu(cpu) { 1347 if (!cpumask_test_cpu(cpu, thread_mask)) { 1348 ++count; 1349 cpumask_or(thread_mask, thread_mask, 1350 topology_sibling_cpumask(cpu)); 1351 } 1352 } 1353 1354 free_cpumask_var(thread_mask); 1355 } 1356 1357 if (count > EF4_MAX_RX_QUEUES) { 1358 netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn, 1359 "Reducing number of rx queues from %u to %u.\n", 1360 count, EF4_MAX_RX_QUEUES); 1361 count = EF4_MAX_RX_QUEUES; 1362 } 1363 1364 return count; 1365 } 1366 1367 /* Probe the number and type of interrupts we are able to obtain, and 1368 * the resulting numbers of channels and RX queues. 1369 */ 1370 static int ef4_probe_interrupts(struct ef4_nic *efx) 1371 { 1372 unsigned int extra_channels = 0; 1373 unsigned int i, j; 1374 int rc; 1375 1376 for (i = 0; i < EF4_MAX_EXTRA_CHANNELS; i++) 1377 if (efx->extra_channel_type[i]) 1378 ++extra_channels; 1379 1380 if (efx->interrupt_mode == EF4_INT_MODE_MSIX) { 1381 struct msix_entry xentries[EF4_MAX_CHANNELS]; 1382 unsigned int n_channels; 1383 1384 n_channels = ef4_wanted_parallelism(efx); 1385 if (ef4_separate_tx_channels) 1386 n_channels *= 2; 1387 n_channels += extra_channels; 1388 n_channels = min(n_channels, efx->max_channels); 1389 1390 for (i = 0; i < n_channels; i++) 1391 xentries[i].entry = i; 1392 rc = pci_enable_msix_range(efx->pci_dev, 1393 xentries, 1, n_channels); 1394 if (rc < 0) { 1395 /* Fall back to single channel MSI */ 1396 efx->interrupt_mode = EF4_INT_MODE_MSI; 1397 netif_err(efx, drv, efx->net_dev, 1398 "could not enable MSI-X\n"); 1399 } else if (rc < n_channels) { 1400 netif_err(efx, drv, efx->net_dev, 1401 "WARNING: Insufficient MSI-X vectors" 1402 " available (%d < %u).\n", rc, n_channels); 1403 netif_err(efx, drv, efx->net_dev, 1404 "WARNING: Performance may be reduced.\n"); 1405 n_channels = rc; 1406 } 1407 1408 if (rc > 0) { 1409 efx->n_channels = n_channels; 1410 if (n_channels > extra_channels) 1411 n_channels -= extra_channels; 1412 if (ef4_separate_tx_channels) { 1413 efx->n_tx_channels = min(max(n_channels / 2, 1414 1U), 1415 efx->max_tx_channels); 1416 efx->n_rx_channels = max(n_channels - 1417 efx->n_tx_channels, 1418 1U); 1419 } else { 1420 efx->n_tx_channels = min(n_channels, 1421 efx->max_tx_channels); 1422 efx->n_rx_channels = n_channels; 1423 } 1424 for (i = 0; i < efx->n_channels; i++) 1425 ef4_get_channel(efx, i)->irq = 1426 xentries[i].vector; 1427 } 1428 } 1429 1430 /* Try single interrupt MSI */ 1431 if (efx->interrupt_mode == EF4_INT_MODE_MSI) { 1432 efx->n_channels = 1; 1433 efx->n_rx_channels = 1; 1434 efx->n_tx_channels = 1; 1435 rc = pci_enable_msi(efx->pci_dev); 1436 if (rc == 0) { 1437 ef4_get_channel(efx, 0)->irq = efx->pci_dev->irq; 1438 } else { 1439 netif_err(efx, drv, efx->net_dev, 1440 "could not enable MSI\n"); 1441 efx->interrupt_mode = EF4_INT_MODE_LEGACY; 1442 } 1443 } 1444 1445 /* Assume legacy interrupts */ 1446 if (efx->interrupt_mode == EF4_INT_MODE_LEGACY) { 1447 efx->n_channels = 1 + (ef4_separate_tx_channels ? 1 : 0); 1448 efx->n_rx_channels = 1; 1449 efx->n_tx_channels = 1; 1450 efx->legacy_irq = efx->pci_dev->irq; 1451 } 1452 1453 /* Assign extra channels if possible */ 1454 j = efx->n_channels; 1455 for (i = 0; i < EF4_MAX_EXTRA_CHANNELS; i++) { 1456 if (!efx->extra_channel_type[i]) 1457 continue; 1458 if (efx->interrupt_mode != EF4_INT_MODE_MSIX || 1459 efx->n_channels <= extra_channels) { 1460 efx->extra_channel_type[i]->handle_no_channel(efx); 1461 } else { 1462 --j; 1463 ef4_get_channel(efx, j)->type = 1464 efx->extra_channel_type[i]; 1465 } 1466 } 1467 1468 efx->rss_spread = efx->n_rx_channels; 1469 1470 return 0; 1471 } 1472 1473 static int ef4_soft_enable_interrupts(struct ef4_nic *efx) 1474 { 1475 struct ef4_channel *channel, *end_channel; 1476 int rc; 1477 1478 BUG_ON(efx->state == STATE_DISABLED); 1479 1480 efx->irq_soft_enabled = true; 1481 smp_wmb(); 1482 1483 ef4_for_each_channel(channel, efx) { 1484 if (!channel->type->keep_eventq) { 1485 rc = ef4_init_eventq(channel); 1486 if (rc) 1487 goto fail; 1488 } 1489 ef4_start_eventq(channel); 1490 } 1491 1492 return 0; 1493 fail: 1494 end_channel = channel; 1495 ef4_for_each_channel(channel, efx) { 1496 if (channel == end_channel) 1497 break; 1498 ef4_stop_eventq(channel); 1499 if (!channel->type->keep_eventq) 1500 ef4_fini_eventq(channel); 1501 } 1502 1503 return rc; 1504 } 1505 1506 static void ef4_soft_disable_interrupts(struct ef4_nic *efx) 1507 { 1508 struct ef4_channel *channel; 1509 1510 if (efx->state == STATE_DISABLED) 1511 return; 1512 1513 efx->irq_soft_enabled = false; 1514 smp_wmb(); 1515 1516 if (efx->legacy_irq) 1517 synchronize_irq(efx->legacy_irq); 1518 1519 ef4_for_each_channel(channel, efx) { 1520 if (channel->irq) 1521 synchronize_irq(channel->irq); 1522 1523 ef4_stop_eventq(channel); 1524 if (!channel->type->keep_eventq) 1525 ef4_fini_eventq(channel); 1526 } 1527 } 1528 1529 static int ef4_enable_interrupts(struct ef4_nic *efx) 1530 { 1531 struct ef4_channel *channel, *end_channel; 1532 int rc; 1533 1534 BUG_ON(efx->state == STATE_DISABLED); 1535 1536 if (efx->eeh_disabled_legacy_irq) { 1537 enable_irq(efx->legacy_irq); 1538 efx->eeh_disabled_legacy_irq = false; 1539 } 1540 1541 efx->type->irq_enable_master(efx); 1542 1543 ef4_for_each_channel(channel, efx) { 1544 if (channel->type->keep_eventq) { 1545 rc = ef4_init_eventq(channel); 1546 if (rc) 1547 goto fail; 1548 } 1549 } 1550 1551 rc = ef4_soft_enable_interrupts(efx); 1552 if (rc) 1553 goto fail; 1554 1555 return 0; 1556 1557 fail: 1558 end_channel = channel; 1559 ef4_for_each_channel(channel, efx) { 1560 if (channel == end_channel) 1561 break; 1562 if (channel->type->keep_eventq) 1563 ef4_fini_eventq(channel); 1564 } 1565 1566 efx->type->irq_disable_non_ev(efx); 1567 1568 return rc; 1569 } 1570 1571 static void ef4_disable_interrupts(struct ef4_nic *efx) 1572 { 1573 struct ef4_channel *channel; 1574 1575 ef4_soft_disable_interrupts(efx); 1576 1577 ef4_for_each_channel(channel, efx) { 1578 if (channel->type->keep_eventq) 1579 ef4_fini_eventq(channel); 1580 } 1581 1582 efx->type->irq_disable_non_ev(efx); 1583 } 1584 1585 static void ef4_remove_interrupts(struct ef4_nic *efx) 1586 { 1587 struct ef4_channel *channel; 1588 1589 /* Remove MSI/MSI-X interrupts */ 1590 ef4_for_each_channel(channel, efx) 1591 channel->irq = 0; 1592 pci_disable_msi(efx->pci_dev); 1593 pci_disable_msix(efx->pci_dev); 1594 1595 /* Remove legacy interrupt */ 1596 efx->legacy_irq = 0; 1597 } 1598 1599 static void ef4_set_channels(struct ef4_nic *efx) 1600 { 1601 struct ef4_channel *channel; 1602 struct ef4_tx_queue *tx_queue; 1603 1604 efx->tx_channel_offset = 1605 ef4_separate_tx_channels ? 1606 efx->n_channels - efx->n_tx_channels : 0; 1607 1608 /* We need to mark which channels really have RX and TX 1609 * queues, and adjust the TX queue numbers if we have separate 1610 * RX-only and TX-only channels. 1611 */ 1612 ef4_for_each_channel(channel, efx) { 1613 if (channel->channel < efx->n_rx_channels) 1614 channel->rx_queue.core_index = channel->channel; 1615 else 1616 channel->rx_queue.core_index = -1; 1617 1618 ef4_for_each_channel_tx_queue(tx_queue, channel) 1619 tx_queue->queue -= (efx->tx_channel_offset * 1620 EF4_TXQ_TYPES); 1621 } 1622 } 1623 1624 static int ef4_probe_nic(struct ef4_nic *efx) 1625 { 1626 int rc; 1627 1628 netif_dbg(efx, probe, efx->net_dev, "creating NIC\n"); 1629 1630 /* Carry out hardware-type specific initialisation */ 1631 rc = efx->type->probe(efx); 1632 if (rc) 1633 return rc; 1634 1635 do { 1636 if (!efx->max_channels || !efx->max_tx_channels) { 1637 netif_err(efx, drv, efx->net_dev, 1638 "Insufficient resources to allocate" 1639 " any channels\n"); 1640 rc = -ENOSPC; 1641 goto fail1; 1642 } 1643 1644 /* Determine the number of channels and queues by trying 1645 * to hook in MSI-X interrupts. 1646 */ 1647 rc = ef4_probe_interrupts(efx); 1648 if (rc) 1649 goto fail1; 1650 1651 ef4_set_channels(efx); 1652 1653 /* dimension_resources can fail with EAGAIN */ 1654 rc = efx->type->dimension_resources(efx); 1655 if (rc != 0 && rc != -EAGAIN) 1656 goto fail2; 1657 1658 if (rc == -EAGAIN) 1659 /* try again with new max_channels */ 1660 ef4_remove_interrupts(efx); 1661 1662 } while (rc == -EAGAIN); 1663 1664 if (efx->n_channels > 1) 1665 netdev_rss_key_fill(&efx->rx_hash_key, 1666 sizeof(efx->rx_hash_key)); 1667 ef4_set_default_rx_indir_table(efx); 1668 1669 netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels); 1670 netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels); 1671 1672 /* Initialise the interrupt moderation settings */ 1673 efx->irq_mod_step_us = DIV_ROUND_UP(efx->timer_quantum_ns, 1000); 1674 ef4_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true, 1675 true); 1676 1677 return 0; 1678 1679 fail2: 1680 ef4_remove_interrupts(efx); 1681 fail1: 1682 efx->type->remove(efx); 1683 return rc; 1684 } 1685 1686 static void ef4_remove_nic(struct ef4_nic *efx) 1687 { 1688 netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n"); 1689 1690 ef4_remove_interrupts(efx); 1691 efx->type->remove(efx); 1692 } 1693 1694 static int ef4_probe_filters(struct ef4_nic *efx) 1695 { 1696 int rc; 1697 1698 spin_lock_init(&efx->filter_lock); 1699 init_rwsem(&efx->filter_sem); 1700 mutex_lock(&efx->mac_lock); 1701 down_write(&efx->filter_sem); 1702 rc = efx->type->filter_table_probe(efx); 1703 if (rc) 1704 goto out_unlock; 1705 1706 #ifdef CONFIG_RFS_ACCEL 1707 if (efx->type->offload_features & NETIF_F_NTUPLE) { 1708 struct ef4_channel *channel; 1709 int i, success = 1; 1710 1711 ef4_for_each_channel(channel, efx) { 1712 channel->rps_flow_id = 1713 kcalloc(efx->type->max_rx_ip_filters, 1714 sizeof(*channel->rps_flow_id), 1715 GFP_KERNEL); 1716 if (!channel->rps_flow_id) 1717 success = 0; 1718 else 1719 for (i = 0; 1720 i < efx->type->max_rx_ip_filters; 1721 ++i) 1722 channel->rps_flow_id[i] = 1723 RPS_FLOW_ID_INVALID; 1724 } 1725 1726 if (!success) { 1727 ef4_for_each_channel(channel, efx) 1728 kfree(channel->rps_flow_id); 1729 efx->type->filter_table_remove(efx); 1730 rc = -ENOMEM; 1731 goto out_unlock; 1732 } 1733 1734 efx->rps_expire_index = efx->rps_expire_channel = 0; 1735 } 1736 #endif 1737 out_unlock: 1738 up_write(&efx->filter_sem); 1739 mutex_unlock(&efx->mac_lock); 1740 return rc; 1741 } 1742 1743 static void ef4_remove_filters(struct ef4_nic *efx) 1744 { 1745 #ifdef CONFIG_RFS_ACCEL 1746 struct ef4_channel *channel; 1747 1748 ef4_for_each_channel(channel, efx) 1749 kfree(channel->rps_flow_id); 1750 #endif 1751 down_write(&efx->filter_sem); 1752 efx->type->filter_table_remove(efx); 1753 up_write(&efx->filter_sem); 1754 } 1755 1756 static void ef4_restore_filters(struct ef4_nic *efx) 1757 { 1758 down_read(&efx->filter_sem); 1759 efx->type->filter_table_restore(efx); 1760 up_read(&efx->filter_sem); 1761 } 1762 1763 /************************************************************************** 1764 * 1765 * NIC startup/shutdown 1766 * 1767 *************************************************************************/ 1768 1769 static int ef4_probe_all(struct ef4_nic *efx) 1770 { 1771 int rc; 1772 1773 rc = ef4_probe_nic(efx); 1774 if (rc) { 1775 netif_err(efx, probe, efx->net_dev, "failed to create NIC\n"); 1776 goto fail1; 1777 } 1778 1779 rc = ef4_probe_port(efx); 1780 if (rc) { 1781 netif_err(efx, probe, efx->net_dev, "failed to create port\n"); 1782 goto fail2; 1783 } 1784 1785 BUILD_BUG_ON(EF4_DEFAULT_DMAQ_SIZE < EF4_RXQ_MIN_ENT); 1786 if (WARN_ON(EF4_DEFAULT_DMAQ_SIZE < EF4_TXQ_MIN_ENT(efx))) { 1787 rc = -EINVAL; 1788 goto fail3; 1789 } 1790 efx->rxq_entries = efx->txq_entries = EF4_DEFAULT_DMAQ_SIZE; 1791 1792 rc = ef4_probe_filters(efx); 1793 if (rc) { 1794 netif_err(efx, probe, efx->net_dev, 1795 "failed to create filter tables\n"); 1796 goto fail4; 1797 } 1798 1799 rc = ef4_probe_channels(efx); 1800 if (rc) 1801 goto fail5; 1802 1803 return 0; 1804 1805 fail5: 1806 ef4_remove_filters(efx); 1807 fail4: 1808 fail3: 1809 ef4_remove_port(efx); 1810 fail2: 1811 ef4_remove_nic(efx); 1812 fail1: 1813 return rc; 1814 } 1815 1816 /* If the interface is supposed to be running but is not, start 1817 * the hardware and software data path, regular activity for the port 1818 * (MAC statistics, link polling, etc.) and schedule the port to be 1819 * reconfigured. Interrupts must already be enabled. This function 1820 * is safe to call multiple times, so long as the NIC is not disabled. 1821 * Requires the RTNL lock. 1822 */ 1823 static void ef4_start_all(struct ef4_nic *efx) 1824 { 1825 EF4_ASSERT_RESET_SERIALISED(efx); 1826 BUG_ON(efx->state == STATE_DISABLED); 1827 1828 /* Check that it is appropriate to restart the interface. All 1829 * of these flags are safe to read under just the rtnl lock */ 1830 if (efx->port_enabled || !netif_running(efx->net_dev) || 1831 efx->reset_pending) 1832 return; 1833 1834 ef4_start_port(efx); 1835 ef4_start_datapath(efx); 1836 1837 /* Start the hardware monitor if there is one */ 1838 if (efx->type->monitor != NULL) 1839 queue_delayed_work(efx->workqueue, &efx->monitor_work, 1840 ef4_monitor_interval); 1841 1842 efx->type->start_stats(efx); 1843 efx->type->pull_stats(efx); 1844 spin_lock_bh(&efx->stats_lock); 1845 efx->type->update_stats(efx, NULL, NULL); 1846 spin_unlock_bh(&efx->stats_lock); 1847 } 1848 1849 /* Quiesce the hardware and software data path, and regular activity 1850 * for the port without bringing the link down. Safe to call multiple 1851 * times with the NIC in almost any state, but interrupts should be 1852 * enabled. Requires the RTNL lock. 1853 */ 1854 static void ef4_stop_all(struct ef4_nic *efx) 1855 { 1856 EF4_ASSERT_RESET_SERIALISED(efx); 1857 1858 /* port_enabled can be read safely under the rtnl lock */ 1859 if (!efx->port_enabled) 1860 return; 1861 1862 /* update stats before we go down so we can accurately count 1863 * rx_nodesc_drops 1864 */ 1865 efx->type->pull_stats(efx); 1866 spin_lock_bh(&efx->stats_lock); 1867 efx->type->update_stats(efx, NULL, NULL); 1868 spin_unlock_bh(&efx->stats_lock); 1869 efx->type->stop_stats(efx); 1870 ef4_stop_port(efx); 1871 1872 /* Stop the kernel transmit interface. This is only valid if 1873 * the device is stopped or detached; otherwise the watchdog 1874 * may fire immediately. 1875 */ 1876 WARN_ON(netif_running(efx->net_dev) && 1877 netif_device_present(efx->net_dev)); 1878 netif_tx_disable(efx->net_dev); 1879 1880 ef4_stop_datapath(efx); 1881 } 1882 1883 static void ef4_remove_all(struct ef4_nic *efx) 1884 { 1885 ef4_remove_channels(efx); 1886 ef4_remove_filters(efx); 1887 ef4_remove_port(efx); 1888 ef4_remove_nic(efx); 1889 } 1890 1891 /************************************************************************** 1892 * 1893 * Interrupt moderation 1894 * 1895 **************************************************************************/ 1896 unsigned int ef4_usecs_to_ticks(struct ef4_nic *efx, unsigned int usecs) 1897 { 1898 if (usecs == 0) 1899 return 0; 1900 if (usecs * 1000 < efx->timer_quantum_ns) 1901 return 1; /* never round down to 0 */ 1902 return usecs * 1000 / efx->timer_quantum_ns; 1903 } 1904 1905 unsigned int ef4_ticks_to_usecs(struct ef4_nic *efx, unsigned int ticks) 1906 { 1907 /* We must round up when converting ticks to microseconds 1908 * because we round down when converting the other way. 1909 */ 1910 return DIV_ROUND_UP(ticks * efx->timer_quantum_ns, 1000); 1911 } 1912 1913 /* Set interrupt moderation parameters */ 1914 int ef4_init_irq_moderation(struct ef4_nic *efx, unsigned int tx_usecs, 1915 unsigned int rx_usecs, bool rx_adaptive, 1916 bool rx_may_override_tx) 1917 { 1918 struct ef4_channel *channel; 1919 unsigned int timer_max_us; 1920 1921 EF4_ASSERT_RESET_SERIALISED(efx); 1922 1923 timer_max_us = efx->timer_max_ns / 1000; 1924 1925 if (tx_usecs > timer_max_us || rx_usecs > timer_max_us) 1926 return -EINVAL; 1927 1928 if (tx_usecs != rx_usecs && efx->tx_channel_offset == 0 && 1929 !rx_may_override_tx) { 1930 netif_err(efx, drv, efx->net_dev, "Channels are shared. " 1931 "RX and TX IRQ moderation must be equal\n"); 1932 return -EINVAL; 1933 } 1934 1935 efx->irq_rx_adaptive = rx_adaptive; 1936 efx->irq_rx_moderation_us = rx_usecs; 1937 ef4_for_each_channel(channel, efx) { 1938 if (ef4_channel_has_rx_queue(channel)) 1939 channel->irq_moderation_us = rx_usecs; 1940 else if (ef4_channel_has_tx_queues(channel)) 1941 channel->irq_moderation_us = tx_usecs; 1942 } 1943 1944 return 0; 1945 } 1946 1947 void ef4_get_irq_moderation(struct ef4_nic *efx, unsigned int *tx_usecs, 1948 unsigned int *rx_usecs, bool *rx_adaptive) 1949 { 1950 *rx_adaptive = efx->irq_rx_adaptive; 1951 *rx_usecs = efx->irq_rx_moderation_us; 1952 1953 /* If channels are shared between RX and TX, so is IRQ 1954 * moderation. Otherwise, IRQ moderation is the same for all 1955 * TX channels and is not adaptive. 1956 */ 1957 if (efx->tx_channel_offset == 0) { 1958 *tx_usecs = *rx_usecs; 1959 } else { 1960 struct ef4_channel *tx_channel; 1961 1962 tx_channel = efx->channel[efx->tx_channel_offset]; 1963 *tx_usecs = tx_channel->irq_moderation_us; 1964 } 1965 } 1966 1967 /************************************************************************** 1968 * 1969 * Hardware monitor 1970 * 1971 **************************************************************************/ 1972 1973 /* Run periodically off the general workqueue */ 1974 static void ef4_monitor(struct work_struct *data) 1975 { 1976 struct ef4_nic *efx = container_of(data, struct ef4_nic, 1977 monitor_work.work); 1978 1979 netif_vdbg(efx, timer, efx->net_dev, 1980 "hardware monitor executing on CPU %d\n", 1981 raw_smp_processor_id()); 1982 BUG_ON(efx->type->monitor == NULL); 1983 1984 /* If the mac_lock is already held then it is likely a port 1985 * reconfiguration is already in place, which will likely do 1986 * most of the work of monitor() anyway. */ 1987 if (mutex_trylock(&efx->mac_lock)) { 1988 if (efx->port_enabled) 1989 efx->type->monitor(efx); 1990 mutex_unlock(&efx->mac_lock); 1991 } 1992 1993 queue_delayed_work(efx->workqueue, &efx->monitor_work, 1994 ef4_monitor_interval); 1995 } 1996 1997 /************************************************************************** 1998 * 1999 * ioctls 2000 * 2001 *************************************************************************/ 2002 2003 /* Net device ioctl 2004 * Context: process, rtnl_lock() held. 2005 */ 2006 static int ef4_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd) 2007 { 2008 struct ef4_nic *efx = netdev_priv(net_dev); 2009 struct mii_ioctl_data *data = if_mii(ifr); 2010 2011 /* Convert phy_id from older PRTAD/DEVAD format */ 2012 if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) && 2013 (data->phy_id & 0xfc00) == 0x0400) 2014 data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400; 2015 2016 return mdio_mii_ioctl(&efx->mdio, data, cmd); 2017 } 2018 2019 /************************************************************************** 2020 * 2021 * NAPI interface 2022 * 2023 **************************************************************************/ 2024 2025 static void ef4_init_napi_channel(struct ef4_channel *channel) 2026 { 2027 struct ef4_nic *efx = channel->efx; 2028 2029 channel->napi_dev = efx->net_dev; 2030 netif_napi_add(channel->napi_dev, &channel->napi_str, 2031 ef4_poll, napi_weight); 2032 } 2033 2034 static void ef4_init_napi(struct ef4_nic *efx) 2035 { 2036 struct ef4_channel *channel; 2037 2038 ef4_for_each_channel(channel, efx) 2039 ef4_init_napi_channel(channel); 2040 } 2041 2042 static void ef4_fini_napi_channel(struct ef4_channel *channel) 2043 { 2044 if (channel->napi_dev) 2045 netif_napi_del(&channel->napi_str); 2046 2047 channel->napi_dev = NULL; 2048 } 2049 2050 static void ef4_fini_napi(struct ef4_nic *efx) 2051 { 2052 struct ef4_channel *channel; 2053 2054 ef4_for_each_channel(channel, efx) 2055 ef4_fini_napi_channel(channel); 2056 } 2057 2058 /************************************************************************** 2059 * 2060 * Kernel netpoll interface 2061 * 2062 *************************************************************************/ 2063 2064 #ifdef CONFIG_NET_POLL_CONTROLLER 2065 2066 /* Although in the common case interrupts will be disabled, this is not 2067 * guaranteed. However, all our work happens inside the NAPI callback, 2068 * so no locking is required. 2069 */ 2070 static void ef4_netpoll(struct net_device *net_dev) 2071 { 2072 struct ef4_nic *efx = netdev_priv(net_dev); 2073 struct ef4_channel *channel; 2074 2075 ef4_for_each_channel(channel, efx) 2076 ef4_schedule_channel(channel); 2077 } 2078 2079 #endif 2080 2081 /************************************************************************** 2082 * 2083 * Kernel net device interface 2084 * 2085 *************************************************************************/ 2086 2087 /* Context: process, rtnl_lock() held. */ 2088 int ef4_net_open(struct net_device *net_dev) 2089 { 2090 struct ef4_nic *efx = netdev_priv(net_dev); 2091 int rc; 2092 2093 netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n", 2094 raw_smp_processor_id()); 2095 2096 rc = ef4_check_disabled(efx); 2097 if (rc) 2098 return rc; 2099 if (efx->phy_mode & PHY_MODE_SPECIAL) 2100 return -EBUSY; 2101 2102 /* Notify the kernel of the link state polled during driver load, 2103 * before the monitor starts running */ 2104 ef4_link_status_changed(efx); 2105 2106 ef4_start_all(efx); 2107 ef4_selftest_async_start(efx); 2108 return 0; 2109 } 2110 2111 /* Context: process, rtnl_lock() held. 2112 * Note that the kernel will ignore our return code; this method 2113 * should really be a void. 2114 */ 2115 int ef4_net_stop(struct net_device *net_dev) 2116 { 2117 struct ef4_nic *efx = netdev_priv(net_dev); 2118 2119 netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n", 2120 raw_smp_processor_id()); 2121 2122 /* Stop the device and flush all the channels */ 2123 ef4_stop_all(efx); 2124 2125 return 0; 2126 } 2127 2128 /* Context: process, dev_base_lock or RTNL held, non-blocking. */ 2129 static void ef4_net_stats(struct net_device *net_dev, 2130 struct rtnl_link_stats64 *stats) 2131 { 2132 struct ef4_nic *efx = netdev_priv(net_dev); 2133 2134 spin_lock_bh(&efx->stats_lock); 2135 efx->type->update_stats(efx, NULL, stats); 2136 spin_unlock_bh(&efx->stats_lock); 2137 } 2138 2139 /* Context: netif_tx_lock held, BHs disabled. */ 2140 static void ef4_watchdog(struct net_device *net_dev) 2141 { 2142 struct ef4_nic *efx = netdev_priv(net_dev); 2143 2144 netif_err(efx, tx_err, efx->net_dev, 2145 "TX stuck with port_enabled=%d: resetting channels\n", 2146 efx->port_enabled); 2147 2148 ef4_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG); 2149 } 2150 2151 2152 /* Context: process, rtnl_lock() held. */ 2153 static int ef4_change_mtu(struct net_device *net_dev, int new_mtu) 2154 { 2155 struct ef4_nic *efx = netdev_priv(net_dev); 2156 int rc; 2157 2158 rc = ef4_check_disabled(efx); 2159 if (rc) 2160 return rc; 2161 2162 netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu); 2163 2164 ef4_device_detach_sync(efx); 2165 ef4_stop_all(efx); 2166 2167 mutex_lock(&efx->mac_lock); 2168 net_dev->mtu = new_mtu; 2169 ef4_mac_reconfigure(efx); 2170 mutex_unlock(&efx->mac_lock); 2171 2172 ef4_start_all(efx); 2173 netif_device_attach(efx->net_dev); 2174 return 0; 2175 } 2176 2177 static int ef4_set_mac_address(struct net_device *net_dev, void *data) 2178 { 2179 struct ef4_nic *efx = netdev_priv(net_dev); 2180 struct sockaddr *addr = data; 2181 u8 *new_addr = addr->sa_data; 2182 u8 old_addr[6]; 2183 int rc; 2184 2185 if (!is_valid_ether_addr(new_addr)) { 2186 netif_err(efx, drv, efx->net_dev, 2187 "invalid ethernet MAC address requested: %pM\n", 2188 new_addr); 2189 return -EADDRNOTAVAIL; 2190 } 2191 2192 /* save old address */ 2193 ether_addr_copy(old_addr, net_dev->dev_addr); 2194 ether_addr_copy(net_dev->dev_addr, new_addr); 2195 if (efx->type->set_mac_address) { 2196 rc = efx->type->set_mac_address(efx); 2197 if (rc) { 2198 ether_addr_copy(net_dev->dev_addr, old_addr); 2199 return rc; 2200 } 2201 } 2202 2203 /* Reconfigure the MAC */ 2204 mutex_lock(&efx->mac_lock); 2205 ef4_mac_reconfigure(efx); 2206 mutex_unlock(&efx->mac_lock); 2207 2208 return 0; 2209 } 2210 2211 /* Context: netif_addr_lock held, BHs disabled. */ 2212 static void ef4_set_rx_mode(struct net_device *net_dev) 2213 { 2214 struct ef4_nic *efx = netdev_priv(net_dev); 2215 2216 if (efx->port_enabled) 2217 queue_work(efx->workqueue, &efx->mac_work); 2218 /* Otherwise ef4_start_port() will do this */ 2219 } 2220 2221 static int ef4_set_features(struct net_device *net_dev, netdev_features_t data) 2222 { 2223 struct ef4_nic *efx = netdev_priv(net_dev); 2224 int rc; 2225 2226 /* If disabling RX n-tuple filtering, clear existing filters */ 2227 if (net_dev->features & ~data & NETIF_F_NTUPLE) { 2228 rc = efx->type->filter_clear_rx(efx, EF4_FILTER_PRI_MANUAL); 2229 if (rc) 2230 return rc; 2231 } 2232 2233 /* If Rx VLAN filter is changed, update filters via mac_reconfigure */ 2234 if ((net_dev->features ^ data) & NETIF_F_HW_VLAN_CTAG_FILTER) { 2235 /* ef4_set_rx_mode() will schedule MAC work to update filters 2236 * when a new features are finally set in net_dev. 2237 */ 2238 ef4_set_rx_mode(net_dev); 2239 } 2240 2241 return 0; 2242 } 2243 2244 static const struct net_device_ops ef4_netdev_ops = { 2245 .ndo_open = ef4_net_open, 2246 .ndo_stop = ef4_net_stop, 2247 .ndo_get_stats64 = ef4_net_stats, 2248 .ndo_tx_timeout = ef4_watchdog, 2249 .ndo_start_xmit = ef4_hard_start_xmit, 2250 .ndo_validate_addr = eth_validate_addr, 2251 .ndo_do_ioctl = ef4_ioctl, 2252 .ndo_change_mtu = ef4_change_mtu, 2253 .ndo_set_mac_address = ef4_set_mac_address, 2254 .ndo_set_rx_mode = ef4_set_rx_mode, 2255 .ndo_set_features = ef4_set_features, 2256 #ifdef CONFIG_NET_POLL_CONTROLLER 2257 .ndo_poll_controller = ef4_netpoll, 2258 #endif 2259 .ndo_setup_tc = ef4_setup_tc, 2260 #ifdef CONFIG_RFS_ACCEL 2261 .ndo_rx_flow_steer = ef4_filter_rfs, 2262 #endif 2263 }; 2264 2265 static void ef4_update_name(struct ef4_nic *efx) 2266 { 2267 strcpy(efx->name, efx->net_dev->name); 2268 ef4_mtd_rename(efx); 2269 ef4_set_channel_names(efx); 2270 } 2271 2272 static int ef4_netdev_event(struct notifier_block *this, 2273 unsigned long event, void *ptr) 2274 { 2275 struct net_device *net_dev = netdev_notifier_info_to_dev(ptr); 2276 2277 if ((net_dev->netdev_ops == &ef4_netdev_ops) && 2278 event == NETDEV_CHANGENAME) 2279 ef4_update_name(netdev_priv(net_dev)); 2280 2281 return NOTIFY_DONE; 2282 } 2283 2284 static struct notifier_block ef4_netdev_notifier = { 2285 .notifier_call = ef4_netdev_event, 2286 }; 2287 2288 static ssize_t 2289 show_phy_type(struct device *dev, struct device_attribute *attr, char *buf) 2290 { 2291 struct ef4_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2292 return sprintf(buf, "%d\n", efx->phy_type); 2293 } 2294 static DEVICE_ATTR(phy_type, 0444, show_phy_type, NULL); 2295 2296 static int ef4_register_netdev(struct ef4_nic *efx) 2297 { 2298 struct net_device *net_dev = efx->net_dev; 2299 struct ef4_channel *channel; 2300 int rc; 2301 2302 net_dev->watchdog_timeo = 5 * HZ; 2303 net_dev->irq = efx->pci_dev->irq; 2304 net_dev->netdev_ops = &ef4_netdev_ops; 2305 net_dev->ethtool_ops = &ef4_ethtool_ops; 2306 net_dev->gso_max_segs = EF4_TSO_MAX_SEGS; 2307 net_dev->min_mtu = EF4_MIN_MTU; 2308 net_dev->max_mtu = EF4_MAX_MTU; 2309 2310 rtnl_lock(); 2311 2312 /* Enable resets to be scheduled and check whether any were 2313 * already requested. If so, the NIC is probably hosed so we 2314 * abort. 2315 */ 2316 efx->state = STATE_READY; 2317 smp_mb(); /* ensure we change state before checking reset_pending */ 2318 if (efx->reset_pending) { 2319 netif_err(efx, probe, efx->net_dev, 2320 "aborting probe due to scheduled reset\n"); 2321 rc = -EIO; 2322 goto fail_locked; 2323 } 2324 2325 rc = dev_alloc_name(net_dev, net_dev->name); 2326 if (rc < 0) 2327 goto fail_locked; 2328 ef4_update_name(efx); 2329 2330 /* Always start with carrier off; PHY events will detect the link */ 2331 netif_carrier_off(net_dev); 2332 2333 rc = register_netdevice(net_dev); 2334 if (rc) 2335 goto fail_locked; 2336 2337 ef4_for_each_channel(channel, efx) { 2338 struct ef4_tx_queue *tx_queue; 2339 ef4_for_each_channel_tx_queue(tx_queue, channel) 2340 ef4_init_tx_queue_core_txq(tx_queue); 2341 } 2342 2343 ef4_associate(efx); 2344 2345 rtnl_unlock(); 2346 2347 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2348 if (rc) { 2349 netif_err(efx, drv, efx->net_dev, 2350 "failed to init net dev attributes\n"); 2351 goto fail_registered; 2352 } 2353 return 0; 2354 2355 fail_registered: 2356 rtnl_lock(); 2357 ef4_dissociate(efx); 2358 unregister_netdevice(net_dev); 2359 fail_locked: 2360 efx->state = STATE_UNINIT; 2361 rtnl_unlock(); 2362 netif_err(efx, drv, efx->net_dev, "could not register net dev\n"); 2363 return rc; 2364 } 2365 2366 static void ef4_unregister_netdev(struct ef4_nic *efx) 2367 { 2368 if (!efx->net_dev) 2369 return; 2370 2371 BUG_ON(netdev_priv(efx->net_dev) != efx); 2372 2373 if (ef4_dev_registered(efx)) { 2374 strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name)); 2375 device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2376 unregister_netdev(efx->net_dev); 2377 } 2378 } 2379 2380 /************************************************************************** 2381 * 2382 * Device reset and suspend 2383 * 2384 **************************************************************************/ 2385 2386 /* Tears down the entire software state and most of the hardware state 2387 * before reset. */ 2388 void ef4_reset_down(struct ef4_nic *efx, enum reset_type method) 2389 { 2390 EF4_ASSERT_RESET_SERIALISED(efx); 2391 2392 ef4_stop_all(efx); 2393 ef4_disable_interrupts(efx); 2394 2395 mutex_lock(&efx->mac_lock); 2396 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 2397 method != RESET_TYPE_DATAPATH) 2398 efx->phy_op->fini(efx); 2399 efx->type->fini(efx); 2400 } 2401 2402 /* This function will always ensure that the locks acquired in 2403 * ef4_reset_down() are released. A failure return code indicates 2404 * that we were unable to reinitialise the hardware, and the 2405 * driver should be disabled. If ok is false, then the rx and tx 2406 * engines are not restarted, pending a RESET_DISABLE. */ 2407 int ef4_reset_up(struct ef4_nic *efx, enum reset_type method, bool ok) 2408 { 2409 int rc; 2410 2411 EF4_ASSERT_RESET_SERIALISED(efx); 2412 2413 /* Ensure that SRAM is initialised even if we're disabling the device */ 2414 rc = efx->type->init(efx); 2415 if (rc) { 2416 netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n"); 2417 goto fail; 2418 } 2419 2420 if (!ok) 2421 goto fail; 2422 2423 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 2424 method != RESET_TYPE_DATAPATH) { 2425 rc = efx->phy_op->init(efx); 2426 if (rc) 2427 goto fail; 2428 rc = efx->phy_op->reconfigure(efx); 2429 if (rc && rc != -EPERM) 2430 netif_err(efx, drv, efx->net_dev, 2431 "could not restore PHY settings\n"); 2432 } 2433 2434 rc = ef4_enable_interrupts(efx); 2435 if (rc) 2436 goto fail; 2437 2438 down_read(&efx->filter_sem); 2439 ef4_restore_filters(efx); 2440 up_read(&efx->filter_sem); 2441 2442 mutex_unlock(&efx->mac_lock); 2443 2444 ef4_start_all(efx); 2445 2446 return 0; 2447 2448 fail: 2449 efx->port_initialized = false; 2450 2451 mutex_unlock(&efx->mac_lock); 2452 2453 return rc; 2454 } 2455 2456 /* Reset the NIC using the specified method. Note that the reset may 2457 * fail, in which case the card will be left in an unusable state. 2458 * 2459 * Caller must hold the rtnl_lock. 2460 */ 2461 int ef4_reset(struct ef4_nic *efx, enum reset_type method) 2462 { 2463 int rc, rc2; 2464 bool disabled; 2465 2466 netif_info(efx, drv, efx->net_dev, "resetting (%s)\n", 2467 RESET_TYPE(method)); 2468 2469 ef4_device_detach_sync(efx); 2470 ef4_reset_down(efx, method); 2471 2472 rc = efx->type->reset(efx, method); 2473 if (rc) { 2474 netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n"); 2475 goto out; 2476 } 2477 2478 /* Clear flags for the scopes we covered. We assume the NIC and 2479 * driver are now quiescent so that there is no race here. 2480 */ 2481 if (method < RESET_TYPE_MAX_METHOD) 2482 efx->reset_pending &= -(1 << (method + 1)); 2483 else /* it doesn't fit into the well-ordered scope hierarchy */ 2484 __clear_bit(method, &efx->reset_pending); 2485 2486 /* Reinitialise bus-mastering, which may have been turned off before 2487 * the reset was scheduled. This is still appropriate, even in the 2488 * RESET_TYPE_DISABLE since this driver generally assumes the hardware 2489 * can respond to requests. */ 2490 pci_set_master(efx->pci_dev); 2491 2492 out: 2493 /* Leave device stopped if necessary */ 2494 disabled = rc || 2495 method == RESET_TYPE_DISABLE || 2496 method == RESET_TYPE_RECOVER_OR_DISABLE; 2497 rc2 = ef4_reset_up(efx, method, !disabled); 2498 if (rc2) { 2499 disabled = true; 2500 if (!rc) 2501 rc = rc2; 2502 } 2503 2504 if (disabled) { 2505 dev_close(efx->net_dev); 2506 netif_err(efx, drv, efx->net_dev, "has been disabled\n"); 2507 efx->state = STATE_DISABLED; 2508 } else { 2509 netif_dbg(efx, drv, efx->net_dev, "reset complete\n"); 2510 netif_device_attach(efx->net_dev); 2511 } 2512 return rc; 2513 } 2514 2515 /* Try recovery mechanisms. 2516 * For now only EEH is supported. 2517 * Returns 0 if the recovery mechanisms are unsuccessful. 2518 * Returns a non-zero value otherwise. 2519 */ 2520 int ef4_try_recovery(struct ef4_nic *efx) 2521 { 2522 #ifdef CONFIG_EEH 2523 /* A PCI error can occur and not be seen by EEH because nothing 2524 * happens on the PCI bus. In this case the driver may fail and 2525 * schedule a 'recover or reset', leading to this recovery handler. 2526 * Manually call the eeh failure check function. 2527 */ 2528 struct eeh_dev *eehdev = pci_dev_to_eeh_dev(efx->pci_dev); 2529 if (eeh_dev_check_failure(eehdev)) { 2530 /* The EEH mechanisms will handle the error and reset the 2531 * device if necessary. 2532 */ 2533 return 1; 2534 } 2535 #endif 2536 return 0; 2537 } 2538 2539 /* The worker thread exists so that code that cannot sleep can 2540 * schedule a reset for later. 2541 */ 2542 static void ef4_reset_work(struct work_struct *data) 2543 { 2544 struct ef4_nic *efx = container_of(data, struct ef4_nic, reset_work); 2545 unsigned long pending; 2546 enum reset_type method; 2547 2548 pending = ACCESS_ONCE(efx->reset_pending); 2549 method = fls(pending) - 1; 2550 2551 if ((method == RESET_TYPE_RECOVER_OR_DISABLE || 2552 method == RESET_TYPE_RECOVER_OR_ALL) && 2553 ef4_try_recovery(efx)) 2554 return; 2555 2556 if (!pending) 2557 return; 2558 2559 rtnl_lock(); 2560 2561 /* We checked the state in ef4_schedule_reset() but it may 2562 * have changed by now. Now that we have the RTNL lock, 2563 * it cannot change again. 2564 */ 2565 if (efx->state == STATE_READY) 2566 (void)ef4_reset(efx, method); 2567 2568 rtnl_unlock(); 2569 } 2570 2571 void ef4_schedule_reset(struct ef4_nic *efx, enum reset_type type) 2572 { 2573 enum reset_type method; 2574 2575 if (efx->state == STATE_RECOVERY) { 2576 netif_dbg(efx, drv, efx->net_dev, 2577 "recovering: skip scheduling %s reset\n", 2578 RESET_TYPE(type)); 2579 return; 2580 } 2581 2582 switch (type) { 2583 case RESET_TYPE_INVISIBLE: 2584 case RESET_TYPE_ALL: 2585 case RESET_TYPE_RECOVER_OR_ALL: 2586 case RESET_TYPE_WORLD: 2587 case RESET_TYPE_DISABLE: 2588 case RESET_TYPE_RECOVER_OR_DISABLE: 2589 case RESET_TYPE_DATAPATH: 2590 method = type; 2591 netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n", 2592 RESET_TYPE(method)); 2593 break; 2594 default: 2595 method = efx->type->map_reset_reason(type); 2596 netif_dbg(efx, drv, efx->net_dev, 2597 "scheduling %s reset for %s\n", 2598 RESET_TYPE(method), RESET_TYPE(type)); 2599 break; 2600 } 2601 2602 set_bit(method, &efx->reset_pending); 2603 smp_mb(); /* ensure we change reset_pending before checking state */ 2604 2605 /* If we're not READY then just leave the flags set as the cue 2606 * to abort probing or reschedule the reset later. 2607 */ 2608 if (ACCESS_ONCE(efx->state) != STATE_READY) 2609 return; 2610 2611 queue_work(reset_workqueue, &efx->reset_work); 2612 } 2613 2614 /************************************************************************** 2615 * 2616 * List of NICs we support 2617 * 2618 **************************************************************************/ 2619 2620 /* PCI device ID table */ 2621 static const struct pci_device_id ef4_pci_table[] = { 2622 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 2623 PCI_DEVICE_ID_SOLARFLARE_SFC4000A_0), 2624 .driver_data = (unsigned long) &falcon_a1_nic_type}, 2625 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 2626 PCI_DEVICE_ID_SOLARFLARE_SFC4000B), 2627 .driver_data = (unsigned long) &falcon_b0_nic_type}, 2628 {0} /* end of list */ 2629 }; 2630 2631 /************************************************************************** 2632 * 2633 * Dummy PHY/MAC operations 2634 * 2635 * Can be used for some unimplemented operations 2636 * Needed so all function pointers are valid and do not have to be tested 2637 * before use 2638 * 2639 **************************************************************************/ 2640 int ef4_port_dummy_op_int(struct ef4_nic *efx) 2641 { 2642 return 0; 2643 } 2644 void ef4_port_dummy_op_void(struct ef4_nic *efx) {} 2645 2646 static bool ef4_port_dummy_op_poll(struct ef4_nic *efx) 2647 { 2648 return false; 2649 } 2650 2651 static const struct ef4_phy_operations ef4_dummy_phy_operations = { 2652 .init = ef4_port_dummy_op_int, 2653 .reconfigure = ef4_port_dummy_op_int, 2654 .poll = ef4_port_dummy_op_poll, 2655 .fini = ef4_port_dummy_op_void, 2656 }; 2657 2658 /************************************************************************** 2659 * 2660 * Data housekeeping 2661 * 2662 **************************************************************************/ 2663 2664 /* This zeroes out and then fills in the invariants in a struct 2665 * ef4_nic (including all sub-structures). 2666 */ 2667 static int ef4_init_struct(struct ef4_nic *efx, 2668 struct pci_dev *pci_dev, struct net_device *net_dev) 2669 { 2670 int i; 2671 2672 /* Initialise common structures */ 2673 INIT_LIST_HEAD(&efx->node); 2674 INIT_LIST_HEAD(&efx->secondary_list); 2675 spin_lock_init(&efx->biu_lock); 2676 #ifdef CONFIG_SFC_FALCON_MTD 2677 INIT_LIST_HEAD(&efx->mtd_list); 2678 #endif 2679 INIT_WORK(&efx->reset_work, ef4_reset_work); 2680 INIT_DELAYED_WORK(&efx->monitor_work, ef4_monitor); 2681 INIT_DELAYED_WORK(&efx->selftest_work, ef4_selftest_async_work); 2682 efx->pci_dev = pci_dev; 2683 efx->msg_enable = debug; 2684 efx->state = STATE_UNINIT; 2685 strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name)); 2686 2687 efx->net_dev = net_dev; 2688 efx->rx_prefix_size = efx->type->rx_prefix_size; 2689 efx->rx_ip_align = 2690 NET_IP_ALIGN ? (efx->rx_prefix_size + NET_IP_ALIGN) % 4 : 0; 2691 efx->rx_packet_hash_offset = 2692 efx->type->rx_hash_offset - efx->type->rx_prefix_size; 2693 efx->rx_packet_ts_offset = 2694 efx->type->rx_ts_offset - efx->type->rx_prefix_size; 2695 spin_lock_init(&efx->stats_lock); 2696 mutex_init(&efx->mac_lock); 2697 efx->phy_op = &ef4_dummy_phy_operations; 2698 efx->mdio.dev = net_dev; 2699 INIT_WORK(&efx->mac_work, ef4_mac_work); 2700 init_waitqueue_head(&efx->flush_wq); 2701 2702 for (i = 0; i < EF4_MAX_CHANNELS; i++) { 2703 efx->channel[i] = ef4_alloc_channel(efx, i, NULL); 2704 if (!efx->channel[i]) 2705 goto fail; 2706 efx->msi_context[i].efx = efx; 2707 efx->msi_context[i].index = i; 2708 } 2709 2710 /* Higher numbered interrupt modes are less capable! */ 2711 efx->interrupt_mode = max(efx->type->max_interrupt_mode, 2712 interrupt_mode); 2713 2714 /* Would be good to use the net_dev name, but we're too early */ 2715 snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s", 2716 pci_name(pci_dev)); 2717 efx->workqueue = create_singlethread_workqueue(efx->workqueue_name); 2718 if (!efx->workqueue) 2719 goto fail; 2720 2721 return 0; 2722 2723 fail: 2724 ef4_fini_struct(efx); 2725 return -ENOMEM; 2726 } 2727 2728 static void ef4_fini_struct(struct ef4_nic *efx) 2729 { 2730 int i; 2731 2732 for (i = 0; i < EF4_MAX_CHANNELS; i++) 2733 kfree(efx->channel[i]); 2734 2735 kfree(efx->vpd_sn); 2736 2737 if (efx->workqueue) { 2738 destroy_workqueue(efx->workqueue); 2739 efx->workqueue = NULL; 2740 } 2741 } 2742 2743 void ef4_update_sw_stats(struct ef4_nic *efx, u64 *stats) 2744 { 2745 u64 n_rx_nodesc_trunc = 0; 2746 struct ef4_channel *channel; 2747 2748 ef4_for_each_channel(channel, efx) 2749 n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc; 2750 stats[GENERIC_STAT_rx_nodesc_trunc] = n_rx_nodesc_trunc; 2751 stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops); 2752 } 2753 2754 /************************************************************************** 2755 * 2756 * PCI interface 2757 * 2758 **************************************************************************/ 2759 2760 /* Main body of final NIC shutdown code 2761 * This is called only at module unload (or hotplug removal). 2762 */ 2763 static void ef4_pci_remove_main(struct ef4_nic *efx) 2764 { 2765 /* Flush reset_work. It can no longer be scheduled since we 2766 * are not READY. 2767 */ 2768 BUG_ON(efx->state == STATE_READY); 2769 cancel_work_sync(&efx->reset_work); 2770 2771 ef4_disable_interrupts(efx); 2772 ef4_nic_fini_interrupt(efx); 2773 ef4_fini_port(efx); 2774 efx->type->fini(efx); 2775 ef4_fini_napi(efx); 2776 ef4_remove_all(efx); 2777 } 2778 2779 /* Final NIC shutdown 2780 * This is called only at module unload (or hotplug removal). A PF can call 2781 * this on its VFs to ensure they are unbound first. 2782 */ 2783 static void ef4_pci_remove(struct pci_dev *pci_dev) 2784 { 2785 struct ef4_nic *efx; 2786 2787 efx = pci_get_drvdata(pci_dev); 2788 if (!efx) 2789 return; 2790 2791 /* Mark the NIC as fini, then stop the interface */ 2792 rtnl_lock(); 2793 ef4_dissociate(efx); 2794 dev_close(efx->net_dev); 2795 ef4_disable_interrupts(efx); 2796 efx->state = STATE_UNINIT; 2797 rtnl_unlock(); 2798 2799 ef4_unregister_netdev(efx); 2800 2801 ef4_mtd_remove(efx); 2802 2803 ef4_pci_remove_main(efx); 2804 2805 ef4_fini_io(efx); 2806 netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n"); 2807 2808 ef4_fini_struct(efx); 2809 free_netdev(efx->net_dev); 2810 2811 pci_disable_pcie_error_reporting(pci_dev); 2812 }; 2813 2814 /* NIC VPD information 2815 * Called during probe to display the part number of the 2816 * installed NIC. VPD is potentially very large but this should 2817 * always appear within the first 512 bytes. 2818 */ 2819 #define SFC_VPD_LEN 512 2820 static void ef4_probe_vpd_strings(struct ef4_nic *efx) 2821 { 2822 struct pci_dev *dev = efx->pci_dev; 2823 char vpd_data[SFC_VPD_LEN]; 2824 ssize_t vpd_size; 2825 int ro_start, ro_size, i, j; 2826 2827 /* Get the vpd data from the device */ 2828 vpd_size = pci_read_vpd(dev, 0, sizeof(vpd_data), vpd_data); 2829 if (vpd_size <= 0) { 2830 netif_err(efx, drv, efx->net_dev, "Unable to read VPD\n"); 2831 return; 2832 } 2833 2834 /* Get the Read only section */ 2835 ro_start = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA); 2836 if (ro_start < 0) { 2837 netif_err(efx, drv, efx->net_dev, "VPD Read-only not found\n"); 2838 return; 2839 } 2840 2841 ro_size = pci_vpd_lrdt_size(&vpd_data[ro_start]); 2842 j = ro_size; 2843 i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 2844 if (i + j > vpd_size) 2845 j = vpd_size - i; 2846 2847 /* Get the Part number */ 2848 i = pci_vpd_find_info_keyword(vpd_data, i, j, "PN"); 2849 if (i < 0) { 2850 netif_err(efx, drv, efx->net_dev, "Part number not found\n"); 2851 return; 2852 } 2853 2854 j = pci_vpd_info_field_size(&vpd_data[i]); 2855 i += PCI_VPD_INFO_FLD_HDR_SIZE; 2856 if (i + j > vpd_size) { 2857 netif_err(efx, drv, efx->net_dev, "Incomplete part number\n"); 2858 return; 2859 } 2860 2861 netif_info(efx, drv, efx->net_dev, 2862 "Part Number : %.*s\n", j, &vpd_data[i]); 2863 2864 i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 2865 j = ro_size; 2866 i = pci_vpd_find_info_keyword(vpd_data, i, j, "SN"); 2867 if (i < 0) { 2868 netif_err(efx, drv, efx->net_dev, "Serial number not found\n"); 2869 return; 2870 } 2871 2872 j = pci_vpd_info_field_size(&vpd_data[i]); 2873 i += PCI_VPD_INFO_FLD_HDR_SIZE; 2874 if (i + j > vpd_size) { 2875 netif_err(efx, drv, efx->net_dev, "Incomplete serial number\n"); 2876 return; 2877 } 2878 2879 efx->vpd_sn = kmalloc(j + 1, GFP_KERNEL); 2880 if (!efx->vpd_sn) 2881 return; 2882 2883 snprintf(efx->vpd_sn, j + 1, "%s", &vpd_data[i]); 2884 } 2885 2886 2887 /* Main body of NIC initialisation 2888 * This is called at module load (or hotplug insertion, theoretically). 2889 */ 2890 static int ef4_pci_probe_main(struct ef4_nic *efx) 2891 { 2892 int rc; 2893 2894 /* Do start-of-day initialisation */ 2895 rc = ef4_probe_all(efx); 2896 if (rc) 2897 goto fail1; 2898 2899 ef4_init_napi(efx); 2900 2901 rc = efx->type->init(efx); 2902 if (rc) { 2903 netif_err(efx, probe, efx->net_dev, 2904 "failed to initialise NIC\n"); 2905 goto fail3; 2906 } 2907 2908 rc = ef4_init_port(efx); 2909 if (rc) { 2910 netif_err(efx, probe, efx->net_dev, 2911 "failed to initialise port\n"); 2912 goto fail4; 2913 } 2914 2915 rc = ef4_nic_init_interrupt(efx); 2916 if (rc) 2917 goto fail5; 2918 rc = ef4_enable_interrupts(efx); 2919 if (rc) 2920 goto fail6; 2921 2922 return 0; 2923 2924 fail6: 2925 ef4_nic_fini_interrupt(efx); 2926 fail5: 2927 ef4_fini_port(efx); 2928 fail4: 2929 efx->type->fini(efx); 2930 fail3: 2931 ef4_fini_napi(efx); 2932 ef4_remove_all(efx); 2933 fail1: 2934 return rc; 2935 } 2936 2937 /* NIC initialisation 2938 * 2939 * This is called at module load (or hotplug insertion, 2940 * theoretically). It sets up PCI mappings, resets the NIC, 2941 * sets up and registers the network devices with the kernel and hooks 2942 * the interrupt service routine. It does not prepare the device for 2943 * transmission; this is left to the first time one of the network 2944 * interfaces is brought up (i.e. ef4_net_open). 2945 */ 2946 static int ef4_pci_probe(struct pci_dev *pci_dev, 2947 const struct pci_device_id *entry) 2948 { 2949 struct net_device *net_dev; 2950 struct ef4_nic *efx; 2951 int rc; 2952 2953 /* Allocate and initialise a struct net_device and struct ef4_nic */ 2954 net_dev = alloc_etherdev_mqs(sizeof(*efx), EF4_MAX_CORE_TX_QUEUES, 2955 EF4_MAX_RX_QUEUES); 2956 if (!net_dev) 2957 return -ENOMEM; 2958 efx = netdev_priv(net_dev); 2959 efx->type = (const struct ef4_nic_type *) entry->driver_data; 2960 efx->fixed_features |= NETIF_F_HIGHDMA; 2961 2962 pci_set_drvdata(pci_dev, efx); 2963 SET_NETDEV_DEV(net_dev, &pci_dev->dev); 2964 rc = ef4_init_struct(efx, pci_dev, net_dev); 2965 if (rc) 2966 goto fail1; 2967 2968 netif_info(efx, probe, efx->net_dev, 2969 "Solarflare NIC detected\n"); 2970 2971 ef4_probe_vpd_strings(efx); 2972 2973 /* Set up basic I/O (BAR mappings etc) */ 2974 rc = ef4_init_io(efx); 2975 if (rc) 2976 goto fail2; 2977 2978 rc = ef4_pci_probe_main(efx); 2979 if (rc) 2980 goto fail3; 2981 2982 net_dev->features |= (efx->type->offload_features | NETIF_F_SG | 2983 NETIF_F_RXCSUM); 2984 /* Mask for features that also apply to VLAN devices */ 2985 net_dev->vlan_features |= (NETIF_F_HW_CSUM | NETIF_F_SG | 2986 NETIF_F_HIGHDMA | NETIF_F_RXCSUM); 2987 2988 net_dev->hw_features = net_dev->features & ~efx->fixed_features; 2989 2990 /* Disable VLAN filtering by default. It may be enforced if 2991 * the feature is fixed (i.e. VLAN filters are required to 2992 * receive VLAN tagged packets due to vPort restrictions). 2993 */ 2994 net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 2995 net_dev->features |= efx->fixed_features; 2996 2997 rc = ef4_register_netdev(efx); 2998 if (rc) 2999 goto fail4; 3000 3001 netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n"); 3002 3003 /* Try to create MTDs, but allow this to fail */ 3004 rtnl_lock(); 3005 rc = ef4_mtd_probe(efx); 3006 rtnl_unlock(); 3007 if (rc && rc != -EPERM) 3008 netif_warn(efx, probe, efx->net_dev, 3009 "failed to create MTDs (%d)\n", rc); 3010 3011 rc = pci_enable_pcie_error_reporting(pci_dev); 3012 if (rc && rc != -EINVAL) 3013 netif_notice(efx, probe, efx->net_dev, 3014 "PCIE error reporting unavailable (%d).\n", 3015 rc); 3016 3017 return 0; 3018 3019 fail4: 3020 ef4_pci_remove_main(efx); 3021 fail3: 3022 ef4_fini_io(efx); 3023 fail2: 3024 ef4_fini_struct(efx); 3025 fail1: 3026 WARN_ON(rc > 0); 3027 netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc); 3028 free_netdev(net_dev); 3029 return rc; 3030 } 3031 3032 static int ef4_pm_freeze(struct device *dev) 3033 { 3034 struct ef4_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 3035 3036 rtnl_lock(); 3037 3038 if (efx->state != STATE_DISABLED) { 3039 efx->state = STATE_UNINIT; 3040 3041 ef4_device_detach_sync(efx); 3042 3043 ef4_stop_all(efx); 3044 ef4_disable_interrupts(efx); 3045 } 3046 3047 rtnl_unlock(); 3048 3049 return 0; 3050 } 3051 3052 static int ef4_pm_thaw(struct device *dev) 3053 { 3054 int rc; 3055 struct ef4_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 3056 3057 rtnl_lock(); 3058 3059 if (efx->state != STATE_DISABLED) { 3060 rc = ef4_enable_interrupts(efx); 3061 if (rc) 3062 goto fail; 3063 3064 mutex_lock(&efx->mac_lock); 3065 efx->phy_op->reconfigure(efx); 3066 mutex_unlock(&efx->mac_lock); 3067 3068 ef4_start_all(efx); 3069 3070 netif_device_attach(efx->net_dev); 3071 3072 efx->state = STATE_READY; 3073 3074 efx->type->resume_wol(efx); 3075 } 3076 3077 rtnl_unlock(); 3078 3079 /* Reschedule any quenched resets scheduled during ef4_pm_freeze() */ 3080 queue_work(reset_workqueue, &efx->reset_work); 3081 3082 return 0; 3083 3084 fail: 3085 rtnl_unlock(); 3086 3087 return rc; 3088 } 3089 3090 static int ef4_pm_poweroff(struct device *dev) 3091 { 3092 struct pci_dev *pci_dev = to_pci_dev(dev); 3093 struct ef4_nic *efx = pci_get_drvdata(pci_dev); 3094 3095 efx->type->fini(efx); 3096 3097 efx->reset_pending = 0; 3098 3099 pci_save_state(pci_dev); 3100 return pci_set_power_state(pci_dev, PCI_D3hot); 3101 } 3102 3103 /* Used for both resume and restore */ 3104 static int ef4_pm_resume(struct device *dev) 3105 { 3106 struct pci_dev *pci_dev = to_pci_dev(dev); 3107 struct ef4_nic *efx = pci_get_drvdata(pci_dev); 3108 int rc; 3109 3110 rc = pci_set_power_state(pci_dev, PCI_D0); 3111 if (rc) 3112 return rc; 3113 pci_restore_state(pci_dev); 3114 rc = pci_enable_device(pci_dev); 3115 if (rc) 3116 return rc; 3117 pci_set_master(efx->pci_dev); 3118 rc = efx->type->reset(efx, RESET_TYPE_ALL); 3119 if (rc) 3120 return rc; 3121 rc = efx->type->init(efx); 3122 if (rc) 3123 return rc; 3124 rc = ef4_pm_thaw(dev); 3125 return rc; 3126 } 3127 3128 static int ef4_pm_suspend(struct device *dev) 3129 { 3130 int rc; 3131 3132 ef4_pm_freeze(dev); 3133 rc = ef4_pm_poweroff(dev); 3134 if (rc) 3135 ef4_pm_resume(dev); 3136 return rc; 3137 } 3138 3139 static const struct dev_pm_ops ef4_pm_ops = { 3140 .suspend = ef4_pm_suspend, 3141 .resume = ef4_pm_resume, 3142 .freeze = ef4_pm_freeze, 3143 .thaw = ef4_pm_thaw, 3144 .poweroff = ef4_pm_poweroff, 3145 .restore = ef4_pm_resume, 3146 }; 3147 3148 /* A PCI error affecting this device was detected. 3149 * At this point MMIO and DMA may be disabled. 3150 * Stop the software path and request a slot reset. 3151 */ 3152 static pci_ers_result_t ef4_io_error_detected(struct pci_dev *pdev, 3153 enum pci_channel_state state) 3154 { 3155 pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3156 struct ef4_nic *efx = pci_get_drvdata(pdev); 3157 3158 if (state == pci_channel_io_perm_failure) 3159 return PCI_ERS_RESULT_DISCONNECT; 3160 3161 rtnl_lock(); 3162 3163 if (efx->state != STATE_DISABLED) { 3164 efx->state = STATE_RECOVERY; 3165 efx->reset_pending = 0; 3166 3167 ef4_device_detach_sync(efx); 3168 3169 ef4_stop_all(efx); 3170 ef4_disable_interrupts(efx); 3171 3172 status = PCI_ERS_RESULT_NEED_RESET; 3173 } else { 3174 /* If the interface is disabled we don't want to do anything 3175 * with it. 3176 */ 3177 status = PCI_ERS_RESULT_RECOVERED; 3178 } 3179 3180 rtnl_unlock(); 3181 3182 pci_disable_device(pdev); 3183 3184 return status; 3185 } 3186 3187 /* Fake a successful reset, which will be performed later in ef4_io_resume. */ 3188 static pci_ers_result_t ef4_io_slot_reset(struct pci_dev *pdev) 3189 { 3190 struct ef4_nic *efx = pci_get_drvdata(pdev); 3191 pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3192 int rc; 3193 3194 if (pci_enable_device(pdev)) { 3195 netif_err(efx, hw, efx->net_dev, 3196 "Cannot re-enable PCI device after reset.\n"); 3197 status = PCI_ERS_RESULT_DISCONNECT; 3198 } 3199 3200 rc = pci_cleanup_aer_uncorrect_error_status(pdev); 3201 if (rc) { 3202 netif_err(efx, hw, efx->net_dev, 3203 "pci_cleanup_aer_uncorrect_error_status failed (%d)\n", rc); 3204 /* Non-fatal error. Continue. */ 3205 } 3206 3207 return status; 3208 } 3209 3210 /* Perform the actual reset and resume I/O operations. */ 3211 static void ef4_io_resume(struct pci_dev *pdev) 3212 { 3213 struct ef4_nic *efx = pci_get_drvdata(pdev); 3214 int rc; 3215 3216 rtnl_lock(); 3217 3218 if (efx->state == STATE_DISABLED) 3219 goto out; 3220 3221 rc = ef4_reset(efx, RESET_TYPE_ALL); 3222 if (rc) { 3223 netif_err(efx, hw, efx->net_dev, 3224 "ef4_reset failed after PCI error (%d)\n", rc); 3225 } else { 3226 efx->state = STATE_READY; 3227 netif_dbg(efx, hw, efx->net_dev, 3228 "Done resetting and resuming IO after PCI error.\n"); 3229 } 3230 3231 out: 3232 rtnl_unlock(); 3233 } 3234 3235 /* For simplicity and reliability, we always require a slot reset and try to 3236 * reset the hardware when a pci error affecting the device is detected. 3237 * We leave both the link_reset and mmio_enabled callback unimplemented: 3238 * with our request for slot reset the mmio_enabled callback will never be 3239 * called, and the link_reset callback is not used by AER or EEH mechanisms. 3240 */ 3241 static const struct pci_error_handlers ef4_err_handlers = { 3242 .error_detected = ef4_io_error_detected, 3243 .slot_reset = ef4_io_slot_reset, 3244 .resume = ef4_io_resume, 3245 }; 3246 3247 static struct pci_driver ef4_pci_driver = { 3248 .name = KBUILD_MODNAME, 3249 .id_table = ef4_pci_table, 3250 .probe = ef4_pci_probe, 3251 .remove = ef4_pci_remove, 3252 .driver.pm = &ef4_pm_ops, 3253 .err_handler = &ef4_err_handlers, 3254 }; 3255 3256 /************************************************************************** 3257 * 3258 * Kernel module interface 3259 * 3260 *************************************************************************/ 3261 3262 module_param(interrupt_mode, uint, 0444); 3263 MODULE_PARM_DESC(interrupt_mode, 3264 "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)"); 3265 3266 static int __init ef4_init_module(void) 3267 { 3268 int rc; 3269 3270 printk(KERN_INFO "Solarflare Falcon driver v" EF4_DRIVER_VERSION "\n"); 3271 3272 rc = register_netdevice_notifier(&ef4_netdev_notifier); 3273 if (rc) 3274 goto err_notifier; 3275 3276 reset_workqueue = create_singlethread_workqueue("sfc_reset"); 3277 if (!reset_workqueue) { 3278 rc = -ENOMEM; 3279 goto err_reset; 3280 } 3281 3282 rc = pci_register_driver(&ef4_pci_driver); 3283 if (rc < 0) 3284 goto err_pci; 3285 3286 return 0; 3287 3288 err_pci: 3289 destroy_workqueue(reset_workqueue); 3290 err_reset: 3291 unregister_netdevice_notifier(&ef4_netdev_notifier); 3292 err_notifier: 3293 return rc; 3294 } 3295 3296 static void __exit ef4_exit_module(void) 3297 { 3298 printk(KERN_INFO "Solarflare Falcon driver unloading\n"); 3299 3300 pci_unregister_driver(&ef4_pci_driver); 3301 destroy_workqueue(reset_workqueue); 3302 unregister_netdevice_notifier(&ef4_netdev_notifier); 3303 3304 } 3305 3306 module_init(ef4_init_module); 3307 module_exit(ef4_exit_module); 3308 3309 MODULE_AUTHOR("Solarflare Communications and " 3310 "Michael Brown <mbrown@fensystems.co.uk>"); 3311 MODULE_DESCRIPTION("Solarflare Falcon network driver"); 3312 MODULE_LICENSE("GPL"); 3313 MODULE_DEVICE_TABLE(pci, ef4_pci_table); 3314 MODULE_VERSION(EF4_DRIVER_VERSION); 3315