1 /**************************************************************************** 2 * Driver for Solarflare network controllers and boards 3 * Copyright 2005-2006 Fen Systems Ltd. 4 * Copyright 2005-2013 Solarflare Communications Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation, incorporated herein by reference. 9 */ 10 11 #include <linux/module.h> 12 #include <linux/pci.h> 13 #include <linux/netdevice.h> 14 #include <linux/etherdevice.h> 15 #include <linux/delay.h> 16 #include <linux/notifier.h> 17 #include <linux/ip.h> 18 #include <linux/tcp.h> 19 #include <linux/in.h> 20 #include <linux/ethtool.h> 21 #include <linux/topology.h> 22 #include <linux/gfp.h> 23 #include <linux/aer.h> 24 #include <linux/interrupt.h> 25 #include "net_driver.h" 26 #include "efx.h" 27 #include "nic.h" 28 #include "selftest.h" 29 30 #include "workarounds.h" 31 32 /************************************************************************** 33 * 34 * Type name strings 35 * 36 ************************************************************************** 37 */ 38 39 /* Loopback mode names (see LOOPBACK_MODE()) */ 40 const unsigned int ef4_loopback_mode_max = LOOPBACK_MAX; 41 const char *const ef4_loopback_mode_names[] = { 42 [LOOPBACK_NONE] = "NONE", 43 [LOOPBACK_DATA] = "DATAPATH", 44 [LOOPBACK_GMAC] = "GMAC", 45 [LOOPBACK_XGMII] = "XGMII", 46 [LOOPBACK_XGXS] = "XGXS", 47 [LOOPBACK_XAUI] = "XAUI", 48 [LOOPBACK_GMII] = "GMII", 49 [LOOPBACK_SGMII] = "SGMII", 50 [LOOPBACK_XGBR] = "XGBR", 51 [LOOPBACK_XFI] = "XFI", 52 [LOOPBACK_XAUI_FAR] = "XAUI_FAR", 53 [LOOPBACK_GMII_FAR] = "GMII_FAR", 54 [LOOPBACK_SGMII_FAR] = "SGMII_FAR", 55 [LOOPBACK_XFI_FAR] = "XFI_FAR", 56 [LOOPBACK_GPHY] = "GPHY", 57 [LOOPBACK_PHYXS] = "PHYXS", 58 [LOOPBACK_PCS] = "PCS", 59 [LOOPBACK_PMAPMD] = "PMA/PMD", 60 [LOOPBACK_XPORT] = "XPORT", 61 [LOOPBACK_XGMII_WS] = "XGMII_WS", 62 [LOOPBACK_XAUI_WS] = "XAUI_WS", 63 [LOOPBACK_XAUI_WS_FAR] = "XAUI_WS_FAR", 64 [LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR", 65 [LOOPBACK_GMII_WS] = "GMII_WS", 66 [LOOPBACK_XFI_WS] = "XFI_WS", 67 [LOOPBACK_XFI_WS_FAR] = "XFI_WS_FAR", 68 [LOOPBACK_PHYXS_WS] = "PHYXS_WS", 69 }; 70 71 const unsigned int ef4_reset_type_max = RESET_TYPE_MAX; 72 const char *const ef4_reset_type_names[] = { 73 [RESET_TYPE_INVISIBLE] = "INVISIBLE", 74 [RESET_TYPE_ALL] = "ALL", 75 [RESET_TYPE_RECOVER_OR_ALL] = "RECOVER_OR_ALL", 76 [RESET_TYPE_WORLD] = "WORLD", 77 [RESET_TYPE_RECOVER_OR_DISABLE] = "RECOVER_OR_DISABLE", 78 [RESET_TYPE_DATAPATH] = "DATAPATH", 79 [RESET_TYPE_DISABLE] = "DISABLE", 80 [RESET_TYPE_TX_WATCHDOG] = "TX_WATCHDOG", 81 [RESET_TYPE_INT_ERROR] = "INT_ERROR", 82 [RESET_TYPE_RX_RECOVERY] = "RX_RECOVERY", 83 [RESET_TYPE_DMA_ERROR] = "DMA_ERROR", 84 [RESET_TYPE_TX_SKIP] = "TX_SKIP", 85 }; 86 87 /* Reset workqueue. If any NIC has a hardware failure then a reset will be 88 * queued onto this work queue. This is not a per-nic work queue, because 89 * ef4_reset_work() acquires the rtnl lock, so resets are naturally serialised. 90 */ 91 static struct workqueue_struct *reset_workqueue; 92 93 /* How often and how many times to poll for a reset while waiting for a 94 * BIST that another function started to complete. 95 */ 96 #define BIST_WAIT_DELAY_MS 100 97 #define BIST_WAIT_DELAY_COUNT 100 98 99 /************************************************************************** 100 * 101 * Configurable values 102 * 103 *************************************************************************/ 104 105 /* 106 * Use separate channels for TX and RX events 107 * 108 * Set this to 1 to use separate channels for TX and RX. It allows us 109 * to control interrupt affinity separately for TX and RX. 110 * 111 * This is only used in MSI-X interrupt mode 112 */ 113 bool ef4_separate_tx_channels; 114 module_param(ef4_separate_tx_channels, bool, 0444); 115 MODULE_PARM_DESC(ef4_separate_tx_channels, 116 "Use separate channels for TX and RX"); 117 118 /* This is the weight assigned to each of the (per-channel) virtual 119 * NAPI devices. 120 */ 121 static int napi_weight = 64; 122 123 /* This is the time (in jiffies) between invocations of the hardware 124 * monitor. 125 * On Falcon-based NICs, this will: 126 * - Check the on-board hardware monitor; 127 * - Poll the link state and reconfigure the hardware as necessary. 128 * On Siena-based NICs for power systems with EEH support, this will give EEH a 129 * chance to start. 130 */ 131 static unsigned int ef4_monitor_interval = 1 * HZ; 132 133 /* Initial interrupt moderation settings. They can be modified after 134 * module load with ethtool. 135 * 136 * The default for RX should strike a balance between increasing the 137 * round-trip latency and reducing overhead. 138 */ 139 static unsigned int rx_irq_mod_usec = 60; 140 141 /* Initial interrupt moderation settings. They can be modified after 142 * module load with ethtool. 143 * 144 * This default is chosen to ensure that a 10G link does not go idle 145 * while a TX queue is stopped after it has become full. A queue is 146 * restarted when it drops below half full. The time this takes (assuming 147 * worst case 3 descriptors per packet and 1024 descriptors) is 148 * 512 / 3 * 1.2 = 205 usec. 149 */ 150 static unsigned int tx_irq_mod_usec = 150; 151 152 /* This is the first interrupt mode to try out of: 153 * 0 => MSI-X 154 * 1 => MSI 155 * 2 => legacy 156 */ 157 static unsigned int interrupt_mode; 158 159 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS), 160 * i.e. the number of CPUs among which we may distribute simultaneous 161 * interrupt handling. 162 * 163 * Cards without MSI-X will only target one CPU via legacy or MSI interrupt. 164 * The default (0) means to assign an interrupt to each core. 165 */ 166 static unsigned int rss_cpus; 167 module_param(rss_cpus, uint, 0444); 168 MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling"); 169 170 static bool phy_flash_cfg; 171 module_param(phy_flash_cfg, bool, 0644); 172 MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially"); 173 174 static unsigned irq_adapt_low_thresh = 8000; 175 module_param(irq_adapt_low_thresh, uint, 0644); 176 MODULE_PARM_DESC(irq_adapt_low_thresh, 177 "Threshold score for reducing IRQ moderation"); 178 179 static unsigned irq_adapt_high_thresh = 16000; 180 module_param(irq_adapt_high_thresh, uint, 0644); 181 MODULE_PARM_DESC(irq_adapt_high_thresh, 182 "Threshold score for increasing IRQ moderation"); 183 184 static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 185 NETIF_MSG_LINK | NETIF_MSG_IFDOWN | 186 NETIF_MSG_IFUP | NETIF_MSG_RX_ERR | 187 NETIF_MSG_TX_ERR | NETIF_MSG_HW); 188 module_param(debug, uint, 0); 189 MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value"); 190 191 /************************************************************************** 192 * 193 * Utility functions and prototypes 194 * 195 *************************************************************************/ 196 197 static int ef4_soft_enable_interrupts(struct ef4_nic *efx); 198 static void ef4_soft_disable_interrupts(struct ef4_nic *efx); 199 static void ef4_remove_channel(struct ef4_channel *channel); 200 static void ef4_remove_channels(struct ef4_nic *efx); 201 static const struct ef4_channel_type ef4_default_channel_type; 202 static void ef4_remove_port(struct ef4_nic *efx); 203 static void ef4_init_napi_channel(struct ef4_channel *channel); 204 static void ef4_fini_napi(struct ef4_nic *efx); 205 static void ef4_fini_napi_channel(struct ef4_channel *channel); 206 static void ef4_fini_struct(struct ef4_nic *efx); 207 static void ef4_start_all(struct ef4_nic *efx); 208 static void ef4_stop_all(struct ef4_nic *efx); 209 210 #define EF4_ASSERT_RESET_SERIALISED(efx) \ 211 do { \ 212 if ((efx->state == STATE_READY) || \ 213 (efx->state == STATE_RECOVERY) || \ 214 (efx->state == STATE_DISABLED)) \ 215 ASSERT_RTNL(); \ 216 } while (0) 217 218 static int ef4_check_disabled(struct ef4_nic *efx) 219 { 220 if (efx->state == STATE_DISABLED || efx->state == STATE_RECOVERY) { 221 netif_err(efx, drv, efx->net_dev, 222 "device is disabled due to earlier errors\n"); 223 return -EIO; 224 } 225 return 0; 226 } 227 228 /************************************************************************** 229 * 230 * Event queue processing 231 * 232 *************************************************************************/ 233 234 /* Process channel's event queue 235 * 236 * This function is responsible for processing the event queue of a 237 * single channel. The caller must guarantee that this function will 238 * never be concurrently called more than once on the same channel, 239 * though different channels may be being processed concurrently. 240 */ 241 static int ef4_process_channel(struct ef4_channel *channel, int budget) 242 { 243 struct ef4_tx_queue *tx_queue; 244 int spent; 245 246 if (unlikely(!channel->enabled)) 247 return 0; 248 249 ef4_for_each_channel_tx_queue(tx_queue, channel) { 250 tx_queue->pkts_compl = 0; 251 tx_queue->bytes_compl = 0; 252 } 253 254 spent = ef4_nic_process_eventq(channel, budget); 255 if (spent && ef4_channel_has_rx_queue(channel)) { 256 struct ef4_rx_queue *rx_queue = 257 ef4_channel_get_rx_queue(channel); 258 259 ef4_rx_flush_packet(channel); 260 ef4_fast_push_rx_descriptors(rx_queue, true); 261 } 262 263 /* Update BQL */ 264 ef4_for_each_channel_tx_queue(tx_queue, channel) { 265 if (tx_queue->bytes_compl) { 266 netdev_tx_completed_queue(tx_queue->core_txq, 267 tx_queue->pkts_compl, tx_queue->bytes_compl); 268 } 269 } 270 271 return spent; 272 } 273 274 /* NAPI poll handler 275 * 276 * NAPI guarantees serialisation of polls of the same device, which 277 * provides the guarantee required by ef4_process_channel(). 278 */ 279 static void ef4_update_irq_mod(struct ef4_nic *efx, struct ef4_channel *channel) 280 { 281 int step = efx->irq_mod_step_us; 282 283 if (channel->irq_mod_score < irq_adapt_low_thresh) { 284 if (channel->irq_moderation_us > step) { 285 channel->irq_moderation_us -= step; 286 efx->type->push_irq_moderation(channel); 287 } 288 } else if (channel->irq_mod_score > irq_adapt_high_thresh) { 289 if (channel->irq_moderation_us < 290 efx->irq_rx_moderation_us) { 291 channel->irq_moderation_us += step; 292 efx->type->push_irq_moderation(channel); 293 } 294 } 295 296 channel->irq_count = 0; 297 channel->irq_mod_score = 0; 298 } 299 300 static int ef4_poll(struct napi_struct *napi, int budget) 301 { 302 struct ef4_channel *channel = 303 container_of(napi, struct ef4_channel, napi_str); 304 struct ef4_nic *efx = channel->efx; 305 int spent; 306 307 if (!ef4_channel_lock_napi(channel)) 308 return budget; 309 310 netif_vdbg(efx, intr, efx->net_dev, 311 "channel %d NAPI poll executing on CPU %d\n", 312 channel->channel, raw_smp_processor_id()); 313 314 spent = ef4_process_channel(channel, budget); 315 316 if (spent < budget) { 317 if (ef4_channel_has_rx_queue(channel) && 318 efx->irq_rx_adaptive && 319 unlikely(++channel->irq_count == 1000)) { 320 ef4_update_irq_mod(efx, channel); 321 } 322 323 ef4_filter_rfs_expire(channel); 324 325 /* There is no race here; although napi_disable() will 326 * only wait for napi_complete(), this isn't a problem 327 * since ef4_nic_eventq_read_ack() will have no effect if 328 * interrupts have already been disabled. 329 */ 330 napi_complete(napi); 331 ef4_nic_eventq_read_ack(channel); 332 } 333 334 ef4_channel_unlock_napi(channel); 335 return spent; 336 } 337 338 /* Create event queue 339 * Event queue memory allocations are done only once. If the channel 340 * is reset, the memory buffer will be reused; this guards against 341 * errors during channel reset and also simplifies interrupt handling. 342 */ 343 static int ef4_probe_eventq(struct ef4_channel *channel) 344 { 345 struct ef4_nic *efx = channel->efx; 346 unsigned long entries; 347 348 netif_dbg(efx, probe, efx->net_dev, 349 "chan %d create event queue\n", channel->channel); 350 351 /* Build an event queue with room for one event per tx and rx buffer, 352 * plus some extra for link state events and MCDI completions. */ 353 entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128); 354 EF4_BUG_ON_PARANOID(entries > EF4_MAX_EVQ_SIZE); 355 channel->eventq_mask = max(entries, EF4_MIN_EVQ_SIZE) - 1; 356 357 return ef4_nic_probe_eventq(channel); 358 } 359 360 /* Prepare channel's event queue */ 361 static int ef4_init_eventq(struct ef4_channel *channel) 362 { 363 struct ef4_nic *efx = channel->efx; 364 int rc; 365 366 EF4_WARN_ON_PARANOID(channel->eventq_init); 367 368 netif_dbg(efx, drv, efx->net_dev, 369 "chan %d init event queue\n", channel->channel); 370 371 rc = ef4_nic_init_eventq(channel); 372 if (rc == 0) { 373 efx->type->push_irq_moderation(channel); 374 channel->eventq_read_ptr = 0; 375 channel->eventq_init = true; 376 } 377 return rc; 378 } 379 380 /* Enable event queue processing and NAPI */ 381 void ef4_start_eventq(struct ef4_channel *channel) 382 { 383 netif_dbg(channel->efx, ifup, channel->efx->net_dev, 384 "chan %d start event queue\n", channel->channel); 385 386 /* Make sure the NAPI handler sees the enabled flag set */ 387 channel->enabled = true; 388 smp_wmb(); 389 390 ef4_channel_enable(channel); 391 napi_enable(&channel->napi_str); 392 ef4_nic_eventq_read_ack(channel); 393 } 394 395 /* Disable event queue processing and NAPI */ 396 void ef4_stop_eventq(struct ef4_channel *channel) 397 { 398 if (!channel->enabled) 399 return; 400 401 napi_disable(&channel->napi_str); 402 while (!ef4_channel_disable(channel)) 403 usleep_range(1000, 20000); 404 channel->enabled = false; 405 } 406 407 static void ef4_fini_eventq(struct ef4_channel *channel) 408 { 409 if (!channel->eventq_init) 410 return; 411 412 netif_dbg(channel->efx, drv, channel->efx->net_dev, 413 "chan %d fini event queue\n", channel->channel); 414 415 ef4_nic_fini_eventq(channel); 416 channel->eventq_init = false; 417 } 418 419 static void ef4_remove_eventq(struct ef4_channel *channel) 420 { 421 netif_dbg(channel->efx, drv, channel->efx->net_dev, 422 "chan %d remove event queue\n", channel->channel); 423 424 ef4_nic_remove_eventq(channel); 425 } 426 427 /************************************************************************** 428 * 429 * Channel handling 430 * 431 *************************************************************************/ 432 433 /* Allocate and initialise a channel structure. */ 434 static struct ef4_channel * 435 ef4_alloc_channel(struct ef4_nic *efx, int i, struct ef4_channel *old_channel) 436 { 437 struct ef4_channel *channel; 438 struct ef4_rx_queue *rx_queue; 439 struct ef4_tx_queue *tx_queue; 440 int j; 441 442 channel = kzalloc(sizeof(*channel), GFP_KERNEL); 443 if (!channel) 444 return NULL; 445 446 channel->efx = efx; 447 channel->channel = i; 448 channel->type = &ef4_default_channel_type; 449 450 for (j = 0; j < EF4_TXQ_TYPES; j++) { 451 tx_queue = &channel->tx_queue[j]; 452 tx_queue->efx = efx; 453 tx_queue->queue = i * EF4_TXQ_TYPES + j; 454 tx_queue->channel = channel; 455 } 456 457 rx_queue = &channel->rx_queue; 458 rx_queue->efx = efx; 459 setup_timer(&rx_queue->slow_fill, ef4_rx_slow_fill, 460 (unsigned long)rx_queue); 461 462 return channel; 463 } 464 465 /* Allocate and initialise a channel structure, copying parameters 466 * (but not resources) from an old channel structure. 467 */ 468 static struct ef4_channel * 469 ef4_copy_channel(const struct ef4_channel *old_channel) 470 { 471 struct ef4_channel *channel; 472 struct ef4_rx_queue *rx_queue; 473 struct ef4_tx_queue *tx_queue; 474 int j; 475 476 channel = kmalloc(sizeof(*channel), GFP_KERNEL); 477 if (!channel) 478 return NULL; 479 480 *channel = *old_channel; 481 482 channel->napi_dev = NULL; 483 INIT_HLIST_NODE(&channel->napi_str.napi_hash_node); 484 channel->napi_str.napi_id = 0; 485 channel->napi_str.state = 0; 486 memset(&channel->eventq, 0, sizeof(channel->eventq)); 487 488 for (j = 0; j < EF4_TXQ_TYPES; j++) { 489 tx_queue = &channel->tx_queue[j]; 490 if (tx_queue->channel) 491 tx_queue->channel = channel; 492 tx_queue->buffer = NULL; 493 memset(&tx_queue->txd, 0, sizeof(tx_queue->txd)); 494 } 495 496 rx_queue = &channel->rx_queue; 497 rx_queue->buffer = NULL; 498 memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd)); 499 setup_timer(&rx_queue->slow_fill, ef4_rx_slow_fill, 500 (unsigned long)rx_queue); 501 502 return channel; 503 } 504 505 static int ef4_probe_channel(struct ef4_channel *channel) 506 { 507 struct ef4_tx_queue *tx_queue; 508 struct ef4_rx_queue *rx_queue; 509 int rc; 510 511 netif_dbg(channel->efx, probe, channel->efx->net_dev, 512 "creating channel %d\n", channel->channel); 513 514 rc = channel->type->pre_probe(channel); 515 if (rc) 516 goto fail; 517 518 rc = ef4_probe_eventq(channel); 519 if (rc) 520 goto fail; 521 522 ef4_for_each_channel_tx_queue(tx_queue, channel) { 523 rc = ef4_probe_tx_queue(tx_queue); 524 if (rc) 525 goto fail; 526 } 527 528 ef4_for_each_channel_rx_queue(rx_queue, channel) { 529 rc = ef4_probe_rx_queue(rx_queue); 530 if (rc) 531 goto fail; 532 } 533 534 return 0; 535 536 fail: 537 ef4_remove_channel(channel); 538 return rc; 539 } 540 541 static void 542 ef4_get_channel_name(struct ef4_channel *channel, char *buf, size_t len) 543 { 544 struct ef4_nic *efx = channel->efx; 545 const char *type; 546 int number; 547 548 number = channel->channel; 549 if (efx->tx_channel_offset == 0) { 550 type = ""; 551 } else if (channel->channel < efx->tx_channel_offset) { 552 type = "-rx"; 553 } else { 554 type = "-tx"; 555 number -= efx->tx_channel_offset; 556 } 557 snprintf(buf, len, "%s%s-%d", efx->name, type, number); 558 } 559 560 static void ef4_set_channel_names(struct ef4_nic *efx) 561 { 562 struct ef4_channel *channel; 563 564 ef4_for_each_channel(channel, efx) 565 channel->type->get_name(channel, 566 efx->msi_context[channel->channel].name, 567 sizeof(efx->msi_context[0].name)); 568 } 569 570 static int ef4_probe_channels(struct ef4_nic *efx) 571 { 572 struct ef4_channel *channel; 573 int rc; 574 575 /* Restart special buffer allocation */ 576 efx->next_buffer_table = 0; 577 578 /* Probe channels in reverse, so that any 'extra' channels 579 * use the start of the buffer table. This allows the traffic 580 * channels to be resized without moving them or wasting the 581 * entries before them. 582 */ 583 ef4_for_each_channel_rev(channel, efx) { 584 rc = ef4_probe_channel(channel); 585 if (rc) { 586 netif_err(efx, probe, efx->net_dev, 587 "failed to create channel %d\n", 588 channel->channel); 589 goto fail; 590 } 591 } 592 ef4_set_channel_names(efx); 593 594 return 0; 595 596 fail: 597 ef4_remove_channels(efx); 598 return rc; 599 } 600 601 /* Channels are shutdown and reinitialised whilst the NIC is running 602 * to propagate configuration changes (mtu, checksum offload), or 603 * to clear hardware error conditions 604 */ 605 static void ef4_start_datapath(struct ef4_nic *efx) 606 { 607 netdev_features_t old_features = efx->net_dev->features; 608 bool old_rx_scatter = efx->rx_scatter; 609 struct ef4_tx_queue *tx_queue; 610 struct ef4_rx_queue *rx_queue; 611 struct ef4_channel *channel; 612 size_t rx_buf_len; 613 614 /* Calculate the rx buffer allocation parameters required to 615 * support the current MTU, including padding for header 616 * alignment and overruns. 617 */ 618 efx->rx_dma_len = (efx->rx_prefix_size + 619 EF4_MAX_FRAME_LEN(efx->net_dev->mtu) + 620 efx->type->rx_buffer_padding); 621 rx_buf_len = (sizeof(struct ef4_rx_page_state) + 622 efx->rx_ip_align + efx->rx_dma_len); 623 if (rx_buf_len <= PAGE_SIZE) { 624 efx->rx_scatter = efx->type->always_rx_scatter; 625 efx->rx_buffer_order = 0; 626 } else if (efx->type->can_rx_scatter) { 627 BUILD_BUG_ON(EF4_RX_USR_BUF_SIZE % L1_CACHE_BYTES); 628 BUILD_BUG_ON(sizeof(struct ef4_rx_page_state) + 629 2 * ALIGN(NET_IP_ALIGN + EF4_RX_USR_BUF_SIZE, 630 EF4_RX_BUF_ALIGNMENT) > 631 PAGE_SIZE); 632 efx->rx_scatter = true; 633 efx->rx_dma_len = EF4_RX_USR_BUF_SIZE; 634 efx->rx_buffer_order = 0; 635 } else { 636 efx->rx_scatter = false; 637 efx->rx_buffer_order = get_order(rx_buf_len); 638 } 639 640 ef4_rx_config_page_split(efx); 641 if (efx->rx_buffer_order) 642 netif_dbg(efx, drv, efx->net_dev, 643 "RX buf len=%u; page order=%u batch=%u\n", 644 efx->rx_dma_len, efx->rx_buffer_order, 645 efx->rx_pages_per_batch); 646 else 647 netif_dbg(efx, drv, efx->net_dev, 648 "RX buf len=%u step=%u bpp=%u; page batch=%u\n", 649 efx->rx_dma_len, efx->rx_page_buf_step, 650 efx->rx_bufs_per_page, efx->rx_pages_per_batch); 651 652 /* Restore previously fixed features in hw_features and remove 653 * features which are fixed now 654 */ 655 efx->net_dev->hw_features |= efx->net_dev->features; 656 efx->net_dev->hw_features &= ~efx->fixed_features; 657 efx->net_dev->features |= efx->fixed_features; 658 if (efx->net_dev->features != old_features) 659 netdev_features_change(efx->net_dev); 660 661 /* RX filters may also have scatter-enabled flags */ 662 if (efx->rx_scatter != old_rx_scatter) 663 efx->type->filter_update_rx_scatter(efx); 664 665 /* We must keep at least one descriptor in a TX ring empty. 666 * We could avoid this when the queue size does not exactly 667 * match the hardware ring size, but it's not that important. 668 * Therefore we stop the queue when one more skb might fill 669 * the ring completely. We wake it when half way back to 670 * empty. 671 */ 672 efx->txq_stop_thresh = efx->txq_entries - ef4_tx_max_skb_descs(efx); 673 efx->txq_wake_thresh = efx->txq_stop_thresh / 2; 674 675 /* Initialise the channels */ 676 ef4_for_each_channel(channel, efx) { 677 ef4_for_each_channel_tx_queue(tx_queue, channel) { 678 ef4_init_tx_queue(tx_queue); 679 atomic_inc(&efx->active_queues); 680 } 681 682 ef4_for_each_channel_rx_queue(rx_queue, channel) { 683 ef4_init_rx_queue(rx_queue); 684 atomic_inc(&efx->active_queues); 685 ef4_stop_eventq(channel); 686 ef4_fast_push_rx_descriptors(rx_queue, false); 687 ef4_start_eventq(channel); 688 } 689 690 WARN_ON(channel->rx_pkt_n_frags); 691 } 692 693 if (netif_device_present(efx->net_dev)) 694 netif_tx_wake_all_queues(efx->net_dev); 695 } 696 697 static void ef4_stop_datapath(struct ef4_nic *efx) 698 { 699 struct ef4_channel *channel; 700 struct ef4_tx_queue *tx_queue; 701 struct ef4_rx_queue *rx_queue; 702 int rc; 703 704 EF4_ASSERT_RESET_SERIALISED(efx); 705 BUG_ON(efx->port_enabled); 706 707 /* Stop RX refill */ 708 ef4_for_each_channel(channel, efx) { 709 ef4_for_each_channel_rx_queue(rx_queue, channel) 710 rx_queue->refill_enabled = false; 711 } 712 713 ef4_for_each_channel(channel, efx) { 714 /* RX packet processing is pipelined, so wait for the 715 * NAPI handler to complete. At least event queue 0 716 * might be kept active by non-data events, so don't 717 * use napi_synchronize() but actually disable NAPI 718 * temporarily. 719 */ 720 if (ef4_channel_has_rx_queue(channel)) { 721 ef4_stop_eventq(channel); 722 ef4_start_eventq(channel); 723 } 724 } 725 726 rc = efx->type->fini_dmaq(efx); 727 if (rc && EF4_WORKAROUND_7803(efx)) { 728 /* Schedule a reset to recover from the flush failure. The 729 * descriptor caches reference memory we're about to free, 730 * but falcon_reconfigure_mac_wrapper() won't reconnect 731 * the MACs because of the pending reset. 732 */ 733 netif_err(efx, drv, efx->net_dev, 734 "Resetting to recover from flush failure\n"); 735 ef4_schedule_reset(efx, RESET_TYPE_ALL); 736 } else if (rc) { 737 netif_err(efx, drv, efx->net_dev, "failed to flush queues\n"); 738 } else { 739 netif_dbg(efx, drv, efx->net_dev, 740 "successfully flushed all queues\n"); 741 } 742 743 ef4_for_each_channel(channel, efx) { 744 ef4_for_each_channel_rx_queue(rx_queue, channel) 745 ef4_fini_rx_queue(rx_queue); 746 ef4_for_each_possible_channel_tx_queue(tx_queue, channel) 747 ef4_fini_tx_queue(tx_queue); 748 } 749 } 750 751 static void ef4_remove_channel(struct ef4_channel *channel) 752 { 753 struct ef4_tx_queue *tx_queue; 754 struct ef4_rx_queue *rx_queue; 755 756 netif_dbg(channel->efx, drv, channel->efx->net_dev, 757 "destroy chan %d\n", channel->channel); 758 759 ef4_for_each_channel_rx_queue(rx_queue, channel) 760 ef4_remove_rx_queue(rx_queue); 761 ef4_for_each_possible_channel_tx_queue(tx_queue, channel) 762 ef4_remove_tx_queue(tx_queue); 763 ef4_remove_eventq(channel); 764 channel->type->post_remove(channel); 765 } 766 767 static void ef4_remove_channels(struct ef4_nic *efx) 768 { 769 struct ef4_channel *channel; 770 771 ef4_for_each_channel(channel, efx) 772 ef4_remove_channel(channel); 773 } 774 775 int 776 ef4_realloc_channels(struct ef4_nic *efx, u32 rxq_entries, u32 txq_entries) 777 { 778 struct ef4_channel *other_channel[EF4_MAX_CHANNELS], *channel; 779 u32 old_rxq_entries, old_txq_entries; 780 unsigned i, next_buffer_table = 0; 781 int rc, rc2; 782 783 rc = ef4_check_disabled(efx); 784 if (rc) 785 return rc; 786 787 /* Not all channels should be reallocated. We must avoid 788 * reallocating their buffer table entries. 789 */ 790 ef4_for_each_channel(channel, efx) { 791 struct ef4_rx_queue *rx_queue; 792 struct ef4_tx_queue *tx_queue; 793 794 if (channel->type->copy) 795 continue; 796 next_buffer_table = max(next_buffer_table, 797 channel->eventq.index + 798 channel->eventq.entries); 799 ef4_for_each_channel_rx_queue(rx_queue, channel) 800 next_buffer_table = max(next_buffer_table, 801 rx_queue->rxd.index + 802 rx_queue->rxd.entries); 803 ef4_for_each_channel_tx_queue(tx_queue, channel) 804 next_buffer_table = max(next_buffer_table, 805 tx_queue->txd.index + 806 tx_queue->txd.entries); 807 } 808 809 ef4_device_detach_sync(efx); 810 ef4_stop_all(efx); 811 ef4_soft_disable_interrupts(efx); 812 813 /* Clone channels (where possible) */ 814 memset(other_channel, 0, sizeof(other_channel)); 815 for (i = 0; i < efx->n_channels; i++) { 816 channel = efx->channel[i]; 817 if (channel->type->copy) 818 channel = channel->type->copy(channel); 819 if (!channel) { 820 rc = -ENOMEM; 821 goto out; 822 } 823 other_channel[i] = channel; 824 } 825 826 /* Swap entry counts and channel pointers */ 827 old_rxq_entries = efx->rxq_entries; 828 old_txq_entries = efx->txq_entries; 829 efx->rxq_entries = rxq_entries; 830 efx->txq_entries = txq_entries; 831 for (i = 0; i < efx->n_channels; i++) { 832 channel = efx->channel[i]; 833 efx->channel[i] = other_channel[i]; 834 other_channel[i] = channel; 835 } 836 837 /* Restart buffer table allocation */ 838 efx->next_buffer_table = next_buffer_table; 839 840 for (i = 0; i < efx->n_channels; i++) { 841 channel = efx->channel[i]; 842 if (!channel->type->copy) 843 continue; 844 rc = ef4_probe_channel(channel); 845 if (rc) 846 goto rollback; 847 ef4_init_napi_channel(efx->channel[i]); 848 } 849 850 out: 851 /* Destroy unused channel structures */ 852 for (i = 0; i < efx->n_channels; i++) { 853 channel = other_channel[i]; 854 if (channel && channel->type->copy) { 855 ef4_fini_napi_channel(channel); 856 ef4_remove_channel(channel); 857 kfree(channel); 858 } 859 } 860 861 rc2 = ef4_soft_enable_interrupts(efx); 862 if (rc2) { 863 rc = rc ? rc : rc2; 864 netif_err(efx, drv, efx->net_dev, 865 "unable to restart interrupts on channel reallocation\n"); 866 ef4_schedule_reset(efx, RESET_TYPE_DISABLE); 867 } else { 868 ef4_start_all(efx); 869 netif_device_attach(efx->net_dev); 870 } 871 return rc; 872 873 rollback: 874 /* Swap back */ 875 efx->rxq_entries = old_rxq_entries; 876 efx->txq_entries = old_txq_entries; 877 for (i = 0; i < efx->n_channels; i++) { 878 channel = efx->channel[i]; 879 efx->channel[i] = other_channel[i]; 880 other_channel[i] = channel; 881 } 882 goto out; 883 } 884 885 void ef4_schedule_slow_fill(struct ef4_rx_queue *rx_queue) 886 { 887 mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(100)); 888 } 889 890 static const struct ef4_channel_type ef4_default_channel_type = { 891 .pre_probe = ef4_channel_dummy_op_int, 892 .post_remove = ef4_channel_dummy_op_void, 893 .get_name = ef4_get_channel_name, 894 .copy = ef4_copy_channel, 895 .keep_eventq = false, 896 }; 897 898 int ef4_channel_dummy_op_int(struct ef4_channel *channel) 899 { 900 return 0; 901 } 902 903 void ef4_channel_dummy_op_void(struct ef4_channel *channel) 904 { 905 } 906 907 /************************************************************************** 908 * 909 * Port handling 910 * 911 **************************************************************************/ 912 913 /* This ensures that the kernel is kept informed (via 914 * netif_carrier_on/off) of the link status, and also maintains the 915 * link status's stop on the port's TX queue. 916 */ 917 void ef4_link_status_changed(struct ef4_nic *efx) 918 { 919 struct ef4_link_state *link_state = &efx->link_state; 920 921 /* SFC Bug 5356: A net_dev notifier is registered, so we must ensure 922 * that no events are triggered between unregister_netdev() and the 923 * driver unloading. A more general condition is that NETDEV_CHANGE 924 * can only be generated between NETDEV_UP and NETDEV_DOWN */ 925 if (!netif_running(efx->net_dev)) 926 return; 927 928 if (link_state->up != netif_carrier_ok(efx->net_dev)) { 929 efx->n_link_state_changes++; 930 931 if (link_state->up) 932 netif_carrier_on(efx->net_dev); 933 else 934 netif_carrier_off(efx->net_dev); 935 } 936 937 /* Status message for kernel log */ 938 if (link_state->up) 939 netif_info(efx, link, efx->net_dev, 940 "link up at %uMbps %s-duplex (MTU %d)\n", 941 link_state->speed, link_state->fd ? "full" : "half", 942 efx->net_dev->mtu); 943 else 944 netif_info(efx, link, efx->net_dev, "link down\n"); 945 } 946 947 void ef4_link_set_advertising(struct ef4_nic *efx, u32 advertising) 948 { 949 efx->link_advertising = advertising; 950 if (advertising) { 951 if (advertising & ADVERTISED_Pause) 952 efx->wanted_fc |= (EF4_FC_TX | EF4_FC_RX); 953 else 954 efx->wanted_fc &= ~(EF4_FC_TX | EF4_FC_RX); 955 if (advertising & ADVERTISED_Asym_Pause) 956 efx->wanted_fc ^= EF4_FC_TX; 957 } 958 } 959 960 void ef4_link_set_wanted_fc(struct ef4_nic *efx, u8 wanted_fc) 961 { 962 efx->wanted_fc = wanted_fc; 963 if (efx->link_advertising) { 964 if (wanted_fc & EF4_FC_RX) 965 efx->link_advertising |= (ADVERTISED_Pause | 966 ADVERTISED_Asym_Pause); 967 else 968 efx->link_advertising &= ~(ADVERTISED_Pause | 969 ADVERTISED_Asym_Pause); 970 if (wanted_fc & EF4_FC_TX) 971 efx->link_advertising ^= ADVERTISED_Asym_Pause; 972 } 973 } 974 975 static void ef4_fini_port(struct ef4_nic *efx); 976 977 /* We assume that efx->type->reconfigure_mac will always try to sync RX 978 * filters and therefore needs to read-lock the filter table against freeing 979 */ 980 void ef4_mac_reconfigure(struct ef4_nic *efx) 981 { 982 down_read(&efx->filter_sem); 983 efx->type->reconfigure_mac(efx); 984 up_read(&efx->filter_sem); 985 } 986 987 /* Push loopback/power/transmit disable settings to the PHY, and reconfigure 988 * the MAC appropriately. All other PHY configuration changes are pushed 989 * through phy_op->set_settings(), and pushed asynchronously to the MAC 990 * through ef4_monitor(). 991 * 992 * Callers must hold the mac_lock 993 */ 994 int __ef4_reconfigure_port(struct ef4_nic *efx) 995 { 996 enum ef4_phy_mode phy_mode; 997 int rc; 998 999 WARN_ON(!mutex_is_locked(&efx->mac_lock)); 1000 1001 /* Disable PHY transmit in mac level loopbacks */ 1002 phy_mode = efx->phy_mode; 1003 if (LOOPBACK_INTERNAL(efx)) 1004 efx->phy_mode |= PHY_MODE_TX_DISABLED; 1005 else 1006 efx->phy_mode &= ~PHY_MODE_TX_DISABLED; 1007 1008 rc = efx->type->reconfigure_port(efx); 1009 1010 if (rc) 1011 efx->phy_mode = phy_mode; 1012 1013 return rc; 1014 } 1015 1016 /* Reinitialise the MAC to pick up new PHY settings, even if the port is 1017 * disabled. */ 1018 int ef4_reconfigure_port(struct ef4_nic *efx) 1019 { 1020 int rc; 1021 1022 EF4_ASSERT_RESET_SERIALISED(efx); 1023 1024 mutex_lock(&efx->mac_lock); 1025 rc = __ef4_reconfigure_port(efx); 1026 mutex_unlock(&efx->mac_lock); 1027 1028 return rc; 1029 } 1030 1031 /* Asynchronous work item for changing MAC promiscuity and multicast 1032 * hash. Avoid a drain/rx_ingress enable by reconfiguring the current 1033 * MAC directly. */ 1034 static void ef4_mac_work(struct work_struct *data) 1035 { 1036 struct ef4_nic *efx = container_of(data, struct ef4_nic, mac_work); 1037 1038 mutex_lock(&efx->mac_lock); 1039 if (efx->port_enabled) 1040 ef4_mac_reconfigure(efx); 1041 mutex_unlock(&efx->mac_lock); 1042 } 1043 1044 static int ef4_probe_port(struct ef4_nic *efx) 1045 { 1046 int rc; 1047 1048 netif_dbg(efx, probe, efx->net_dev, "create port\n"); 1049 1050 if (phy_flash_cfg) 1051 efx->phy_mode = PHY_MODE_SPECIAL; 1052 1053 /* Connect up MAC/PHY operations table */ 1054 rc = efx->type->probe_port(efx); 1055 if (rc) 1056 return rc; 1057 1058 /* Initialise MAC address to permanent address */ 1059 ether_addr_copy(efx->net_dev->dev_addr, efx->net_dev->perm_addr); 1060 1061 return 0; 1062 } 1063 1064 static int ef4_init_port(struct ef4_nic *efx) 1065 { 1066 int rc; 1067 1068 netif_dbg(efx, drv, efx->net_dev, "init port\n"); 1069 1070 mutex_lock(&efx->mac_lock); 1071 1072 rc = efx->phy_op->init(efx); 1073 if (rc) 1074 goto fail1; 1075 1076 efx->port_initialized = true; 1077 1078 /* Reconfigure the MAC before creating dma queues (required for 1079 * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */ 1080 ef4_mac_reconfigure(efx); 1081 1082 /* Ensure the PHY advertises the correct flow control settings */ 1083 rc = efx->phy_op->reconfigure(efx); 1084 if (rc && rc != -EPERM) 1085 goto fail2; 1086 1087 mutex_unlock(&efx->mac_lock); 1088 return 0; 1089 1090 fail2: 1091 efx->phy_op->fini(efx); 1092 fail1: 1093 mutex_unlock(&efx->mac_lock); 1094 return rc; 1095 } 1096 1097 static void ef4_start_port(struct ef4_nic *efx) 1098 { 1099 netif_dbg(efx, ifup, efx->net_dev, "start port\n"); 1100 BUG_ON(efx->port_enabled); 1101 1102 mutex_lock(&efx->mac_lock); 1103 efx->port_enabled = true; 1104 1105 /* Ensure MAC ingress/egress is enabled */ 1106 ef4_mac_reconfigure(efx); 1107 1108 mutex_unlock(&efx->mac_lock); 1109 } 1110 1111 /* Cancel work for MAC reconfiguration, periodic hardware monitoring 1112 * and the async self-test, wait for them to finish and prevent them 1113 * being scheduled again. This doesn't cover online resets, which 1114 * should only be cancelled when removing the device. 1115 */ 1116 static void ef4_stop_port(struct ef4_nic *efx) 1117 { 1118 netif_dbg(efx, ifdown, efx->net_dev, "stop port\n"); 1119 1120 EF4_ASSERT_RESET_SERIALISED(efx); 1121 1122 mutex_lock(&efx->mac_lock); 1123 efx->port_enabled = false; 1124 mutex_unlock(&efx->mac_lock); 1125 1126 /* Serialise against ef4_set_multicast_list() */ 1127 netif_addr_lock_bh(efx->net_dev); 1128 netif_addr_unlock_bh(efx->net_dev); 1129 1130 cancel_delayed_work_sync(&efx->monitor_work); 1131 ef4_selftest_async_cancel(efx); 1132 cancel_work_sync(&efx->mac_work); 1133 } 1134 1135 static void ef4_fini_port(struct ef4_nic *efx) 1136 { 1137 netif_dbg(efx, drv, efx->net_dev, "shut down port\n"); 1138 1139 if (!efx->port_initialized) 1140 return; 1141 1142 efx->phy_op->fini(efx); 1143 efx->port_initialized = false; 1144 1145 efx->link_state.up = false; 1146 ef4_link_status_changed(efx); 1147 } 1148 1149 static void ef4_remove_port(struct ef4_nic *efx) 1150 { 1151 netif_dbg(efx, drv, efx->net_dev, "destroying port\n"); 1152 1153 efx->type->remove_port(efx); 1154 } 1155 1156 /************************************************************************** 1157 * 1158 * NIC handling 1159 * 1160 **************************************************************************/ 1161 1162 static LIST_HEAD(ef4_primary_list); 1163 static LIST_HEAD(ef4_unassociated_list); 1164 1165 static bool ef4_same_controller(struct ef4_nic *left, struct ef4_nic *right) 1166 { 1167 return left->type == right->type && 1168 left->vpd_sn && right->vpd_sn && 1169 !strcmp(left->vpd_sn, right->vpd_sn); 1170 } 1171 1172 static void ef4_associate(struct ef4_nic *efx) 1173 { 1174 struct ef4_nic *other, *next; 1175 1176 if (efx->primary == efx) { 1177 /* Adding primary function; look for secondaries */ 1178 1179 netif_dbg(efx, probe, efx->net_dev, "adding to primary list\n"); 1180 list_add_tail(&efx->node, &ef4_primary_list); 1181 1182 list_for_each_entry_safe(other, next, &ef4_unassociated_list, 1183 node) { 1184 if (ef4_same_controller(efx, other)) { 1185 list_del(&other->node); 1186 netif_dbg(other, probe, other->net_dev, 1187 "moving to secondary list of %s %s\n", 1188 pci_name(efx->pci_dev), 1189 efx->net_dev->name); 1190 list_add_tail(&other->node, 1191 &efx->secondary_list); 1192 other->primary = efx; 1193 } 1194 } 1195 } else { 1196 /* Adding secondary function; look for primary */ 1197 1198 list_for_each_entry(other, &ef4_primary_list, node) { 1199 if (ef4_same_controller(efx, other)) { 1200 netif_dbg(efx, probe, efx->net_dev, 1201 "adding to secondary list of %s %s\n", 1202 pci_name(other->pci_dev), 1203 other->net_dev->name); 1204 list_add_tail(&efx->node, 1205 &other->secondary_list); 1206 efx->primary = other; 1207 return; 1208 } 1209 } 1210 1211 netif_dbg(efx, probe, efx->net_dev, 1212 "adding to unassociated list\n"); 1213 list_add_tail(&efx->node, &ef4_unassociated_list); 1214 } 1215 } 1216 1217 static void ef4_dissociate(struct ef4_nic *efx) 1218 { 1219 struct ef4_nic *other, *next; 1220 1221 list_del(&efx->node); 1222 efx->primary = NULL; 1223 1224 list_for_each_entry_safe(other, next, &efx->secondary_list, node) { 1225 list_del(&other->node); 1226 netif_dbg(other, probe, other->net_dev, 1227 "moving to unassociated list\n"); 1228 list_add_tail(&other->node, &ef4_unassociated_list); 1229 other->primary = NULL; 1230 } 1231 } 1232 1233 /* This configures the PCI device to enable I/O and DMA. */ 1234 static int ef4_init_io(struct ef4_nic *efx) 1235 { 1236 struct pci_dev *pci_dev = efx->pci_dev; 1237 dma_addr_t dma_mask = efx->type->max_dma_mask; 1238 unsigned int mem_map_size = efx->type->mem_map_size(efx); 1239 int rc, bar; 1240 1241 netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n"); 1242 1243 bar = efx->type->mem_bar; 1244 1245 rc = pci_enable_device(pci_dev); 1246 if (rc) { 1247 netif_err(efx, probe, efx->net_dev, 1248 "failed to enable PCI device\n"); 1249 goto fail1; 1250 } 1251 1252 pci_set_master(pci_dev); 1253 1254 /* Set the PCI DMA mask. Try all possibilities from our 1255 * genuine mask down to 32 bits, because some architectures 1256 * (e.g. x86_64 with iommu_sac_force set) will allow 40 bit 1257 * masks event though they reject 46 bit masks. 1258 */ 1259 while (dma_mask > 0x7fffffffUL) { 1260 rc = dma_set_mask_and_coherent(&pci_dev->dev, dma_mask); 1261 if (rc == 0) 1262 break; 1263 dma_mask >>= 1; 1264 } 1265 if (rc) { 1266 netif_err(efx, probe, efx->net_dev, 1267 "could not find a suitable DMA mask\n"); 1268 goto fail2; 1269 } 1270 netif_dbg(efx, probe, efx->net_dev, 1271 "using DMA mask %llx\n", (unsigned long long) dma_mask); 1272 1273 efx->membase_phys = pci_resource_start(efx->pci_dev, bar); 1274 rc = pci_request_region(pci_dev, bar, "sfc"); 1275 if (rc) { 1276 netif_err(efx, probe, efx->net_dev, 1277 "request for memory BAR failed\n"); 1278 rc = -EIO; 1279 goto fail3; 1280 } 1281 efx->membase = ioremap_nocache(efx->membase_phys, mem_map_size); 1282 if (!efx->membase) { 1283 netif_err(efx, probe, efx->net_dev, 1284 "could not map memory BAR at %llx+%x\n", 1285 (unsigned long long)efx->membase_phys, mem_map_size); 1286 rc = -ENOMEM; 1287 goto fail4; 1288 } 1289 netif_dbg(efx, probe, efx->net_dev, 1290 "memory BAR at %llx+%x (virtual %p)\n", 1291 (unsigned long long)efx->membase_phys, mem_map_size, 1292 efx->membase); 1293 1294 return 0; 1295 1296 fail4: 1297 pci_release_region(efx->pci_dev, bar); 1298 fail3: 1299 efx->membase_phys = 0; 1300 fail2: 1301 pci_disable_device(efx->pci_dev); 1302 fail1: 1303 return rc; 1304 } 1305 1306 static void ef4_fini_io(struct ef4_nic *efx) 1307 { 1308 int bar; 1309 1310 netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n"); 1311 1312 if (efx->membase) { 1313 iounmap(efx->membase); 1314 efx->membase = NULL; 1315 } 1316 1317 if (efx->membase_phys) { 1318 bar = efx->type->mem_bar; 1319 pci_release_region(efx->pci_dev, bar); 1320 efx->membase_phys = 0; 1321 } 1322 1323 /* Don't disable bus-mastering if VFs are assigned */ 1324 if (!pci_vfs_assigned(efx->pci_dev)) 1325 pci_disable_device(efx->pci_dev); 1326 } 1327 1328 void ef4_set_default_rx_indir_table(struct ef4_nic *efx) 1329 { 1330 size_t i; 1331 1332 for (i = 0; i < ARRAY_SIZE(efx->rx_indir_table); i++) 1333 efx->rx_indir_table[i] = 1334 ethtool_rxfh_indir_default(i, efx->rss_spread); 1335 } 1336 1337 static unsigned int ef4_wanted_parallelism(struct ef4_nic *efx) 1338 { 1339 cpumask_var_t thread_mask; 1340 unsigned int count; 1341 int cpu; 1342 1343 if (rss_cpus) { 1344 count = rss_cpus; 1345 } else { 1346 if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) { 1347 netif_warn(efx, probe, efx->net_dev, 1348 "RSS disabled due to allocation failure\n"); 1349 return 1; 1350 } 1351 1352 count = 0; 1353 for_each_online_cpu(cpu) { 1354 if (!cpumask_test_cpu(cpu, thread_mask)) { 1355 ++count; 1356 cpumask_or(thread_mask, thread_mask, 1357 topology_sibling_cpumask(cpu)); 1358 } 1359 } 1360 1361 free_cpumask_var(thread_mask); 1362 } 1363 1364 return count; 1365 } 1366 1367 /* Probe the number and type of interrupts we are able to obtain, and 1368 * the resulting numbers of channels and RX queues. 1369 */ 1370 static int ef4_probe_interrupts(struct ef4_nic *efx) 1371 { 1372 unsigned int extra_channels = 0; 1373 unsigned int i, j; 1374 int rc; 1375 1376 for (i = 0; i < EF4_MAX_EXTRA_CHANNELS; i++) 1377 if (efx->extra_channel_type[i]) 1378 ++extra_channels; 1379 1380 if (efx->interrupt_mode == EF4_INT_MODE_MSIX) { 1381 struct msix_entry xentries[EF4_MAX_CHANNELS]; 1382 unsigned int n_channels; 1383 1384 n_channels = ef4_wanted_parallelism(efx); 1385 if (ef4_separate_tx_channels) 1386 n_channels *= 2; 1387 n_channels += extra_channels; 1388 n_channels = min(n_channels, efx->max_channels); 1389 1390 for (i = 0; i < n_channels; i++) 1391 xentries[i].entry = i; 1392 rc = pci_enable_msix_range(efx->pci_dev, 1393 xentries, 1, n_channels); 1394 if (rc < 0) { 1395 /* Fall back to single channel MSI */ 1396 efx->interrupt_mode = EF4_INT_MODE_MSI; 1397 netif_err(efx, drv, efx->net_dev, 1398 "could not enable MSI-X\n"); 1399 } else if (rc < n_channels) { 1400 netif_err(efx, drv, efx->net_dev, 1401 "WARNING: Insufficient MSI-X vectors" 1402 " available (%d < %u).\n", rc, n_channels); 1403 netif_err(efx, drv, efx->net_dev, 1404 "WARNING: Performance may be reduced.\n"); 1405 n_channels = rc; 1406 } 1407 1408 if (rc > 0) { 1409 efx->n_channels = n_channels; 1410 if (n_channels > extra_channels) 1411 n_channels -= extra_channels; 1412 if (ef4_separate_tx_channels) { 1413 efx->n_tx_channels = min(max(n_channels / 2, 1414 1U), 1415 efx->max_tx_channels); 1416 efx->n_rx_channels = max(n_channels - 1417 efx->n_tx_channels, 1418 1U); 1419 } else { 1420 efx->n_tx_channels = min(n_channels, 1421 efx->max_tx_channels); 1422 efx->n_rx_channels = n_channels; 1423 } 1424 for (i = 0; i < efx->n_channels; i++) 1425 ef4_get_channel(efx, i)->irq = 1426 xentries[i].vector; 1427 } 1428 } 1429 1430 /* Try single interrupt MSI */ 1431 if (efx->interrupt_mode == EF4_INT_MODE_MSI) { 1432 efx->n_channels = 1; 1433 efx->n_rx_channels = 1; 1434 efx->n_tx_channels = 1; 1435 rc = pci_enable_msi(efx->pci_dev); 1436 if (rc == 0) { 1437 ef4_get_channel(efx, 0)->irq = efx->pci_dev->irq; 1438 } else { 1439 netif_err(efx, drv, efx->net_dev, 1440 "could not enable MSI\n"); 1441 efx->interrupt_mode = EF4_INT_MODE_LEGACY; 1442 } 1443 } 1444 1445 /* Assume legacy interrupts */ 1446 if (efx->interrupt_mode == EF4_INT_MODE_LEGACY) { 1447 efx->n_channels = 1 + (ef4_separate_tx_channels ? 1 : 0); 1448 efx->n_rx_channels = 1; 1449 efx->n_tx_channels = 1; 1450 efx->legacy_irq = efx->pci_dev->irq; 1451 } 1452 1453 /* Assign extra channels if possible */ 1454 j = efx->n_channels; 1455 for (i = 0; i < EF4_MAX_EXTRA_CHANNELS; i++) { 1456 if (!efx->extra_channel_type[i]) 1457 continue; 1458 if (efx->interrupt_mode != EF4_INT_MODE_MSIX || 1459 efx->n_channels <= extra_channels) { 1460 efx->extra_channel_type[i]->handle_no_channel(efx); 1461 } else { 1462 --j; 1463 ef4_get_channel(efx, j)->type = 1464 efx->extra_channel_type[i]; 1465 } 1466 } 1467 1468 efx->rss_spread = efx->n_rx_channels; 1469 1470 return 0; 1471 } 1472 1473 static int ef4_soft_enable_interrupts(struct ef4_nic *efx) 1474 { 1475 struct ef4_channel *channel, *end_channel; 1476 int rc; 1477 1478 BUG_ON(efx->state == STATE_DISABLED); 1479 1480 efx->irq_soft_enabled = true; 1481 smp_wmb(); 1482 1483 ef4_for_each_channel(channel, efx) { 1484 if (!channel->type->keep_eventq) { 1485 rc = ef4_init_eventq(channel); 1486 if (rc) 1487 goto fail; 1488 } 1489 ef4_start_eventq(channel); 1490 } 1491 1492 return 0; 1493 fail: 1494 end_channel = channel; 1495 ef4_for_each_channel(channel, efx) { 1496 if (channel == end_channel) 1497 break; 1498 ef4_stop_eventq(channel); 1499 if (!channel->type->keep_eventq) 1500 ef4_fini_eventq(channel); 1501 } 1502 1503 return rc; 1504 } 1505 1506 static void ef4_soft_disable_interrupts(struct ef4_nic *efx) 1507 { 1508 struct ef4_channel *channel; 1509 1510 if (efx->state == STATE_DISABLED) 1511 return; 1512 1513 efx->irq_soft_enabled = false; 1514 smp_wmb(); 1515 1516 if (efx->legacy_irq) 1517 synchronize_irq(efx->legacy_irq); 1518 1519 ef4_for_each_channel(channel, efx) { 1520 if (channel->irq) 1521 synchronize_irq(channel->irq); 1522 1523 ef4_stop_eventq(channel); 1524 if (!channel->type->keep_eventq) 1525 ef4_fini_eventq(channel); 1526 } 1527 } 1528 1529 static int ef4_enable_interrupts(struct ef4_nic *efx) 1530 { 1531 struct ef4_channel *channel, *end_channel; 1532 int rc; 1533 1534 BUG_ON(efx->state == STATE_DISABLED); 1535 1536 if (efx->eeh_disabled_legacy_irq) { 1537 enable_irq(efx->legacy_irq); 1538 efx->eeh_disabled_legacy_irq = false; 1539 } 1540 1541 efx->type->irq_enable_master(efx); 1542 1543 ef4_for_each_channel(channel, efx) { 1544 if (channel->type->keep_eventq) { 1545 rc = ef4_init_eventq(channel); 1546 if (rc) 1547 goto fail; 1548 } 1549 } 1550 1551 rc = ef4_soft_enable_interrupts(efx); 1552 if (rc) 1553 goto fail; 1554 1555 return 0; 1556 1557 fail: 1558 end_channel = channel; 1559 ef4_for_each_channel(channel, efx) { 1560 if (channel == end_channel) 1561 break; 1562 if (channel->type->keep_eventq) 1563 ef4_fini_eventq(channel); 1564 } 1565 1566 efx->type->irq_disable_non_ev(efx); 1567 1568 return rc; 1569 } 1570 1571 static void ef4_disable_interrupts(struct ef4_nic *efx) 1572 { 1573 struct ef4_channel *channel; 1574 1575 ef4_soft_disable_interrupts(efx); 1576 1577 ef4_for_each_channel(channel, efx) { 1578 if (channel->type->keep_eventq) 1579 ef4_fini_eventq(channel); 1580 } 1581 1582 efx->type->irq_disable_non_ev(efx); 1583 } 1584 1585 static void ef4_remove_interrupts(struct ef4_nic *efx) 1586 { 1587 struct ef4_channel *channel; 1588 1589 /* Remove MSI/MSI-X interrupts */ 1590 ef4_for_each_channel(channel, efx) 1591 channel->irq = 0; 1592 pci_disable_msi(efx->pci_dev); 1593 pci_disable_msix(efx->pci_dev); 1594 1595 /* Remove legacy interrupt */ 1596 efx->legacy_irq = 0; 1597 } 1598 1599 static void ef4_set_channels(struct ef4_nic *efx) 1600 { 1601 struct ef4_channel *channel; 1602 struct ef4_tx_queue *tx_queue; 1603 1604 efx->tx_channel_offset = 1605 ef4_separate_tx_channels ? 1606 efx->n_channels - efx->n_tx_channels : 0; 1607 1608 /* We need to mark which channels really have RX and TX 1609 * queues, and adjust the TX queue numbers if we have separate 1610 * RX-only and TX-only channels. 1611 */ 1612 ef4_for_each_channel(channel, efx) { 1613 if (channel->channel < efx->n_rx_channels) 1614 channel->rx_queue.core_index = channel->channel; 1615 else 1616 channel->rx_queue.core_index = -1; 1617 1618 ef4_for_each_channel_tx_queue(tx_queue, channel) 1619 tx_queue->queue -= (efx->tx_channel_offset * 1620 EF4_TXQ_TYPES); 1621 } 1622 } 1623 1624 static int ef4_probe_nic(struct ef4_nic *efx) 1625 { 1626 int rc; 1627 1628 netif_dbg(efx, probe, efx->net_dev, "creating NIC\n"); 1629 1630 /* Carry out hardware-type specific initialisation */ 1631 rc = efx->type->probe(efx); 1632 if (rc) 1633 return rc; 1634 1635 do { 1636 if (!efx->max_channels || !efx->max_tx_channels) { 1637 netif_err(efx, drv, efx->net_dev, 1638 "Insufficient resources to allocate" 1639 " any channels\n"); 1640 rc = -ENOSPC; 1641 goto fail1; 1642 } 1643 1644 /* Determine the number of channels and queues by trying 1645 * to hook in MSI-X interrupts. 1646 */ 1647 rc = ef4_probe_interrupts(efx); 1648 if (rc) 1649 goto fail1; 1650 1651 ef4_set_channels(efx); 1652 1653 /* dimension_resources can fail with EAGAIN */ 1654 rc = efx->type->dimension_resources(efx); 1655 if (rc != 0 && rc != -EAGAIN) 1656 goto fail2; 1657 1658 if (rc == -EAGAIN) 1659 /* try again with new max_channels */ 1660 ef4_remove_interrupts(efx); 1661 1662 } while (rc == -EAGAIN); 1663 1664 if (efx->n_channels > 1) 1665 netdev_rss_key_fill(&efx->rx_hash_key, 1666 sizeof(efx->rx_hash_key)); 1667 ef4_set_default_rx_indir_table(efx); 1668 1669 netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels); 1670 netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels); 1671 1672 /* Initialise the interrupt moderation settings */ 1673 efx->irq_mod_step_us = DIV_ROUND_UP(efx->timer_quantum_ns, 1000); 1674 ef4_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true, 1675 true); 1676 1677 return 0; 1678 1679 fail2: 1680 ef4_remove_interrupts(efx); 1681 fail1: 1682 efx->type->remove(efx); 1683 return rc; 1684 } 1685 1686 static void ef4_remove_nic(struct ef4_nic *efx) 1687 { 1688 netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n"); 1689 1690 ef4_remove_interrupts(efx); 1691 efx->type->remove(efx); 1692 } 1693 1694 static int ef4_probe_filters(struct ef4_nic *efx) 1695 { 1696 int rc; 1697 1698 spin_lock_init(&efx->filter_lock); 1699 init_rwsem(&efx->filter_sem); 1700 mutex_lock(&efx->mac_lock); 1701 down_write(&efx->filter_sem); 1702 rc = efx->type->filter_table_probe(efx); 1703 if (rc) 1704 goto out_unlock; 1705 1706 #ifdef CONFIG_RFS_ACCEL 1707 if (efx->type->offload_features & NETIF_F_NTUPLE) { 1708 struct ef4_channel *channel; 1709 int i, success = 1; 1710 1711 ef4_for_each_channel(channel, efx) { 1712 channel->rps_flow_id = 1713 kcalloc(efx->type->max_rx_ip_filters, 1714 sizeof(*channel->rps_flow_id), 1715 GFP_KERNEL); 1716 if (!channel->rps_flow_id) 1717 success = 0; 1718 else 1719 for (i = 0; 1720 i < efx->type->max_rx_ip_filters; 1721 ++i) 1722 channel->rps_flow_id[i] = 1723 RPS_FLOW_ID_INVALID; 1724 } 1725 1726 if (!success) { 1727 ef4_for_each_channel(channel, efx) 1728 kfree(channel->rps_flow_id); 1729 efx->type->filter_table_remove(efx); 1730 rc = -ENOMEM; 1731 goto out_unlock; 1732 } 1733 1734 efx->rps_expire_index = efx->rps_expire_channel = 0; 1735 } 1736 #endif 1737 out_unlock: 1738 up_write(&efx->filter_sem); 1739 mutex_unlock(&efx->mac_lock); 1740 return rc; 1741 } 1742 1743 static void ef4_remove_filters(struct ef4_nic *efx) 1744 { 1745 #ifdef CONFIG_RFS_ACCEL 1746 struct ef4_channel *channel; 1747 1748 ef4_for_each_channel(channel, efx) 1749 kfree(channel->rps_flow_id); 1750 #endif 1751 down_write(&efx->filter_sem); 1752 efx->type->filter_table_remove(efx); 1753 up_write(&efx->filter_sem); 1754 } 1755 1756 static void ef4_restore_filters(struct ef4_nic *efx) 1757 { 1758 down_read(&efx->filter_sem); 1759 efx->type->filter_table_restore(efx); 1760 up_read(&efx->filter_sem); 1761 } 1762 1763 /************************************************************************** 1764 * 1765 * NIC startup/shutdown 1766 * 1767 *************************************************************************/ 1768 1769 static int ef4_probe_all(struct ef4_nic *efx) 1770 { 1771 int rc; 1772 1773 rc = ef4_probe_nic(efx); 1774 if (rc) { 1775 netif_err(efx, probe, efx->net_dev, "failed to create NIC\n"); 1776 goto fail1; 1777 } 1778 1779 rc = ef4_probe_port(efx); 1780 if (rc) { 1781 netif_err(efx, probe, efx->net_dev, "failed to create port\n"); 1782 goto fail2; 1783 } 1784 1785 BUILD_BUG_ON(EF4_DEFAULT_DMAQ_SIZE < EF4_RXQ_MIN_ENT); 1786 if (WARN_ON(EF4_DEFAULT_DMAQ_SIZE < EF4_TXQ_MIN_ENT(efx))) { 1787 rc = -EINVAL; 1788 goto fail3; 1789 } 1790 efx->rxq_entries = efx->txq_entries = EF4_DEFAULT_DMAQ_SIZE; 1791 1792 rc = ef4_probe_filters(efx); 1793 if (rc) { 1794 netif_err(efx, probe, efx->net_dev, 1795 "failed to create filter tables\n"); 1796 goto fail4; 1797 } 1798 1799 rc = ef4_probe_channels(efx); 1800 if (rc) 1801 goto fail5; 1802 1803 return 0; 1804 1805 fail5: 1806 ef4_remove_filters(efx); 1807 fail4: 1808 fail3: 1809 ef4_remove_port(efx); 1810 fail2: 1811 ef4_remove_nic(efx); 1812 fail1: 1813 return rc; 1814 } 1815 1816 /* If the interface is supposed to be running but is not, start 1817 * the hardware and software data path, regular activity for the port 1818 * (MAC statistics, link polling, etc.) and schedule the port to be 1819 * reconfigured. Interrupts must already be enabled. This function 1820 * is safe to call multiple times, so long as the NIC is not disabled. 1821 * Requires the RTNL lock. 1822 */ 1823 static void ef4_start_all(struct ef4_nic *efx) 1824 { 1825 EF4_ASSERT_RESET_SERIALISED(efx); 1826 BUG_ON(efx->state == STATE_DISABLED); 1827 1828 /* Check that it is appropriate to restart the interface. All 1829 * of these flags are safe to read under just the rtnl lock */ 1830 if (efx->port_enabled || !netif_running(efx->net_dev) || 1831 efx->reset_pending) 1832 return; 1833 1834 ef4_start_port(efx); 1835 ef4_start_datapath(efx); 1836 1837 /* Start the hardware monitor if there is one */ 1838 if (efx->type->monitor != NULL) 1839 queue_delayed_work(efx->workqueue, &efx->monitor_work, 1840 ef4_monitor_interval); 1841 1842 efx->type->start_stats(efx); 1843 efx->type->pull_stats(efx); 1844 spin_lock_bh(&efx->stats_lock); 1845 efx->type->update_stats(efx, NULL, NULL); 1846 spin_unlock_bh(&efx->stats_lock); 1847 } 1848 1849 /* Quiesce the hardware and software data path, and regular activity 1850 * for the port without bringing the link down. Safe to call multiple 1851 * times with the NIC in almost any state, but interrupts should be 1852 * enabled. Requires the RTNL lock. 1853 */ 1854 static void ef4_stop_all(struct ef4_nic *efx) 1855 { 1856 EF4_ASSERT_RESET_SERIALISED(efx); 1857 1858 /* port_enabled can be read safely under the rtnl lock */ 1859 if (!efx->port_enabled) 1860 return; 1861 1862 /* update stats before we go down so we can accurately count 1863 * rx_nodesc_drops 1864 */ 1865 efx->type->pull_stats(efx); 1866 spin_lock_bh(&efx->stats_lock); 1867 efx->type->update_stats(efx, NULL, NULL); 1868 spin_unlock_bh(&efx->stats_lock); 1869 efx->type->stop_stats(efx); 1870 ef4_stop_port(efx); 1871 1872 /* Stop the kernel transmit interface. This is only valid if 1873 * the device is stopped or detached; otherwise the watchdog 1874 * may fire immediately. 1875 */ 1876 WARN_ON(netif_running(efx->net_dev) && 1877 netif_device_present(efx->net_dev)); 1878 netif_tx_disable(efx->net_dev); 1879 1880 ef4_stop_datapath(efx); 1881 } 1882 1883 static void ef4_remove_all(struct ef4_nic *efx) 1884 { 1885 ef4_remove_channels(efx); 1886 ef4_remove_filters(efx); 1887 ef4_remove_port(efx); 1888 ef4_remove_nic(efx); 1889 } 1890 1891 /************************************************************************** 1892 * 1893 * Interrupt moderation 1894 * 1895 **************************************************************************/ 1896 unsigned int ef4_usecs_to_ticks(struct ef4_nic *efx, unsigned int usecs) 1897 { 1898 if (usecs == 0) 1899 return 0; 1900 if (usecs * 1000 < efx->timer_quantum_ns) 1901 return 1; /* never round down to 0 */ 1902 return usecs * 1000 / efx->timer_quantum_ns; 1903 } 1904 1905 unsigned int ef4_ticks_to_usecs(struct ef4_nic *efx, unsigned int ticks) 1906 { 1907 /* We must round up when converting ticks to microseconds 1908 * because we round down when converting the other way. 1909 */ 1910 return DIV_ROUND_UP(ticks * efx->timer_quantum_ns, 1000); 1911 } 1912 1913 /* Set interrupt moderation parameters */ 1914 int ef4_init_irq_moderation(struct ef4_nic *efx, unsigned int tx_usecs, 1915 unsigned int rx_usecs, bool rx_adaptive, 1916 bool rx_may_override_tx) 1917 { 1918 struct ef4_channel *channel; 1919 unsigned int timer_max_us; 1920 1921 EF4_ASSERT_RESET_SERIALISED(efx); 1922 1923 timer_max_us = efx->timer_max_ns / 1000; 1924 1925 if (tx_usecs > timer_max_us || rx_usecs > timer_max_us) 1926 return -EINVAL; 1927 1928 if (tx_usecs != rx_usecs && efx->tx_channel_offset == 0 && 1929 !rx_may_override_tx) { 1930 netif_err(efx, drv, efx->net_dev, "Channels are shared. " 1931 "RX and TX IRQ moderation must be equal\n"); 1932 return -EINVAL; 1933 } 1934 1935 efx->irq_rx_adaptive = rx_adaptive; 1936 efx->irq_rx_moderation_us = rx_usecs; 1937 ef4_for_each_channel(channel, efx) { 1938 if (ef4_channel_has_rx_queue(channel)) 1939 channel->irq_moderation_us = rx_usecs; 1940 else if (ef4_channel_has_tx_queues(channel)) 1941 channel->irq_moderation_us = tx_usecs; 1942 } 1943 1944 return 0; 1945 } 1946 1947 void ef4_get_irq_moderation(struct ef4_nic *efx, unsigned int *tx_usecs, 1948 unsigned int *rx_usecs, bool *rx_adaptive) 1949 { 1950 *rx_adaptive = efx->irq_rx_adaptive; 1951 *rx_usecs = efx->irq_rx_moderation_us; 1952 1953 /* If channels are shared between RX and TX, so is IRQ 1954 * moderation. Otherwise, IRQ moderation is the same for all 1955 * TX channels and is not adaptive. 1956 */ 1957 if (efx->tx_channel_offset == 0) { 1958 *tx_usecs = *rx_usecs; 1959 } else { 1960 struct ef4_channel *tx_channel; 1961 1962 tx_channel = efx->channel[efx->tx_channel_offset]; 1963 *tx_usecs = tx_channel->irq_moderation_us; 1964 } 1965 } 1966 1967 /************************************************************************** 1968 * 1969 * Hardware monitor 1970 * 1971 **************************************************************************/ 1972 1973 /* Run periodically off the general workqueue */ 1974 static void ef4_monitor(struct work_struct *data) 1975 { 1976 struct ef4_nic *efx = container_of(data, struct ef4_nic, 1977 monitor_work.work); 1978 1979 netif_vdbg(efx, timer, efx->net_dev, 1980 "hardware monitor executing on CPU %d\n", 1981 raw_smp_processor_id()); 1982 BUG_ON(efx->type->monitor == NULL); 1983 1984 /* If the mac_lock is already held then it is likely a port 1985 * reconfiguration is already in place, which will likely do 1986 * most of the work of monitor() anyway. */ 1987 if (mutex_trylock(&efx->mac_lock)) { 1988 if (efx->port_enabled) 1989 efx->type->monitor(efx); 1990 mutex_unlock(&efx->mac_lock); 1991 } 1992 1993 queue_delayed_work(efx->workqueue, &efx->monitor_work, 1994 ef4_monitor_interval); 1995 } 1996 1997 /************************************************************************** 1998 * 1999 * ioctls 2000 * 2001 *************************************************************************/ 2002 2003 /* Net device ioctl 2004 * Context: process, rtnl_lock() held. 2005 */ 2006 static int ef4_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd) 2007 { 2008 struct ef4_nic *efx = netdev_priv(net_dev); 2009 struct mii_ioctl_data *data = if_mii(ifr); 2010 2011 /* Convert phy_id from older PRTAD/DEVAD format */ 2012 if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) && 2013 (data->phy_id & 0xfc00) == 0x0400) 2014 data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400; 2015 2016 return mdio_mii_ioctl(&efx->mdio, data, cmd); 2017 } 2018 2019 /************************************************************************** 2020 * 2021 * NAPI interface 2022 * 2023 **************************************************************************/ 2024 2025 static void ef4_init_napi_channel(struct ef4_channel *channel) 2026 { 2027 struct ef4_nic *efx = channel->efx; 2028 2029 channel->napi_dev = efx->net_dev; 2030 netif_napi_add(channel->napi_dev, &channel->napi_str, 2031 ef4_poll, napi_weight); 2032 ef4_channel_busy_poll_init(channel); 2033 } 2034 2035 static void ef4_init_napi(struct ef4_nic *efx) 2036 { 2037 struct ef4_channel *channel; 2038 2039 ef4_for_each_channel(channel, efx) 2040 ef4_init_napi_channel(channel); 2041 } 2042 2043 static void ef4_fini_napi_channel(struct ef4_channel *channel) 2044 { 2045 if (channel->napi_dev) 2046 netif_napi_del(&channel->napi_str); 2047 2048 channel->napi_dev = NULL; 2049 } 2050 2051 static void ef4_fini_napi(struct ef4_nic *efx) 2052 { 2053 struct ef4_channel *channel; 2054 2055 ef4_for_each_channel(channel, efx) 2056 ef4_fini_napi_channel(channel); 2057 } 2058 2059 /************************************************************************** 2060 * 2061 * Kernel netpoll interface 2062 * 2063 *************************************************************************/ 2064 2065 #ifdef CONFIG_NET_POLL_CONTROLLER 2066 2067 /* Although in the common case interrupts will be disabled, this is not 2068 * guaranteed. However, all our work happens inside the NAPI callback, 2069 * so no locking is required. 2070 */ 2071 static void ef4_netpoll(struct net_device *net_dev) 2072 { 2073 struct ef4_nic *efx = netdev_priv(net_dev); 2074 struct ef4_channel *channel; 2075 2076 ef4_for_each_channel(channel, efx) 2077 ef4_schedule_channel(channel); 2078 } 2079 2080 #endif 2081 2082 #ifdef CONFIG_NET_RX_BUSY_POLL 2083 static int ef4_busy_poll(struct napi_struct *napi) 2084 { 2085 struct ef4_channel *channel = 2086 container_of(napi, struct ef4_channel, napi_str); 2087 struct ef4_nic *efx = channel->efx; 2088 int budget = 4; 2089 int old_rx_packets, rx_packets; 2090 2091 if (!netif_running(efx->net_dev)) 2092 return LL_FLUSH_FAILED; 2093 2094 if (!ef4_channel_try_lock_poll(channel)) 2095 return LL_FLUSH_BUSY; 2096 2097 old_rx_packets = channel->rx_queue.rx_packets; 2098 ef4_process_channel(channel, budget); 2099 2100 rx_packets = channel->rx_queue.rx_packets - old_rx_packets; 2101 2102 /* There is no race condition with NAPI here. 2103 * NAPI will automatically be rescheduled if it yielded during busy 2104 * polling, because it was not able to take the lock and thus returned 2105 * the full budget. 2106 */ 2107 ef4_channel_unlock_poll(channel); 2108 2109 return rx_packets; 2110 } 2111 #endif 2112 2113 /************************************************************************** 2114 * 2115 * Kernel net device interface 2116 * 2117 *************************************************************************/ 2118 2119 /* Context: process, rtnl_lock() held. */ 2120 int ef4_net_open(struct net_device *net_dev) 2121 { 2122 struct ef4_nic *efx = netdev_priv(net_dev); 2123 int rc; 2124 2125 netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n", 2126 raw_smp_processor_id()); 2127 2128 rc = ef4_check_disabled(efx); 2129 if (rc) 2130 return rc; 2131 if (efx->phy_mode & PHY_MODE_SPECIAL) 2132 return -EBUSY; 2133 2134 /* Notify the kernel of the link state polled during driver load, 2135 * before the monitor starts running */ 2136 ef4_link_status_changed(efx); 2137 2138 ef4_start_all(efx); 2139 ef4_selftest_async_start(efx); 2140 return 0; 2141 } 2142 2143 /* Context: process, rtnl_lock() held. 2144 * Note that the kernel will ignore our return code; this method 2145 * should really be a void. 2146 */ 2147 int ef4_net_stop(struct net_device *net_dev) 2148 { 2149 struct ef4_nic *efx = netdev_priv(net_dev); 2150 2151 netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n", 2152 raw_smp_processor_id()); 2153 2154 /* Stop the device and flush all the channels */ 2155 ef4_stop_all(efx); 2156 2157 return 0; 2158 } 2159 2160 /* Context: process, dev_base_lock or RTNL held, non-blocking. */ 2161 static struct rtnl_link_stats64 *ef4_net_stats(struct net_device *net_dev, 2162 struct rtnl_link_stats64 *stats) 2163 { 2164 struct ef4_nic *efx = netdev_priv(net_dev); 2165 2166 spin_lock_bh(&efx->stats_lock); 2167 efx->type->update_stats(efx, NULL, stats); 2168 spin_unlock_bh(&efx->stats_lock); 2169 2170 return stats; 2171 } 2172 2173 /* Context: netif_tx_lock held, BHs disabled. */ 2174 static void ef4_watchdog(struct net_device *net_dev) 2175 { 2176 struct ef4_nic *efx = netdev_priv(net_dev); 2177 2178 netif_err(efx, tx_err, efx->net_dev, 2179 "TX stuck with port_enabled=%d: resetting channels\n", 2180 efx->port_enabled); 2181 2182 ef4_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG); 2183 } 2184 2185 2186 /* Context: process, rtnl_lock() held. */ 2187 static int ef4_change_mtu(struct net_device *net_dev, int new_mtu) 2188 { 2189 struct ef4_nic *efx = netdev_priv(net_dev); 2190 int rc; 2191 2192 rc = ef4_check_disabled(efx); 2193 if (rc) 2194 return rc; 2195 2196 netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu); 2197 2198 ef4_device_detach_sync(efx); 2199 ef4_stop_all(efx); 2200 2201 mutex_lock(&efx->mac_lock); 2202 net_dev->mtu = new_mtu; 2203 ef4_mac_reconfigure(efx); 2204 mutex_unlock(&efx->mac_lock); 2205 2206 ef4_start_all(efx); 2207 netif_device_attach(efx->net_dev); 2208 return 0; 2209 } 2210 2211 static int ef4_set_mac_address(struct net_device *net_dev, void *data) 2212 { 2213 struct ef4_nic *efx = netdev_priv(net_dev); 2214 struct sockaddr *addr = data; 2215 u8 *new_addr = addr->sa_data; 2216 u8 old_addr[6]; 2217 int rc; 2218 2219 if (!is_valid_ether_addr(new_addr)) { 2220 netif_err(efx, drv, efx->net_dev, 2221 "invalid ethernet MAC address requested: %pM\n", 2222 new_addr); 2223 return -EADDRNOTAVAIL; 2224 } 2225 2226 /* save old address */ 2227 ether_addr_copy(old_addr, net_dev->dev_addr); 2228 ether_addr_copy(net_dev->dev_addr, new_addr); 2229 if (efx->type->set_mac_address) { 2230 rc = efx->type->set_mac_address(efx); 2231 if (rc) { 2232 ether_addr_copy(net_dev->dev_addr, old_addr); 2233 return rc; 2234 } 2235 } 2236 2237 /* Reconfigure the MAC */ 2238 mutex_lock(&efx->mac_lock); 2239 ef4_mac_reconfigure(efx); 2240 mutex_unlock(&efx->mac_lock); 2241 2242 return 0; 2243 } 2244 2245 /* Context: netif_addr_lock held, BHs disabled. */ 2246 static void ef4_set_rx_mode(struct net_device *net_dev) 2247 { 2248 struct ef4_nic *efx = netdev_priv(net_dev); 2249 2250 if (efx->port_enabled) 2251 queue_work(efx->workqueue, &efx->mac_work); 2252 /* Otherwise ef4_start_port() will do this */ 2253 } 2254 2255 static int ef4_set_features(struct net_device *net_dev, netdev_features_t data) 2256 { 2257 struct ef4_nic *efx = netdev_priv(net_dev); 2258 int rc; 2259 2260 /* If disabling RX n-tuple filtering, clear existing filters */ 2261 if (net_dev->features & ~data & NETIF_F_NTUPLE) { 2262 rc = efx->type->filter_clear_rx(efx, EF4_FILTER_PRI_MANUAL); 2263 if (rc) 2264 return rc; 2265 } 2266 2267 /* If Rx VLAN filter is changed, update filters via mac_reconfigure */ 2268 if ((net_dev->features ^ data) & NETIF_F_HW_VLAN_CTAG_FILTER) { 2269 /* ef4_set_rx_mode() will schedule MAC work to update filters 2270 * when a new features are finally set in net_dev. 2271 */ 2272 ef4_set_rx_mode(net_dev); 2273 } 2274 2275 return 0; 2276 } 2277 2278 static const struct net_device_ops ef4_netdev_ops = { 2279 .ndo_open = ef4_net_open, 2280 .ndo_stop = ef4_net_stop, 2281 .ndo_get_stats64 = ef4_net_stats, 2282 .ndo_tx_timeout = ef4_watchdog, 2283 .ndo_start_xmit = ef4_hard_start_xmit, 2284 .ndo_validate_addr = eth_validate_addr, 2285 .ndo_do_ioctl = ef4_ioctl, 2286 .ndo_change_mtu = ef4_change_mtu, 2287 .ndo_set_mac_address = ef4_set_mac_address, 2288 .ndo_set_rx_mode = ef4_set_rx_mode, 2289 .ndo_set_features = ef4_set_features, 2290 #ifdef CONFIG_NET_POLL_CONTROLLER 2291 .ndo_poll_controller = ef4_netpoll, 2292 #endif 2293 .ndo_setup_tc = ef4_setup_tc, 2294 #ifdef CONFIG_NET_RX_BUSY_POLL 2295 .ndo_busy_poll = ef4_busy_poll, 2296 #endif 2297 #ifdef CONFIG_RFS_ACCEL 2298 .ndo_rx_flow_steer = ef4_filter_rfs, 2299 #endif 2300 }; 2301 2302 static void ef4_update_name(struct ef4_nic *efx) 2303 { 2304 strcpy(efx->name, efx->net_dev->name); 2305 ef4_mtd_rename(efx); 2306 ef4_set_channel_names(efx); 2307 } 2308 2309 static int ef4_netdev_event(struct notifier_block *this, 2310 unsigned long event, void *ptr) 2311 { 2312 struct net_device *net_dev = netdev_notifier_info_to_dev(ptr); 2313 2314 if ((net_dev->netdev_ops == &ef4_netdev_ops) && 2315 event == NETDEV_CHANGENAME) 2316 ef4_update_name(netdev_priv(net_dev)); 2317 2318 return NOTIFY_DONE; 2319 } 2320 2321 static struct notifier_block ef4_netdev_notifier = { 2322 .notifier_call = ef4_netdev_event, 2323 }; 2324 2325 static ssize_t 2326 show_phy_type(struct device *dev, struct device_attribute *attr, char *buf) 2327 { 2328 struct ef4_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2329 return sprintf(buf, "%d\n", efx->phy_type); 2330 } 2331 static DEVICE_ATTR(phy_type, 0444, show_phy_type, NULL); 2332 2333 static int ef4_register_netdev(struct ef4_nic *efx) 2334 { 2335 struct net_device *net_dev = efx->net_dev; 2336 struct ef4_channel *channel; 2337 int rc; 2338 2339 net_dev->watchdog_timeo = 5 * HZ; 2340 net_dev->irq = efx->pci_dev->irq; 2341 net_dev->netdev_ops = &ef4_netdev_ops; 2342 net_dev->ethtool_ops = &ef4_ethtool_ops; 2343 net_dev->gso_max_segs = EF4_TSO_MAX_SEGS; 2344 net_dev->min_mtu = EF4_MIN_MTU; 2345 net_dev->max_mtu = EF4_MAX_MTU; 2346 2347 rtnl_lock(); 2348 2349 /* Enable resets to be scheduled and check whether any were 2350 * already requested. If so, the NIC is probably hosed so we 2351 * abort. 2352 */ 2353 efx->state = STATE_READY; 2354 smp_mb(); /* ensure we change state before checking reset_pending */ 2355 if (efx->reset_pending) { 2356 netif_err(efx, probe, efx->net_dev, 2357 "aborting probe due to scheduled reset\n"); 2358 rc = -EIO; 2359 goto fail_locked; 2360 } 2361 2362 rc = dev_alloc_name(net_dev, net_dev->name); 2363 if (rc < 0) 2364 goto fail_locked; 2365 ef4_update_name(efx); 2366 2367 /* Always start with carrier off; PHY events will detect the link */ 2368 netif_carrier_off(net_dev); 2369 2370 rc = register_netdevice(net_dev); 2371 if (rc) 2372 goto fail_locked; 2373 2374 ef4_for_each_channel(channel, efx) { 2375 struct ef4_tx_queue *tx_queue; 2376 ef4_for_each_channel_tx_queue(tx_queue, channel) 2377 ef4_init_tx_queue_core_txq(tx_queue); 2378 } 2379 2380 ef4_associate(efx); 2381 2382 rtnl_unlock(); 2383 2384 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2385 if (rc) { 2386 netif_err(efx, drv, efx->net_dev, 2387 "failed to init net dev attributes\n"); 2388 goto fail_registered; 2389 } 2390 return 0; 2391 2392 fail_registered: 2393 rtnl_lock(); 2394 ef4_dissociate(efx); 2395 unregister_netdevice(net_dev); 2396 fail_locked: 2397 efx->state = STATE_UNINIT; 2398 rtnl_unlock(); 2399 netif_err(efx, drv, efx->net_dev, "could not register net dev\n"); 2400 return rc; 2401 } 2402 2403 static void ef4_unregister_netdev(struct ef4_nic *efx) 2404 { 2405 if (!efx->net_dev) 2406 return; 2407 2408 BUG_ON(netdev_priv(efx->net_dev) != efx); 2409 2410 if (ef4_dev_registered(efx)) { 2411 strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name)); 2412 device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2413 unregister_netdev(efx->net_dev); 2414 } 2415 } 2416 2417 /************************************************************************** 2418 * 2419 * Device reset and suspend 2420 * 2421 **************************************************************************/ 2422 2423 /* Tears down the entire software state and most of the hardware state 2424 * before reset. */ 2425 void ef4_reset_down(struct ef4_nic *efx, enum reset_type method) 2426 { 2427 EF4_ASSERT_RESET_SERIALISED(efx); 2428 2429 ef4_stop_all(efx); 2430 ef4_disable_interrupts(efx); 2431 2432 mutex_lock(&efx->mac_lock); 2433 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 2434 method != RESET_TYPE_DATAPATH) 2435 efx->phy_op->fini(efx); 2436 efx->type->fini(efx); 2437 } 2438 2439 /* This function will always ensure that the locks acquired in 2440 * ef4_reset_down() are released. A failure return code indicates 2441 * that we were unable to reinitialise the hardware, and the 2442 * driver should be disabled. If ok is false, then the rx and tx 2443 * engines are not restarted, pending a RESET_DISABLE. */ 2444 int ef4_reset_up(struct ef4_nic *efx, enum reset_type method, bool ok) 2445 { 2446 int rc; 2447 2448 EF4_ASSERT_RESET_SERIALISED(efx); 2449 2450 /* Ensure that SRAM is initialised even if we're disabling the device */ 2451 rc = efx->type->init(efx); 2452 if (rc) { 2453 netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n"); 2454 goto fail; 2455 } 2456 2457 if (!ok) 2458 goto fail; 2459 2460 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 2461 method != RESET_TYPE_DATAPATH) { 2462 rc = efx->phy_op->init(efx); 2463 if (rc) 2464 goto fail; 2465 rc = efx->phy_op->reconfigure(efx); 2466 if (rc && rc != -EPERM) 2467 netif_err(efx, drv, efx->net_dev, 2468 "could not restore PHY settings\n"); 2469 } 2470 2471 rc = ef4_enable_interrupts(efx); 2472 if (rc) 2473 goto fail; 2474 2475 down_read(&efx->filter_sem); 2476 ef4_restore_filters(efx); 2477 up_read(&efx->filter_sem); 2478 2479 mutex_unlock(&efx->mac_lock); 2480 2481 ef4_start_all(efx); 2482 2483 return 0; 2484 2485 fail: 2486 efx->port_initialized = false; 2487 2488 mutex_unlock(&efx->mac_lock); 2489 2490 return rc; 2491 } 2492 2493 /* Reset the NIC using the specified method. Note that the reset may 2494 * fail, in which case the card will be left in an unusable state. 2495 * 2496 * Caller must hold the rtnl_lock. 2497 */ 2498 int ef4_reset(struct ef4_nic *efx, enum reset_type method) 2499 { 2500 int rc, rc2; 2501 bool disabled; 2502 2503 netif_info(efx, drv, efx->net_dev, "resetting (%s)\n", 2504 RESET_TYPE(method)); 2505 2506 ef4_device_detach_sync(efx); 2507 ef4_reset_down(efx, method); 2508 2509 rc = efx->type->reset(efx, method); 2510 if (rc) { 2511 netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n"); 2512 goto out; 2513 } 2514 2515 /* Clear flags for the scopes we covered. We assume the NIC and 2516 * driver are now quiescent so that there is no race here. 2517 */ 2518 if (method < RESET_TYPE_MAX_METHOD) 2519 efx->reset_pending &= -(1 << (method + 1)); 2520 else /* it doesn't fit into the well-ordered scope hierarchy */ 2521 __clear_bit(method, &efx->reset_pending); 2522 2523 /* Reinitialise bus-mastering, which may have been turned off before 2524 * the reset was scheduled. This is still appropriate, even in the 2525 * RESET_TYPE_DISABLE since this driver generally assumes the hardware 2526 * can respond to requests. */ 2527 pci_set_master(efx->pci_dev); 2528 2529 out: 2530 /* Leave device stopped if necessary */ 2531 disabled = rc || 2532 method == RESET_TYPE_DISABLE || 2533 method == RESET_TYPE_RECOVER_OR_DISABLE; 2534 rc2 = ef4_reset_up(efx, method, !disabled); 2535 if (rc2) { 2536 disabled = true; 2537 if (!rc) 2538 rc = rc2; 2539 } 2540 2541 if (disabled) { 2542 dev_close(efx->net_dev); 2543 netif_err(efx, drv, efx->net_dev, "has been disabled\n"); 2544 efx->state = STATE_DISABLED; 2545 } else { 2546 netif_dbg(efx, drv, efx->net_dev, "reset complete\n"); 2547 netif_device_attach(efx->net_dev); 2548 } 2549 return rc; 2550 } 2551 2552 /* Try recovery mechanisms. 2553 * For now only EEH is supported. 2554 * Returns 0 if the recovery mechanisms are unsuccessful. 2555 * Returns a non-zero value otherwise. 2556 */ 2557 int ef4_try_recovery(struct ef4_nic *efx) 2558 { 2559 #ifdef CONFIG_EEH 2560 /* A PCI error can occur and not be seen by EEH because nothing 2561 * happens on the PCI bus. In this case the driver may fail and 2562 * schedule a 'recover or reset', leading to this recovery handler. 2563 * Manually call the eeh failure check function. 2564 */ 2565 struct eeh_dev *eehdev = pci_dev_to_eeh_dev(efx->pci_dev); 2566 if (eeh_dev_check_failure(eehdev)) { 2567 /* The EEH mechanisms will handle the error and reset the 2568 * device if necessary. 2569 */ 2570 return 1; 2571 } 2572 #endif 2573 return 0; 2574 } 2575 2576 /* The worker thread exists so that code that cannot sleep can 2577 * schedule a reset for later. 2578 */ 2579 static void ef4_reset_work(struct work_struct *data) 2580 { 2581 struct ef4_nic *efx = container_of(data, struct ef4_nic, reset_work); 2582 unsigned long pending; 2583 enum reset_type method; 2584 2585 pending = ACCESS_ONCE(efx->reset_pending); 2586 method = fls(pending) - 1; 2587 2588 if ((method == RESET_TYPE_RECOVER_OR_DISABLE || 2589 method == RESET_TYPE_RECOVER_OR_ALL) && 2590 ef4_try_recovery(efx)) 2591 return; 2592 2593 if (!pending) 2594 return; 2595 2596 rtnl_lock(); 2597 2598 /* We checked the state in ef4_schedule_reset() but it may 2599 * have changed by now. Now that we have the RTNL lock, 2600 * it cannot change again. 2601 */ 2602 if (efx->state == STATE_READY) 2603 (void)ef4_reset(efx, method); 2604 2605 rtnl_unlock(); 2606 } 2607 2608 void ef4_schedule_reset(struct ef4_nic *efx, enum reset_type type) 2609 { 2610 enum reset_type method; 2611 2612 if (efx->state == STATE_RECOVERY) { 2613 netif_dbg(efx, drv, efx->net_dev, 2614 "recovering: skip scheduling %s reset\n", 2615 RESET_TYPE(type)); 2616 return; 2617 } 2618 2619 switch (type) { 2620 case RESET_TYPE_INVISIBLE: 2621 case RESET_TYPE_ALL: 2622 case RESET_TYPE_RECOVER_OR_ALL: 2623 case RESET_TYPE_WORLD: 2624 case RESET_TYPE_DISABLE: 2625 case RESET_TYPE_RECOVER_OR_DISABLE: 2626 case RESET_TYPE_DATAPATH: 2627 method = type; 2628 netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n", 2629 RESET_TYPE(method)); 2630 break; 2631 default: 2632 method = efx->type->map_reset_reason(type); 2633 netif_dbg(efx, drv, efx->net_dev, 2634 "scheduling %s reset for %s\n", 2635 RESET_TYPE(method), RESET_TYPE(type)); 2636 break; 2637 } 2638 2639 set_bit(method, &efx->reset_pending); 2640 smp_mb(); /* ensure we change reset_pending before checking state */ 2641 2642 /* If we're not READY then just leave the flags set as the cue 2643 * to abort probing or reschedule the reset later. 2644 */ 2645 if (ACCESS_ONCE(efx->state) != STATE_READY) 2646 return; 2647 2648 queue_work(reset_workqueue, &efx->reset_work); 2649 } 2650 2651 /************************************************************************** 2652 * 2653 * List of NICs we support 2654 * 2655 **************************************************************************/ 2656 2657 /* PCI device ID table */ 2658 static const struct pci_device_id ef4_pci_table[] = { 2659 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 2660 PCI_DEVICE_ID_SOLARFLARE_SFC4000A_0), 2661 .driver_data = (unsigned long) &falcon_a1_nic_type}, 2662 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 2663 PCI_DEVICE_ID_SOLARFLARE_SFC4000B), 2664 .driver_data = (unsigned long) &falcon_b0_nic_type}, 2665 {0} /* end of list */ 2666 }; 2667 2668 /************************************************************************** 2669 * 2670 * Dummy PHY/MAC operations 2671 * 2672 * Can be used for some unimplemented operations 2673 * Needed so all function pointers are valid and do not have to be tested 2674 * before use 2675 * 2676 **************************************************************************/ 2677 int ef4_port_dummy_op_int(struct ef4_nic *efx) 2678 { 2679 return 0; 2680 } 2681 void ef4_port_dummy_op_void(struct ef4_nic *efx) {} 2682 2683 static bool ef4_port_dummy_op_poll(struct ef4_nic *efx) 2684 { 2685 return false; 2686 } 2687 2688 static const struct ef4_phy_operations ef4_dummy_phy_operations = { 2689 .init = ef4_port_dummy_op_int, 2690 .reconfigure = ef4_port_dummy_op_int, 2691 .poll = ef4_port_dummy_op_poll, 2692 .fini = ef4_port_dummy_op_void, 2693 }; 2694 2695 /************************************************************************** 2696 * 2697 * Data housekeeping 2698 * 2699 **************************************************************************/ 2700 2701 /* This zeroes out and then fills in the invariants in a struct 2702 * ef4_nic (including all sub-structures). 2703 */ 2704 static int ef4_init_struct(struct ef4_nic *efx, 2705 struct pci_dev *pci_dev, struct net_device *net_dev) 2706 { 2707 int i; 2708 2709 /* Initialise common structures */ 2710 INIT_LIST_HEAD(&efx->node); 2711 INIT_LIST_HEAD(&efx->secondary_list); 2712 spin_lock_init(&efx->biu_lock); 2713 #ifdef CONFIG_SFC_FALCON_MTD 2714 INIT_LIST_HEAD(&efx->mtd_list); 2715 #endif 2716 INIT_WORK(&efx->reset_work, ef4_reset_work); 2717 INIT_DELAYED_WORK(&efx->monitor_work, ef4_monitor); 2718 INIT_DELAYED_WORK(&efx->selftest_work, ef4_selftest_async_work); 2719 efx->pci_dev = pci_dev; 2720 efx->msg_enable = debug; 2721 efx->state = STATE_UNINIT; 2722 strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name)); 2723 2724 efx->net_dev = net_dev; 2725 efx->rx_prefix_size = efx->type->rx_prefix_size; 2726 efx->rx_ip_align = 2727 NET_IP_ALIGN ? (efx->rx_prefix_size + NET_IP_ALIGN) % 4 : 0; 2728 efx->rx_packet_hash_offset = 2729 efx->type->rx_hash_offset - efx->type->rx_prefix_size; 2730 efx->rx_packet_ts_offset = 2731 efx->type->rx_ts_offset - efx->type->rx_prefix_size; 2732 spin_lock_init(&efx->stats_lock); 2733 mutex_init(&efx->mac_lock); 2734 efx->phy_op = &ef4_dummy_phy_operations; 2735 efx->mdio.dev = net_dev; 2736 INIT_WORK(&efx->mac_work, ef4_mac_work); 2737 init_waitqueue_head(&efx->flush_wq); 2738 2739 for (i = 0; i < EF4_MAX_CHANNELS; i++) { 2740 efx->channel[i] = ef4_alloc_channel(efx, i, NULL); 2741 if (!efx->channel[i]) 2742 goto fail; 2743 efx->msi_context[i].efx = efx; 2744 efx->msi_context[i].index = i; 2745 } 2746 2747 /* Higher numbered interrupt modes are less capable! */ 2748 efx->interrupt_mode = max(efx->type->max_interrupt_mode, 2749 interrupt_mode); 2750 2751 /* Would be good to use the net_dev name, but we're too early */ 2752 snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s", 2753 pci_name(pci_dev)); 2754 efx->workqueue = create_singlethread_workqueue(efx->workqueue_name); 2755 if (!efx->workqueue) 2756 goto fail; 2757 2758 return 0; 2759 2760 fail: 2761 ef4_fini_struct(efx); 2762 return -ENOMEM; 2763 } 2764 2765 static void ef4_fini_struct(struct ef4_nic *efx) 2766 { 2767 int i; 2768 2769 for (i = 0; i < EF4_MAX_CHANNELS; i++) 2770 kfree(efx->channel[i]); 2771 2772 kfree(efx->vpd_sn); 2773 2774 if (efx->workqueue) { 2775 destroy_workqueue(efx->workqueue); 2776 efx->workqueue = NULL; 2777 } 2778 } 2779 2780 void ef4_update_sw_stats(struct ef4_nic *efx, u64 *stats) 2781 { 2782 u64 n_rx_nodesc_trunc = 0; 2783 struct ef4_channel *channel; 2784 2785 ef4_for_each_channel(channel, efx) 2786 n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc; 2787 stats[GENERIC_STAT_rx_nodesc_trunc] = n_rx_nodesc_trunc; 2788 stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops); 2789 } 2790 2791 /************************************************************************** 2792 * 2793 * PCI interface 2794 * 2795 **************************************************************************/ 2796 2797 /* Main body of final NIC shutdown code 2798 * This is called only at module unload (or hotplug removal). 2799 */ 2800 static void ef4_pci_remove_main(struct ef4_nic *efx) 2801 { 2802 /* Flush reset_work. It can no longer be scheduled since we 2803 * are not READY. 2804 */ 2805 BUG_ON(efx->state == STATE_READY); 2806 cancel_work_sync(&efx->reset_work); 2807 2808 ef4_disable_interrupts(efx); 2809 ef4_nic_fini_interrupt(efx); 2810 ef4_fini_port(efx); 2811 efx->type->fini(efx); 2812 ef4_fini_napi(efx); 2813 ef4_remove_all(efx); 2814 } 2815 2816 /* Final NIC shutdown 2817 * This is called only at module unload (or hotplug removal). A PF can call 2818 * this on its VFs to ensure they are unbound first. 2819 */ 2820 static void ef4_pci_remove(struct pci_dev *pci_dev) 2821 { 2822 struct ef4_nic *efx; 2823 2824 efx = pci_get_drvdata(pci_dev); 2825 if (!efx) 2826 return; 2827 2828 /* Mark the NIC as fini, then stop the interface */ 2829 rtnl_lock(); 2830 ef4_dissociate(efx); 2831 dev_close(efx->net_dev); 2832 ef4_disable_interrupts(efx); 2833 efx->state = STATE_UNINIT; 2834 rtnl_unlock(); 2835 2836 ef4_unregister_netdev(efx); 2837 2838 ef4_mtd_remove(efx); 2839 2840 ef4_pci_remove_main(efx); 2841 2842 ef4_fini_io(efx); 2843 netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n"); 2844 2845 ef4_fini_struct(efx); 2846 free_netdev(efx->net_dev); 2847 2848 pci_disable_pcie_error_reporting(pci_dev); 2849 }; 2850 2851 /* NIC VPD information 2852 * Called during probe to display the part number of the 2853 * installed NIC. VPD is potentially very large but this should 2854 * always appear within the first 512 bytes. 2855 */ 2856 #define SFC_VPD_LEN 512 2857 static void ef4_probe_vpd_strings(struct ef4_nic *efx) 2858 { 2859 struct pci_dev *dev = efx->pci_dev; 2860 char vpd_data[SFC_VPD_LEN]; 2861 ssize_t vpd_size; 2862 int ro_start, ro_size, i, j; 2863 2864 /* Get the vpd data from the device */ 2865 vpd_size = pci_read_vpd(dev, 0, sizeof(vpd_data), vpd_data); 2866 if (vpd_size <= 0) { 2867 netif_err(efx, drv, efx->net_dev, "Unable to read VPD\n"); 2868 return; 2869 } 2870 2871 /* Get the Read only section */ 2872 ro_start = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA); 2873 if (ro_start < 0) { 2874 netif_err(efx, drv, efx->net_dev, "VPD Read-only not found\n"); 2875 return; 2876 } 2877 2878 ro_size = pci_vpd_lrdt_size(&vpd_data[ro_start]); 2879 j = ro_size; 2880 i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 2881 if (i + j > vpd_size) 2882 j = vpd_size - i; 2883 2884 /* Get the Part number */ 2885 i = pci_vpd_find_info_keyword(vpd_data, i, j, "PN"); 2886 if (i < 0) { 2887 netif_err(efx, drv, efx->net_dev, "Part number not found\n"); 2888 return; 2889 } 2890 2891 j = pci_vpd_info_field_size(&vpd_data[i]); 2892 i += PCI_VPD_INFO_FLD_HDR_SIZE; 2893 if (i + j > vpd_size) { 2894 netif_err(efx, drv, efx->net_dev, "Incomplete part number\n"); 2895 return; 2896 } 2897 2898 netif_info(efx, drv, efx->net_dev, 2899 "Part Number : %.*s\n", j, &vpd_data[i]); 2900 2901 i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 2902 j = ro_size; 2903 i = pci_vpd_find_info_keyword(vpd_data, i, j, "SN"); 2904 if (i < 0) { 2905 netif_err(efx, drv, efx->net_dev, "Serial number not found\n"); 2906 return; 2907 } 2908 2909 j = pci_vpd_info_field_size(&vpd_data[i]); 2910 i += PCI_VPD_INFO_FLD_HDR_SIZE; 2911 if (i + j > vpd_size) { 2912 netif_err(efx, drv, efx->net_dev, "Incomplete serial number\n"); 2913 return; 2914 } 2915 2916 efx->vpd_sn = kmalloc(j + 1, GFP_KERNEL); 2917 if (!efx->vpd_sn) 2918 return; 2919 2920 snprintf(efx->vpd_sn, j + 1, "%s", &vpd_data[i]); 2921 } 2922 2923 2924 /* Main body of NIC initialisation 2925 * This is called at module load (or hotplug insertion, theoretically). 2926 */ 2927 static int ef4_pci_probe_main(struct ef4_nic *efx) 2928 { 2929 int rc; 2930 2931 /* Do start-of-day initialisation */ 2932 rc = ef4_probe_all(efx); 2933 if (rc) 2934 goto fail1; 2935 2936 ef4_init_napi(efx); 2937 2938 rc = efx->type->init(efx); 2939 if (rc) { 2940 netif_err(efx, probe, efx->net_dev, 2941 "failed to initialise NIC\n"); 2942 goto fail3; 2943 } 2944 2945 rc = ef4_init_port(efx); 2946 if (rc) { 2947 netif_err(efx, probe, efx->net_dev, 2948 "failed to initialise port\n"); 2949 goto fail4; 2950 } 2951 2952 rc = ef4_nic_init_interrupt(efx); 2953 if (rc) 2954 goto fail5; 2955 rc = ef4_enable_interrupts(efx); 2956 if (rc) 2957 goto fail6; 2958 2959 return 0; 2960 2961 fail6: 2962 ef4_nic_fini_interrupt(efx); 2963 fail5: 2964 ef4_fini_port(efx); 2965 fail4: 2966 efx->type->fini(efx); 2967 fail3: 2968 ef4_fini_napi(efx); 2969 ef4_remove_all(efx); 2970 fail1: 2971 return rc; 2972 } 2973 2974 /* NIC initialisation 2975 * 2976 * This is called at module load (or hotplug insertion, 2977 * theoretically). It sets up PCI mappings, resets the NIC, 2978 * sets up and registers the network devices with the kernel and hooks 2979 * the interrupt service routine. It does not prepare the device for 2980 * transmission; this is left to the first time one of the network 2981 * interfaces is brought up (i.e. ef4_net_open). 2982 */ 2983 static int ef4_pci_probe(struct pci_dev *pci_dev, 2984 const struct pci_device_id *entry) 2985 { 2986 struct net_device *net_dev; 2987 struct ef4_nic *efx; 2988 int rc; 2989 2990 /* Allocate and initialise a struct net_device and struct ef4_nic */ 2991 net_dev = alloc_etherdev_mqs(sizeof(*efx), EF4_MAX_CORE_TX_QUEUES, 2992 EF4_MAX_RX_QUEUES); 2993 if (!net_dev) 2994 return -ENOMEM; 2995 efx = netdev_priv(net_dev); 2996 efx->type = (const struct ef4_nic_type *) entry->driver_data; 2997 efx->fixed_features |= NETIF_F_HIGHDMA; 2998 2999 pci_set_drvdata(pci_dev, efx); 3000 SET_NETDEV_DEV(net_dev, &pci_dev->dev); 3001 rc = ef4_init_struct(efx, pci_dev, net_dev); 3002 if (rc) 3003 goto fail1; 3004 3005 netif_info(efx, probe, efx->net_dev, 3006 "Solarflare NIC detected\n"); 3007 3008 ef4_probe_vpd_strings(efx); 3009 3010 /* Set up basic I/O (BAR mappings etc) */ 3011 rc = ef4_init_io(efx); 3012 if (rc) 3013 goto fail2; 3014 3015 rc = ef4_pci_probe_main(efx); 3016 if (rc) 3017 goto fail3; 3018 3019 net_dev->features |= (efx->type->offload_features | NETIF_F_SG | 3020 NETIF_F_RXCSUM); 3021 /* Mask for features that also apply to VLAN devices */ 3022 net_dev->vlan_features |= (NETIF_F_HW_CSUM | NETIF_F_SG | 3023 NETIF_F_HIGHDMA | NETIF_F_RXCSUM); 3024 3025 net_dev->hw_features = net_dev->features & ~efx->fixed_features; 3026 3027 /* Disable VLAN filtering by default. It may be enforced if 3028 * the feature is fixed (i.e. VLAN filters are required to 3029 * receive VLAN tagged packets due to vPort restrictions). 3030 */ 3031 net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 3032 net_dev->features |= efx->fixed_features; 3033 3034 rc = ef4_register_netdev(efx); 3035 if (rc) 3036 goto fail4; 3037 3038 netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n"); 3039 3040 /* Try to create MTDs, but allow this to fail */ 3041 rtnl_lock(); 3042 rc = ef4_mtd_probe(efx); 3043 rtnl_unlock(); 3044 if (rc && rc != -EPERM) 3045 netif_warn(efx, probe, efx->net_dev, 3046 "failed to create MTDs (%d)\n", rc); 3047 3048 rc = pci_enable_pcie_error_reporting(pci_dev); 3049 if (rc && rc != -EINVAL) 3050 netif_notice(efx, probe, efx->net_dev, 3051 "PCIE error reporting unavailable (%d).\n", 3052 rc); 3053 3054 return 0; 3055 3056 fail4: 3057 ef4_pci_remove_main(efx); 3058 fail3: 3059 ef4_fini_io(efx); 3060 fail2: 3061 ef4_fini_struct(efx); 3062 fail1: 3063 WARN_ON(rc > 0); 3064 netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc); 3065 free_netdev(net_dev); 3066 return rc; 3067 } 3068 3069 static int ef4_pm_freeze(struct device *dev) 3070 { 3071 struct ef4_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 3072 3073 rtnl_lock(); 3074 3075 if (efx->state != STATE_DISABLED) { 3076 efx->state = STATE_UNINIT; 3077 3078 ef4_device_detach_sync(efx); 3079 3080 ef4_stop_all(efx); 3081 ef4_disable_interrupts(efx); 3082 } 3083 3084 rtnl_unlock(); 3085 3086 return 0; 3087 } 3088 3089 static int ef4_pm_thaw(struct device *dev) 3090 { 3091 int rc; 3092 struct ef4_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 3093 3094 rtnl_lock(); 3095 3096 if (efx->state != STATE_DISABLED) { 3097 rc = ef4_enable_interrupts(efx); 3098 if (rc) 3099 goto fail; 3100 3101 mutex_lock(&efx->mac_lock); 3102 efx->phy_op->reconfigure(efx); 3103 mutex_unlock(&efx->mac_lock); 3104 3105 ef4_start_all(efx); 3106 3107 netif_device_attach(efx->net_dev); 3108 3109 efx->state = STATE_READY; 3110 3111 efx->type->resume_wol(efx); 3112 } 3113 3114 rtnl_unlock(); 3115 3116 /* Reschedule any quenched resets scheduled during ef4_pm_freeze() */ 3117 queue_work(reset_workqueue, &efx->reset_work); 3118 3119 return 0; 3120 3121 fail: 3122 rtnl_unlock(); 3123 3124 return rc; 3125 } 3126 3127 static int ef4_pm_poweroff(struct device *dev) 3128 { 3129 struct pci_dev *pci_dev = to_pci_dev(dev); 3130 struct ef4_nic *efx = pci_get_drvdata(pci_dev); 3131 3132 efx->type->fini(efx); 3133 3134 efx->reset_pending = 0; 3135 3136 pci_save_state(pci_dev); 3137 return pci_set_power_state(pci_dev, PCI_D3hot); 3138 } 3139 3140 /* Used for both resume and restore */ 3141 static int ef4_pm_resume(struct device *dev) 3142 { 3143 struct pci_dev *pci_dev = to_pci_dev(dev); 3144 struct ef4_nic *efx = pci_get_drvdata(pci_dev); 3145 int rc; 3146 3147 rc = pci_set_power_state(pci_dev, PCI_D0); 3148 if (rc) 3149 return rc; 3150 pci_restore_state(pci_dev); 3151 rc = pci_enable_device(pci_dev); 3152 if (rc) 3153 return rc; 3154 pci_set_master(efx->pci_dev); 3155 rc = efx->type->reset(efx, RESET_TYPE_ALL); 3156 if (rc) 3157 return rc; 3158 rc = efx->type->init(efx); 3159 if (rc) 3160 return rc; 3161 rc = ef4_pm_thaw(dev); 3162 return rc; 3163 } 3164 3165 static int ef4_pm_suspend(struct device *dev) 3166 { 3167 int rc; 3168 3169 ef4_pm_freeze(dev); 3170 rc = ef4_pm_poweroff(dev); 3171 if (rc) 3172 ef4_pm_resume(dev); 3173 return rc; 3174 } 3175 3176 static const struct dev_pm_ops ef4_pm_ops = { 3177 .suspend = ef4_pm_suspend, 3178 .resume = ef4_pm_resume, 3179 .freeze = ef4_pm_freeze, 3180 .thaw = ef4_pm_thaw, 3181 .poweroff = ef4_pm_poweroff, 3182 .restore = ef4_pm_resume, 3183 }; 3184 3185 /* A PCI error affecting this device was detected. 3186 * At this point MMIO and DMA may be disabled. 3187 * Stop the software path and request a slot reset. 3188 */ 3189 static pci_ers_result_t ef4_io_error_detected(struct pci_dev *pdev, 3190 enum pci_channel_state state) 3191 { 3192 pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3193 struct ef4_nic *efx = pci_get_drvdata(pdev); 3194 3195 if (state == pci_channel_io_perm_failure) 3196 return PCI_ERS_RESULT_DISCONNECT; 3197 3198 rtnl_lock(); 3199 3200 if (efx->state != STATE_DISABLED) { 3201 efx->state = STATE_RECOVERY; 3202 efx->reset_pending = 0; 3203 3204 ef4_device_detach_sync(efx); 3205 3206 ef4_stop_all(efx); 3207 ef4_disable_interrupts(efx); 3208 3209 status = PCI_ERS_RESULT_NEED_RESET; 3210 } else { 3211 /* If the interface is disabled we don't want to do anything 3212 * with it. 3213 */ 3214 status = PCI_ERS_RESULT_RECOVERED; 3215 } 3216 3217 rtnl_unlock(); 3218 3219 pci_disable_device(pdev); 3220 3221 return status; 3222 } 3223 3224 /* Fake a successful reset, which will be performed later in ef4_io_resume. */ 3225 static pci_ers_result_t ef4_io_slot_reset(struct pci_dev *pdev) 3226 { 3227 struct ef4_nic *efx = pci_get_drvdata(pdev); 3228 pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3229 int rc; 3230 3231 if (pci_enable_device(pdev)) { 3232 netif_err(efx, hw, efx->net_dev, 3233 "Cannot re-enable PCI device after reset.\n"); 3234 status = PCI_ERS_RESULT_DISCONNECT; 3235 } 3236 3237 rc = pci_cleanup_aer_uncorrect_error_status(pdev); 3238 if (rc) { 3239 netif_err(efx, hw, efx->net_dev, 3240 "pci_cleanup_aer_uncorrect_error_status failed (%d)\n", rc); 3241 /* Non-fatal error. Continue. */ 3242 } 3243 3244 return status; 3245 } 3246 3247 /* Perform the actual reset and resume I/O operations. */ 3248 static void ef4_io_resume(struct pci_dev *pdev) 3249 { 3250 struct ef4_nic *efx = pci_get_drvdata(pdev); 3251 int rc; 3252 3253 rtnl_lock(); 3254 3255 if (efx->state == STATE_DISABLED) 3256 goto out; 3257 3258 rc = ef4_reset(efx, RESET_TYPE_ALL); 3259 if (rc) { 3260 netif_err(efx, hw, efx->net_dev, 3261 "ef4_reset failed after PCI error (%d)\n", rc); 3262 } else { 3263 efx->state = STATE_READY; 3264 netif_dbg(efx, hw, efx->net_dev, 3265 "Done resetting and resuming IO after PCI error.\n"); 3266 } 3267 3268 out: 3269 rtnl_unlock(); 3270 } 3271 3272 /* For simplicity and reliability, we always require a slot reset and try to 3273 * reset the hardware when a pci error affecting the device is detected. 3274 * We leave both the link_reset and mmio_enabled callback unimplemented: 3275 * with our request for slot reset the mmio_enabled callback will never be 3276 * called, and the link_reset callback is not used by AER or EEH mechanisms. 3277 */ 3278 static const struct pci_error_handlers ef4_err_handlers = { 3279 .error_detected = ef4_io_error_detected, 3280 .slot_reset = ef4_io_slot_reset, 3281 .resume = ef4_io_resume, 3282 }; 3283 3284 static struct pci_driver ef4_pci_driver = { 3285 .name = KBUILD_MODNAME, 3286 .id_table = ef4_pci_table, 3287 .probe = ef4_pci_probe, 3288 .remove = ef4_pci_remove, 3289 .driver.pm = &ef4_pm_ops, 3290 .err_handler = &ef4_err_handlers, 3291 }; 3292 3293 /************************************************************************** 3294 * 3295 * Kernel module interface 3296 * 3297 *************************************************************************/ 3298 3299 module_param(interrupt_mode, uint, 0444); 3300 MODULE_PARM_DESC(interrupt_mode, 3301 "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)"); 3302 3303 static int __init ef4_init_module(void) 3304 { 3305 int rc; 3306 3307 printk(KERN_INFO "Solarflare Falcon driver v" EF4_DRIVER_VERSION "\n"); 3308 3309 rc = register_netdevice_notifier(&ef4_netdev_notifier); 3310 if (rc) 3311 goto err_notifier; 3312 3313 reset_workqueue = create_singlethread_workqueue("sfc_reset"); 3314 if (!reset_workqueue) { 3315 rc = -ENOMEM; 3316 goto err_reset; 3317 } 3318 3319 rc = pci_register_driver(&ef4_pci_driver); 3320 if (rc < 0) 3321 goto err_pci; 3322 3323 return 0; 3324 3325 err_pci: 3326 destroy_workqueue(reset_workqueue); 3327 err_reset: 3328 unregister_netdevice_notifier(&ef4_netdev_notifier); 3329 err_notifier: 3330 return rc; 3331 } 3332 3333 static void __exit ef4_exit_module(void) 3334 { 3335 printk(KERN_INFO "Solarflare Falcon driver unloading\n"); 3336 3337 pci_unregister_driver(&ef4_pci_driver); 3338 destroy_workqueue(reset_workqueue); 3339 unregister_netdevice_notifier(&ef4_netdev_notifier); 3340 3341 } 3342 3343 module_init(ef4_init_module); 3344 module_exit(ef4_exit_module); 3345 3346 MODULE_AUTHOR("Solarflare Communications and " 3347 "Michael Brown <mbrown@fensystems.co.uk>"); 3348 MODULE_DESCRIPTION("Solarflare Falcon network driver"); 3349 MODULE_LICENSE("GPL"); 3350 MODULE_DEVICE_TABLE(pci, ef4_pci_table); 3351