1 // SPDX-License-Identifier: GPL-2.0-only 2 /**************************************************************************** 3 * Driver for Solarflare network controllers and boards 4 * Copyright 2005-2006 Fen Systems Ltd. 5 * Copyright 2005-2013 Solarflare Communications Inc. 6 */ 7 8 #include <linux/filter.h> 9 #include <linux/module.h> 10 #include <linux/pci.h> 11 #include <linux/netdevice.h> 12 #include <linux/etherdevice.h> 13 #include <linux/delay.h> 14 #include <linux/notifier.h> 15 #include <linux/ip.h> 16 #include <linux/tcp.h> 17 #include <linux/in.h> 18 #include <linux/ethtool.h> 19 #include <linux/topology.h> 20 #include <linux/gfp.h> 21 #include <linux/interrupt.h> 22 #include "net_driver.h" 23 #include <net/gre.h> 24 #include <net/udp_tunnel.h> 25 #include <net/netdev_queues.h> 26 #include "efx.h" 27 #include "efx_common.h" 28 #include "efx_channels.h" 29 #include "ef100.h" 30 #include "rx_common.h" 31 #include "tx_common.h" 32 #include "nic.h" 33 #include "io.h" 34 #include "selftest.h" 35 #include "sriov.h" 36 #include "efx_devlink.h" 37 #include "efx_cxl.h" 38 39 #include "mcdi_port_common.h" 40 #include "mcdi_pcol.h" 41 #include "workarounds.h" 42 43 /************************************************************************** 44 * 45 * Configurable values 46 * 47 *************************************************************************/ 48 49 module_param_named(interrupt_mode, efx_interrupt_mode, uint, 0444); 50 MODULE_PARM_DESC(interrupt_mode, 51 "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)"); 52 53 module_param(rss_cpus, uint, 0444); 54 MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling"); 55 56 /* 57 * Use separate channels for TX and RX events 58 * 59 * Set this to 1 to use separate channels for TX and RX. It allows us 60 * to control interrupt affinity separately for TX and RX. 61 * 62 * This is only used in MSI-X interrupt mode 63 */ 64 bool efx_separate_tx_channels; 65 module_param(efx_separate_tx_channels, bool, 0444); 66 MODULE_PARM_DESC(efx_separate_tx_channels, 67 "Use separate channels for TX and RX"); 68 69 /* Initial interrupt moderation settings. They can be modified after 70 * module load with ethtool. 71 * 72 * The default for RX should strike a balance between increasing the 73 * round-trip latency and reducing overhead. 74 */ 75 static unsigned int rx_irq_mod_usec = 60; 76 77 /* Initial interrupt moderation settings. They can be modified after 78 * module load with ethtool. 79 * 80 * This default is chosen to ensure that a 10G link does not go idle 81 * while a TX queue is stopped after it has become full. A queue is 82 * restarted when it drops below half full. The time this takes (assuming 83 * worst case 3 descriptors per packet and 1024 descriptors) is 84 * 512 / 3 * 1.2 = 205 usec. 85 */ 86 static unsigned int tx_irq_mod_usec = 150; 87 88 static bool phy_flash_cfg; 89 module_param(phy_flash_cfg, bool, 0644); 90 MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially"); 91 92 static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 93 NETIF_MSG_LINK | NETIF_MSG_IFDOWN | 94 NETIF_MSG_IFUP | NETIF_MSG_RX_ERR | 95 NETIF_MSG_TX_ERR | NETIF_MSG_HW); 96 module_param(debug, uint, 0); 97 MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value"); 98 99 /************************************************************************** 100 * 101 * Utility functions and prototypes 102 * 103 *************************************************************************/ 104 105 static void efx_remove_port(struct efx_nic *efx); 106 static int efx_xdp_setup_prog(struct efx_nic *efx, struct bpf_prog *prog); 107 static int efx_xdp(struct net_device *dev, struct netdev_bpf *xdp); 108 static int efx_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **xdpfs, 109 u32 flags); 110 111 /************************************************************************** 112 * 113 * Port handling 114 * 115 **************************************************************************/ 116 117 static void efx_fini_port(struct efx_nic *efx); 118 119 static int efx_probe_port(struct efx_nic *efx) 120 { 121 int rc; 122 123 netif_dbg(efx, probe, efx->net_dev, "create port\n"); 124 125 if (phy_flash_cfg) 126 efx->phy_mode = PHY_MODE_SPECIAL; 127 128 /* Connect up MAC/PHY operations table */ 129 rc = efx->type->probe_port(efx); 130 if (rc) 131 return rc; 132 133 /* Initialise MAC address to permanent address */ 134 eth_hw_addr_set(efx->net_dev, efx->net_dev->perm_addr); 135 136 return 0; 137 } 138 139 static int efx_init_port(struct efx_nic *efx) 140 { 141 int rc; 142 143 netif_dbg(efx, drv, efx->net_dev, "init port\n"); 144 145 mutex_lock(&efx->mac_lock); 146 147 efx->port_initialized = true; 148 149 /* Ensure the PHY advertises the correct flow control settings */ 150 rc = efx_mcdi_port_reconfigure(efx); 151 if (rc && rc != -EPERM) 152 goto fail; 153 154 mutex_unlock(&efx->mac_lock); 155 return 0; 156 157 fail: 158 mutex_unlock(&efx->mac_lock); 159 return rc; 160 } 161 162 static void efx_fini_port(struct efx_nic *efx) 163 { 164 netif_dbg(efx, drv, efx->net_dev, "shut down port\n"); 165 166 if (!efx->port_initialized) 167 return; 168 169 efx->port_initialized = false; 170 171 efx->link_state.up = false; 172 efx_link_status_changed(efx); 173 } 174 175 static void efx_remove_port(struct efx_nic *efx) 176 { 177 netif_dbg(efx, drv, efx->net_dev, "destroying port\n"); 178 179 efx->type->remove_port(efx); 180 } 181 182 /************************************************************************** 183 * 184 * NIC handling 185 * 186 **************************************************************************/ 187 188 static LIST_HEAD(efx_primary_list); 189 static LIST_HEAD(efx_unassociated_list); 190 191 static bool efx_same_controller(struct efx_nic *left, struct efx_nic *right) 192 { 193 return left->type == right->type && 194 left->vpd_sn && right->vpd_sn && 195 !strcmp(left->vpd_sn, right->vpd_sn); 196 } 197 198 static void efx_associate(struct efx_nic *efx) 199 { 200 struct efx_nic *other, *next; 201 202 if (efx->primary == efx) { 203 /* Adding primary function; look for secondaries */ 204 205 netif_dbg(efx, probe, efx->net_dev, "adding to primary list\n"); 206 list_add_tail(&efx->node, &efx_primary_list); 207 208 list_for_each_entry_safe(other, next, &efx_unassociated_list, 209 node) { 210 if (efx_same_controller(efx, other)) { 211 list_del(&other->node); 212 netif_dbg(other, probe, other->net_dev, 213 "moving to secondary list of %s %s\n", 214 pci_name(efx->pci_dev), 215 efx->net_dev->name); 216 list_add_tail(&other->node, 217 &efx->secondary_list); 218 other->primary = efx; 219 } 220 } 221 } else { 222 /* Adding secondary function; look for primary */ 223 224 list_for_each_entry(other, &efx_primary_list, node) { 225 if (efx_same_controller(efx, other)) { 226 netif_dbg(efx, probe, efx->net_dev, 227 "adding to secondary list of %s %s\n", 228 pci_name(other->pci_dev), 229 other->net_dev->name); 230 list_add_tail(&efx->node, 231 &other->secondary_list); 232 efx->primary = other; 233 return; 234 } 235 } 236 237 netif_dbg(efx, probe, efx->net_dev, 238 "adding to unassociated list\n"); 239 list_add_tail(&efx->node, &efx_unassociated_list); 240 } 241 } 242 243 static void efx_dissociate(struct efx_nic *efx) 244 { 245 struct efx_nic *other, *next; 246 247 list_del(&efx->node); 248 efx->primary = NULL; 249 250 list_for_each_entry_safe(other, next, &efx->secondary_list, node) { 251 list_del(&other->node); 252 netif_dbg(other, probe, other->net_dev, 253 "moving to unassociated list\n"); 254 list_add_tail(&other->node, &efx_unassociated_list); 255 other->primary = NULL; 256 } 257 } 258 259 static int efx_probe_nic(struct efx_nic *efx) 260 { 261 int rc; 262 263 netif_dbg(efx, probe, efx->net_dev, "creating NIC\n"); 264 265 /* Carry out hardware-type specific initialisation */ 266 rc = efx->type->probe(efx); 267 if (rc) 268 return rc; 269 270 do { 271 if (!efx->max_channels || !efx->max_tx_channels) { 272 netif_err(efx, drv, efx->net_dev, 273 "Insufficient resources to allocate" 274 " any channels\n"); 275 rc = -ENOSPC; 276 goto fail1; 277 } 278 279 /* Determine the number of channels and queues by trying 280 * to hook in MSI-X interrupts. 281 */ 282 rc = efx_probe_interrupts(efx); 283 if (rc) 284 goto fail1; 285 286 rc = efx_set_channels(efx); 287 if (rc) 288 goto fail1; 289 290 /* dimension_resources can fail with EAGAIN */ 291 rc = efx->type->dimension_resources(efx); 292 if (rc != 0 && rc != -EAGAIN) 293 goto fail2; 294 295 if (rc == -EAGAIN) 296 /* try again with new max_channels */ 297 efx_remove_interrupts(efx); 298 299 } while (rc == -EAGAIN); 300 301 if (efx->n_channels > 1) 302 netdev_rss_key_fill(efx->rss_context.rx_hash_key, 303 sizeof(efx->rss_context.rx_hash_key)); 304 efx_set_default_rx_indir_table(efx, efx->rss_context.rx_indir_table); 305 306 /* Initialise the interrupt moderation settings */ 307 efx->irq_mod_step_us = DIV_ROUND_UP(efx->timer_quantum_ns, 1000); 308 efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true, 309 true); 310 311 return 0; 312 313 fail2: 314 efx_remove_interrupts(efx); 315 fail1: 316 efx->type->remove(efx); 317 return rc; 318 } 319 320 static void efx_remove_nic(struct efx_nic *efx) 321 { 322 netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n"); 323 324 efx_remove_interrupts(efx); 325 efx->type->remove(efx); 326 } 327 328 /************************************************************************** 329 * 330 * NIC startup/shutdown 331 * 332 *************************************************************************/ 333 334 static int efx_probe_all(struct efx_nic *efx) 335 { 336 int rc; 337 338 rc = efx_probe_nic(efx); 339 if (rc) { 340 netif_err(efx, probe, efx->net_dev, "failed to create NIC\n"); 341 goto fail1; 342 } 343 344 rc = efx_probe_port(efx); 345 if (rc) { 346 netif_err(efx, probe, efx->net_dev, "failed to create port\n"); 347 goto fail2; 348 } 349 350 BUILD_BUG_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_RXQ_MIN_ENT); 351 if (WARN_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_TXQ_MIN_ENT(efx))) { 352 rc = -EINVAL; 353 goto fail3; 354 } 355 356 #ifdef CONFIG_SFC_SRIOV 357 rc = efx->type->vswitching_probe(efx); 358 if (rc) /* not fatal; the PF will still work fine */ 359 netif_warn(efx, probe, efx->net_dev, 360 "failed to setup vswitching rc=%d;" 361 " VFs may not function\n", rc); 362 #endif 363 364 rc = efx_probe_filters(efx); 365 if (rc) { 366 netif_err(efx, probe, efx->net_dev, 367 "failed to create filter tables\n"); 368 goto fail4; 369 } 370 371 rc = efx_probe_channels(efx); 372 if (rc) 373 goto fail5; 374 375 efx->state = STATE_NET_DOWN; 376 377 return 0; 378 379 fail5: 380 efx_remove_filters(efx); 381 fail4: 382 #ifdef CONFIG_SFC_SRIOV 383 efx->type->vswitching_remove(efx); 384 #endif 385 fail3: 386 efx_remove_port(efx); 387 fail2: 388 efx_remove_nic(efx); 389 fail1: 390 return rc; 391 } 392 393 static void efx_remove_all(struct efx_nic *efx) 394 { 395 rtnl_lock(); 396 efx_xdp_setup_prog(efx, NULL); 397 rtnl_unlock(); 398 399 efx_remove_channels(efx); 400 efx_remove_filters(efx); 401 #ifdef CONFIG_SFC_SRIOV 402 efx->type->vswitching_remove(efx); 403 #endif 404 efx_remove_port(efx); 405 efx_remove_nic(efx); 406 } 407 408 /************************************************************************** 409 * 410 * Interrupt moderation 411 * 412 **************************************************************************/ 413 unsigned int efx_usecs_to_ticks(struct efx_nic *efx, unsigned int usecs) 414 { 415 if (usecs == 0) 416 return 0; 417 if (usecs * 1000 < efx->timer_quantum_ns) 418 return 1; /* never round down to 0 */ 419 return usecs * 1000 / efx->timer_quantum_ns; 420 } 421 422 /* Set interrupt moderation parameters */ 423 int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs, 424 unsigned int rx_usecs, bool rx_adaptive, 425 bool rx_may_override_tx) 426 { 427 struct efx_channel *channel; 428 unsigned int timer_max_us; 429 430 EFX_ASSERT_RESET_SERIALISED(efx); 431 432 timer_max_us = efx->timer_max_ns / 1000; 433 434 if (tx_usecs > timer_max_us || rx_usecs > timer_max_us) 435 return -EINVAL; 436 437 if (tx_usecs != rx_usecs && efx->tx_channel_offset == 0 && 438 !rx_may_override_tx) { 439 netif_err(efx, drv, efx->net_dev, "Channels are shared. " 440 "RX and TX IRQ moderation must be equal\n"); 441 return -EINVAL; 442 } 443 444 efx->irq_rx_adaptive = rx_adaptive; 445 efx->irq_rx_moderation_us = rx_usecs; 446 efx_for_each_channel(channel, efx) { 447 if (efx_channel_has_rx_queue(channel)) 448 channel->irq_moderation_us = rx_usecs; 449 else if (efx_channel_has_tx_queues(channel)) 450 channel->irq_moderation_us = tx_usecs; 451 else if (efx_channel_is_xdp_tx(channel)) 452 channel->irq_moderation_us = tx_usecs; 453 } 454 455 return 0; 456 } 457 458 void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs, 459 unsigned int *rx_usecs, bool *rx_adaptive) 460 { 461 *rx_adaptive = efx->irq_rx_adaptive; 462 *rx_usecs = efx->irq_rx_moderation_us; 463 464 /* If channels are shared between RX and TX, so is IRQ 465 * moderation. Otherwise, IRQ moderation is the same for all 466 * TX channels and is not adaptive. 467 */ 468 if (efx->tx_channel_offset == 0) { 469 *tx_usecs = *rx_usecs; 470 } else { 471 struct efx_channel *tx_channel; 472 473 tx_channel = efx->channel[efx->tx_channel_offset]; 474 *tx_usecs = tx_channel->irq_moderation_us; 475 } 476 } 477 478 /************************************************************************** 479 * 480 * Kernel net device interface 481 * 482 *************************************************************************/ 483 484 /* Context: process, rtnl_lock() held. */ 485 int efx_net_open(struct net_device *net_dev) 486 { 487 struct efx_nic *efx = efx_netdev_priv(net_dev); 488 int rc; 489 490 netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n", 491 raw_smp_processor_id()); 492 493 rc = efx_check_disabled(efx); 494 if (rc) 495 return rc; 496 if (efx->phy_mode & PHY_MODE_SPECIAL) 497 return -EBUSY; 498 if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL)) 499 return -EIO; 500 501 /* Notify the kernel of the link state polled during driver load, 502 * before the monitor starts running */ 503 efx_link_status_changed(efx); 504 505 efx_start_all(efx); 506 if (efx->state == STATE_DISABLED || efx->reset_pending) 507 netif_device_detach(efx->net_dev); 508 else 509 efx->state = STATE_NET_UP; 510 511 return 0; 512 } 513 514 /* Context: process, rtnl_lock() held. 515 * Note that the kernel will ignore our return code; this method 516 * should really be a void. 517 */ 518 int efx_net_stop(struct net_device *net_dev) 519 { 520 struct efx_nic *efx = efx_netdev_priv(net_dev); 521 522 netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n", 523 raw_smp_processor_id()); 524 525 /* Stop the device and flush all the channels */ 526 efx_stop_all(efx); 527 528 return 0; 529 } 530 531 static int efx_vlan_rx_add_vid(struct net_device *net_dev, __be16 proto, u16 vid) 532 { 533 struct efx_nic *efx = efx_netdev_priv(net_dev); 534 535 if (efx->type->vlan_rx_add_vid) 536 return efx->type->vlan_rx_add_vid(efx, proto, vid); 537 else 538 return -EOPNOTSUPP; 539 } 540 541 static int efx_vlan_rx_kill_vid(struct net_device *net_dev, __be16 proto, u16 vid) 542 { 543 struct efx_nic *efx = efx_netdev_priv(net_dev); 544 545 if (efx->type->vlan_rx_kill_vid) 546 return efx->type->vlan_rx_kill_vid(efx, proto, vid); 547 else 548 return -EOPNOTSUPP; 549 } 550 551 static int efx_hwtstamp_set(struct net_device *net_dev, 552 struct kernel_hwtstamp_config *config, 553 struct netlink_ext_ack *extack) 554 { 555 struct efx_nic *efx = efx_netdev_priv(net_dev); 556 557 return efx_ptp_set_ts_config(efx, config, extack); 558 } 559 560 static int efx_hwtstamp_get(struct net_device *net_dev, 561 struct kernel_hwtstamp_config *config) 562 { 563 struct efx_nic *efx = efx_netdev_priv(net_dev); 564 565 return efx_ptp_get_ts_config(efx, config); 566 } 567 568 static const struct net_device_ops efx_netdev_ops = { 569 .ndo_open = efx_net_open, 570 .ndo_stop = efx_net_stop, 571 .ndo_get_stats64 = efx_net_stats, 572 .ndo_tx_timeout = efx_watchdog, 573 .ndo_start_xmit = efx_hard_start_xmit, 574 .ndo_validate_addr = eth_validate_addr, 575 .ndo_change_mtu = efx_change_mtu, 576 .ndo_set_mac_address = efx_set_mac_address, 577 .ndo_set_rx_mode = efx_set_rx_mode, 578 .ndo_set_features = efx_set_features, 579 .ndo_features_check = efx_features_check, 580 .ndo_vlan_rx_add_vid = efx_vlan_rx_add_vid, 581 .ndo_vlan_rx_kill_vid = efx_vlan_rx_kill_vid, 582 .ndo_hwtstamp_set = efx_hwtstamp_set, 583 .ndo_hwtstamp_get = efx_hwtstamp_get, 584 #ifdef CONFIG_SFC_SRIOV 585 .ndo_set_vf_mac = efx_sriov_set_vf_mac, 586 .ndo_set_vf_vlan = efx_sriov_set_vf_vlan, 587 .ndo_set_vf_spoofchk = efx_sriov_set_vf_spoofchk, 588 .ndo_get_vf_config = efx_sriov_get_vf_config, 589 .ndo_set_vf_link_state = efx_sriov_set_vf_link_state, 590 #endif 591 .ndo_get_phys_port_id = efx_get_phys_port_id, 592 .ndo_get_phys_port_name = efx_get_phys_port_name, 593 #ifdef CONFIG_RFS_ACCEL 594 .ndo_rx_flow_steer = efx_filter_rfs, 595 #endif 596 .ndo_xdp_xmit = efx_xdp_xmit, 597 .ndo_bpf = efx_xdp 598 }; 599 600 static void efx_get_queue_stats_rx(struct net_device *net_dev, int idx, 601 struct netdev_queue_stats_rx *stats) 602 { 603 struct efx_nic *efx = efx_netdev_priv(net_dev); 604 struct efx_rx_queue *rx_queue; 605 struct efx_channel *channel; 606 607 channel = efx_get_channel(efx, idx); 608 rx_queue = efx_channel_get_rx_queue(channel); 609 /* Count only packets since last time datapath was started */ 610 stats->packets = rx_queue->rx_packets - rx_queue->old_rx_packets; 611 stats->bytes = rx_queue->rx_bytes - rx_queue->old_rx_bytes; 612 stats->hw_drops = efx_get_queue_stat_rx_hw_drops(channel) - 613 channel->old_n_rx_hw_drops; 614 stats->hw_drop_overruns = channel->n_rx_nodesc_trunc - 615 channel->old_n_rx_hw_drop_overruns; 616 } 617 618 static void efx_get_queue_stats_tx(struct net_device *net_dev, int idx, 619 struct netdev_queue_stats_tx *stats) 620 { 621 struct efx_nic *efx = efx_netdev_priv(net_dev); 622 struct efx_tx_queue *tx_queue; 623 struct efx_channel *channel; 624 625 channel = efx_get_tx_channel(efx, idx); 626 stats->packets = 0; 627 stats->bytes = 0; 628 stats->hw_gso_packets = 0; 629 stats->hw_gso_wire_packets = 0; 630 efx_for_each_channel_tx_queue(tx_queue, channel) { 631 stats->packets += tx_queue->complete_packets - 632 tx_queue->old_complete_packets; 633 stats->bytes += tx_queue->complete_bytes - 634 tx_queue->old_complete_bytes; 635 /* Note that, unlike stats->packets and stats->bytes, 636 * these count TXes enqueued, rather than completed, 637 * which may not be what users expect. 638 */ 639 stats->hw_gso_packets += tx_queue->tso_bursts - 640 tx_queue->old_tso_bursts; 641 stats->hw_gso_wire_packets += tx_queue->tso_packets - 642 tx_queue->old_tso_packets; 643 } 644 } 645 646 static void efx_get_base_stats(struct net_device *net_dev, 647 struct netdev_queue_stats_rx *rx, 648 struct netdev_queue_stats_tx *tx) 649 { 650 struct efx_nic *efx = efx_netdev_priv(net_dev); 651 struct efx_tx_queue *tx_queue; 652 struct efx_rx_queue *rx_queue; 653 struct efx_channel *channel; 654 655 rx->packets = 0; 656 rx->bytes = 0; 657 rx->hw_drops = 0; 658 rx->hw_drop_overruns = 0; 659 tx->packets = 0; 660 tx->bytes = 0; 661 tx->hw_gso_packets = 0; 662 tx->hw_gso_wire_packets = 0; 663 664 /* Count all packets on non-core queues, and packets before last 665 * datapath start on core queues. 666 */ 667 efx_for_each_channel(channel, efx) { 668 rx_queue = efx_channel_get_rx_queue(channel); 669 if (channel->channel >= net_dev->real_num_rx_queues) { 670 rx->packets += rx_queue->rx_packets; 671 rx->bytes += rx_queue->rx_bytes; 672 rx->hw_drops += efx_get_queue_stat_rx_hw_drops(channel); 673 rx->hw_drop_overruns += channel->n_rx_nodesc_trunc; 674 } else { 675 rx->packets += rx_queue->old_rx_packets; 676 rx->bytes += rx_queue->old_rx_bytes; 677 rx->hw_drops += channel->old_n_rx_hw_drops; 678 rx->hw_drop_overruns += channel->old_n_rx_hw_drop_overruns; 679 } 680 efx_for_each_channel_tx_queue(tx_queue, channel) { 681 if (channel->channel < efx->tx_channel_offset || 682 channel->channel >= efx->tx_channel_offset + 683 net_dev->real_num_tx_queues) { 684 tx->packets += tx_queue->complete_packets; 685 tx->bytes += tx_queue->complete_bytes; 686 tx->hw_gso_packets += tx_queue->tso_bursts; 687 tx->hw_gso_wire_packets += tx_queue->tso_packets; 688 } else { 689 tx->packets += tx_queue->old_complete_packets; 690 tx->bytes += tx_queue->old_complete_bytes; 691 tx->hw_gso_packets += tx_queue->old_tso_bursts; 692 tx->hw_gso_wire_packets += tx_queue->old_tso_packets; 693 } 694 /* Include XDP TX in device-wide stats */ 695 tx->packets += tx_queue->complete_xdp_packets; 696 tx->bytes += tx_queue->complete_xdp_bytes; 697 } 698 } 699 } 700 701 static const struct netdev_stat_ops efx_stat_ops = { 702 .get_queue_stats_rx = efx_get_queue_stats_rx, 703 .get_queue_stats_tx = efx_get_queue_stats_tx, 704 .get_base_stats = efx_get_base_stats, 705 }; 706 707 static int efx_xdp_setup_prog(struct efx_nic *efx, struct bpf_prog *prog) 708 { 709 struct bpf_prog *old_prog; 710 711 if (efx->xdp_rxq_info_failed) { 712 netif_err(efx, drv, efx->net_dev, 713 "Unable to bind XDP program due to previous failure of rxq_info\n"); 714 return -EINVAL; 715 } 716 717 if (prog && efx->net_dev->mtu > efx_xdp_max_mtu(efx)) { 718 netif_err(efx, drv, efx->net_dev, 719 "Unable to configure XDP with MTU of %d (max: %d)\n", 720 efx->net_dev->mtu, efx_xdp_max_mtu(efx)); 721 return -EINVAL; 722 } 723 724 old_prog = rtnl_dereference(efx->xdp_prog); 725 rcu_assign_pointer(efx->xdp_prog, prog); 726 /* Release the reference that was originally passed by the caller. */ 727 if (old_prog) 728 bpf_prog_put(old_prog); 729 730 return 0; 731 } 732 733 /* Context: process, rtnl_lock() held. */ 734 static int efx_xdp(struct net_device *dev, struct netdev_bpf *xdp) 735 { 736 struct efx_nic *efx = efx_netdev_priv(dev); 737 738 switch (xdp->command) { 739 case XDP_SETUP_PROG: 740 return efx_xdp_setup_prog(efx, xdp->prog); 741 default: 742 return -EINVAL; 743 } 744 } 745 746 static int efx_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **xdpfs, 747 u32 flags) 748 { 749 struct efx_nic *efx = efx_netdev_priv(dev); 750 751 if (!netif_running(dev)) 752 return -EINVAL; 753 754 return efx_xdp_tx_buffers(efx, n, xdpfs, flags & XDP_XMIT_FLUSH); 755 } 756 757 static void efx_update_name(struct efx_nic *efx) 758 { 759 strcpy(efx->name, efx->net_dev->name); 760 efx_mtd_rename(efx); 761 efx_set_channel_names(efx); 762 } 763 764 static int efx_netdev_event(struct notifier_block *this, 765 unsigned long event, void *ptr) 766 { 767 struct net_device *net_dev = netdev_notifier_info_to_dev(ptr); 768 769 if ((net_dev->netdev_ops == &efx_netdev_ops) && 770 event == NETDEV_CHANGENAME) 771 efx_update_name(efx_netdev_priv(net_dev)); 772 773 return NOTIFY_DONE; 774 } 775 776 static struct notifier_block efx_netdev_notifier = { 777 .notifier_call = efx_netdev_event, 778 }; 779 780 static ssize_t phy_type_show(struct device *dev, 781 struct device_attribute *attr, char *buf) 782 { 783 struct efx_nic *efx = dev_get_drvdata(dev); 784 return sprintf(buf, "%d\n", efx->phy_type); 785 } 786 static DEVICE_ATTR_RO(phy_type); 787 788 static int efx_register_netdev(struct efx_nic *efx) 789 { 790 struct net_device *net_dev = efx->net_dev; 791 struct efx_channel *channel; 792 int rc; 793 794 net_dev->watchdog_timeo = 5 * HZ; 795 net_dev->irq = efx->pci_dev->irq; 796 net_dev->netdev_ops = &efx_netdev_ops; 797 net_dev->stat_ops = &efx_stat_ops; 798 if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0) 799 net_dev->priv_flags |= IFF_UNICAST_FLT; 800 net_dev->ethtool_ops = &efx_ethtool_ops; 801 netif_set_tso_max_segs(net_dev, EFX_TSO_MAX_SEGS); 802 net_dev->min_mtu = EFX_MIN_MTU; 803 net_dev->max_mtu = EFX_MAX_MTU; 804 805 rtnl_lock(); 806 807 /* Enable resets to be scheduled and check whether any were 808 * already requested. If so, the NIC is probably hosed so we 809 * abort. 810 */ 811 if (efx->reset_pending) { 812 pci_err(efx->pci_dev, "aborting probe due to scheduled reset\n"); 813 rc = -EIO; 814 goto fail_locked; 815 } 816 817 rc = dev_alloc_name(net_dev, net_dev->name); 818 if (rc < 0) 819 goto fail_locked; 820 efx_update_name(efx); 821 822 /* Always start with carrier off; PHY events will detect the link */ 823 netif_carrier_off(net_dev); 824 825 rc = register_netdevice(net_dev); 826 if (rc) 827 goto fail_locked; 828 829 efx_for_each_channel(channel, efx) { 830 struct efx_tx_queue *tx_queue; 831 efx_for_each_channel_tx_queue(tx_queue, channel) 832 efx_init_tx_queue_core_txq(tx_queue); 833 } 834 835 efx_associate(efx); 836 837 efx->state = STATE_NET_DOWN; 838 839 rtnl_unlock(); 840 841 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type); 842 if (rc) { 843 netif_err(efx, drv, efx->net_dev, 844 "failed to init net dev attributes\n"); 845 goto fail_registered; 846 } 847 848 efx_init_mcdi_logging(efx); 849 850 return 0; 851 852 fail_registered: 853 rtnl_lock(); 854 efx_dissociate(efx); 855 unregister_netdevice(net_dev); 856 fail_locked: 857 efx->state = STATE_UNINIT; 858 rtnl_unlock(); 859 netif_err(efx, drv, efx->net_dev, "could not register net dev\n"); 860 return rc; 861 } 862 863 static void efx_unregister_netdev(struct efx_nic *efx) 864 { 865 if (!efx->net_dev) 866 return; 867 868 if (WARN_ON(efx_netdev_priv(efx->net_dev) != efx)) 869 return; 870 871 if (efx_dev_registered(efx)) { 872 strscpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name)); 873 efx_fini_mcdi_logging(efx); 874 device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 875 unregister_netdev(efx->net_dev); 876 } 877 } 878 879 /************************************************************************** 880 * 881 * List of NICs we support 882 * 883 **************************************************************************/ 884 885 /* PCI device ID table */ 886 static const struct pci_device_id efx_pci_table[] = { 887 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0903), /* SFC9120 PF */ 888 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 889 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1903), /* SFC9120 VF */ 890 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 891 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0923), /* SFC9140 PF */ 892 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 893 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1923), /* SFC9140 VF */ 894 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 895 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0a03), /* SFC9220 PF */ 896 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 897 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1a03), /* SFC9220 VF */ 898 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 899 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0b03), /* SFC9250 PF */ 900 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 901 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1b03), /* SFC9250 VF */ 902 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 903 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0c03), /* X4 PF (FF/LL) */ 904 .driver_data = (unsigned long)&efx_x4_nic_type}, 905 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x2c03), /* X4 PF (FF only) */ 906 .driver_data = (unsigned long)&efx_x4_nic_type}, 907 {0} /* end of list */ 908 }; 909 910 /************************************************************************** 911 * 912 * Data housekeeping 913 * 914 **************************************************************************/ 915 916 void efx_update_sw_stats(struct efx_nic *efx, u64 *stats) 917 { 918 u64 n_rx_nodesc_trunc = 0; 919 struct efx_channel *channel; 920 921 efx_for_each_channel(channel, efx) 922 n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc; 923 stats[GENERIC_STAT_rx_nodesc_trunc] = n_rx_nodesc_trunc; 924 stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops); 925 } 926 927 /************************************************************************** 928 * 929 * PCI interface 930 * 931 **************************************************************************/ 932 933 /* Main body of final NIC shutdown code 934 * This is called only at module unload (or hotplug removal). 935 */ 936 static void efx_pci_remove_main(struct efx_nic *efx) 937 { 938 /* Flush reset_work. It can no longer be scheduled since we 939 * are not READY. 940 */ 941 WARN_ON(efx_net_active(efx->state)); 942 efx_flush_reset_workqueue(efx); 943 944 efx_disable_interrupts(efx); 945 efx_clear_interrupt_affinity(efx); 946 efx_nic_fini_interrupt(efx); 947 efx_fini_port(efx); 948 efx->type->fini(efx); 949 efx_fini_napi(efx); 950 efx_remove_all(efx); 951 } 952 953 /* Final NIC shutdown 954 * This is called only at module unload (or hotplug removal). A PF can call 955 * this on its VFs to ensure they are unbound first. 956 */ 957 static void efx_pci_remove(struct pci_dev *pci_dev) 958 { 959 struct efx_probe_data *probe_data; 960 struct efx_nic *efx; 961 962 efx = pci_get_drvdata(pci_dev); 963 if (!efx) 964 return; 965 966 /* Mark the NIC as fini, then stop the interface */ 967 rtnl_lock(); 968 efx_dissociate(efx); 969 dev_close(efx->net_dev); 970 efx_disable_interrupts(efx); 971 efx->state = STATE_UNINIT; 972 rtnl_unlock(); 973 974 if (efx->type->sriov_fini) 975 efx->type->sriov_fini(efx); 976 977 efx_fini_devlink_lock(efx); 978 efx_unregister_netdev(efx); 979 980 efx_mtd_remove(efx); 981 982 efx_pci_remove_main(efx); 983 984 efx_fini_io(efx); 985 986 probe_data = container_of(efx, struct efx_probe_data, efx); 987 efx_cxl_exit(probe_data); 988 989 pci_dbg(efx->pci_dev, "shutdown successful\n"); 990 991 efx_fini_devlink_and_unlock(efx); 992 efx_fini_struct(efx); 993 free_netdev(efx->net_dev); 994 kfree(probe_data); 995 }; 996 997 /* NIC VPD information 998 * Called during probe to display the part number of the 999 * installed NIC. 1000 */ 1001 static void efx_probe_vpd_strings(struct efx_nic *efx) 1002 { 1003 struct pci_dev *dev = efx->pci_dev; 1004 unsigned int vpd_size, kw_len; 1005 u8 *vpd_data; 1006 int start; 1007 1008 vpd_data = pci_vpd_alloc(dev, &vpd_size); 1009 if (IS_ERR(vpd_data)) { 1010 pci_warn(dev, "Unable to read VPD\n"); 1011 return; 1012 } 1013 1014 start = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, 1015 PCI_VPD_RO_KEYWORD_PARTNO, &kw_len); 1016 if (start < 0) 1017 pci_err(dev, "Part number not found or incomplete\n"); 1018 else 1019 pci_info(dev, "Part Number : %.*s\n", kw_len, vpd_data + start); 1020 1021 start = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, 1022 PCI_VPD_RO_KEYWORD_SERIALNO, &kw_len); 1023 if (start < 0) 1024 pci_err(dev, "Serial number not found or incomplete\n"); 1025 else 1026 efx->vpd_sn = kmemdup_nul(vpd_data + start, kw_len, GFP_KERNEL); 1027 1028 kfree(vpd_data); 1029 } 1030 1031 1032 /* Main body of NIC initialisation 1033 * This is called at module load (or hotplug insertion, theoretically). 1034 */ 1035 static int efx_pci_probe_main(struct efx_nic *efx) 1036 { 1037 int rc; 1038 1039 /* Do start-of-day initialisation */ 1040 rc = efx_probe_all(efx); 1041 if (rc) 1042 goto fail1; 1043 1044 efx_init_napi(efx); 1045 1046 down_write(&efx->filter_sem); 1047 rc = efx->type->init(efx); 1048 up_write(&efx->filter_sem); 1049 if (rc) { 1050 pci_err(efx->pci_dev, "failed to initialise NIC\n"); 1051 goto fail3; 1052 } 1053 1054 rc = efx_init_port(efx); 1055 if (rc) { 1056 netif_err(efx, probe, efx->net_dev, 1057 "failed to initialise port\n"); 1058 goto fail4; 1059 } 1060 1061 rc = efx_nic_init_interrupt(efx); 1062 if (rc) 1063 goto fail5; 1064 1065 efx_set_interrupt_affinity(efx); 1066 rc = efx_enable_interrupts(efx); 1067 if (rc) 1068 goto fail6; 1069 1070 return 0; 1071 1072 fail6: 1073 efx_clear_interrupt_affinity(efx); 1074 efx_nic_fini_interrupt(efx); 1075 fail5: 1076 efx_fini_port(efx); 1077 fail4: 1078 efx->type->fini(efx); 1079 fail3: 1080 efx_fini_napi(efx); 1081 efx_remove_all(efx); 1082 fail1: 1083 return rc; 1084 } 1085 1086 static int efx_pci_probe_post_io(struct efx_nic *efx) 1087 { 1088 struct net_device *net_dev = efx->net_dev; 1089 int rc = efx_pci_probe_main(efx); 1090 1091 if (rc) 1092 return rc; 1093 1094 if (efx->type->sriov_init) { 1095 rc = efx->type->sriov_init(efx); 1096 if (rc) 1097 pci_err(efx->pci_dev, "SR-IOV can't be enabled rc %d\n", 1098 rc); 1099 } 1100 1101 /* Determine netdevice features */ 1102 net_dev->features |= efx->type->offload_features; 1103 1104 /* Add TSO features */ 1105 if (efx->type->tso_versions && efx->type->tso_versions(efx)) 1106 net_dev->features |= NETIF_F_TSO | NETIF_F_TSO6; 1107 1108 /* Mask for features that also apply to VLAN devices */ 1109 net_dev->vlan_features |= (NETIF_F_HW_CSUM | NETIF_F_SG | 1110 NETIF_F_HIGHDMA | NETIF_F_ALL_TSO | 1111 NETIF_F_RXCSUM); 1112 1113 /* Determine user configurable features */ 1114 net_dev->hw_features |= net_dev->features & ~efx->fixed_features; 1115 1116 /* Disable receiving frames with bad FCS, by default. */ 1117 net_dev->features &= ~NETIF_F_RXALL; 1118 1119 /* Disable VLAN filtering by default. It may be enforced if 1120 * the feature is fixed (i.e. VLAN filters are required to 1121 * receive VLAN tagged packets due to vPort restrictions). 1122 */ 1123 net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 1124 net_dev->features |= efx->fixed_features; 1125 1126 net_dev->xdp_features = NETDEV_XDP_ACT_BASIC | 1127 NETDEV_XDP_ACT_REDIRECT | 1128 NETDEV_XDP_ACT_NDO_XMIT; 1129 1130 /* devlink creation, registration and lock */ 1131 rc = efx_probe_devlink_and_lock(efx); 1132 if (rc) 1133 pci_err(efx->pci_dev, "devlink registration failed"); 1134 1135 rc = efx_register_netdev(efx); 1136 efx_probe_devlink_unlock(efx); 1137 if (!rc) 1138 return 0; 1139 1140 efx_pci_remove_main(efx); 1141 return rc; 1142 } 1143 1144 /* NIC initialisation 1145 * 1146 * This is called at module load (or hotplug insertion, 1147 * theoretically). It sets up PCI mappings, resets the NIC, 1148 * sets up and registers the network devices with the kernel and hooks 1149 * the interrupt service routine. It does not prepare the device for 1150 * transmission; this is left to the first time one of the network 1151 * interfaces is brought up (i.e. efx_net_open). 1152 */ 1153 static int efx_pci_probe(struct pci_dev *pci_dev, 1154 const struct pci_device_id *entry) 1155 { 1156 struct efx_probe_data *probe_data, **probe_ptr; 1157 struct net_device *net_dev; 1158 struct efx_nic *efx; 1159 int rc; 1160 1161 /* Allocate probe data and struct efx_nic */ 1162 probe_data = kzalloc_obj(*probe_data); 1163 if (!probe_data) 1164 return -ENOMEM; 1165 probe_data->pci_dev = pci_dev; 1166 efx = &probe_data->efx; 1167 1168 /* Allocate and initialise a struct net_device */ 1169 net_dev = alloc_etherdev_mq(sizeof(probe_data), EFX_MAX_CORE_TX_QUEUES); 1170 if (!net_dev) { 1171 rc = -ENOMEM; 1172 goto fail0; 1173 } 1174 probe_ptr = netdev_priv(net_dev); 1175 *probe_ptr = probe_data; 1176 efx->net_dev = net_dev; 1177 efx->type = (const struct efx_nic_type *) entry->driver_data; 1178 efx->fixed_features |= NETIF_F_HIGHDMA; 1179 1180 pci_set_drvdata(pci_dev, efx); 1181 SET_NETDEV_DEV(net_dev, &pci_dev->dev); 1182 rc = efx_init_struct(efx, pci_dev); 1183 if (rc) 1184 goto fail1; 1185 1186 pci_info(pci_dev, "Solarflare NIC detected\n"); 1187 1188 if (!efx->type->is_vf) 1189 efx_probe_vpd_strings(efx); 1190 1191 /* Set up basic I/O (BAR mappings etc) */ 1192 rc = efx_init_io(efx, efx->type->mem_bar(efx), efx->type->max_dma_mask, 1193 efx->type->mem_map_size(efx)); 1194 if (rc) 1195 goto fail2; 1196 1197 /* A successful cxl initialization implies a CXL region created to be 1198 * used for PIO buffers. If there is no CXL support legacy PIO buffers 1199 * defined at specific PCI BAR regions will be used. If there is CXL 1200 * support and the cxl initialization fails, the driver probe fails. 1201 */ 1202 rc = efx_cxl_init(probe_data); 1203 if (rc) { 1204 pci_err(pci_dev, "CXL initialization failed with error %d\n", rc); 1205 goto fail3; 1206 } 1207 1208 rc = efx_pci_probe_post_io(efx); 1209 if (rc) { 1210 /* On failure, retry once immediately. 1211 * If we aborted probe due to a scheduled reset, dismiss it. 1212 */ 1213 efx->reset_pending = 0; 1214 rc = efx_pci_probe_post_io(efx); 1215 if (rc) { 1216 /* On another failure, retry once more 1217 * after a 50-305ms delay. 1218 */ 1219 unsigned char r; 1220 1221 get_random_bytes(&r, 1); 1222 msleep((unsigned int)r + 50); 1223 efx->reset_pending = 0; 1224 rc = efx_pci_probe_post_io(efx); 1225 } 1226 } 1227 if (rc) 1228 goto fail3; 1229 1230 netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n"); 1231 1232 /* Try to create MTDs, but allow this to fail */ 1233 rtnl_lock(); 1234 rc = efx_mtd_probe(efx); 1235 rtnl_unlock(); 1236 if (rc && rc != -EPERM) 1237 netif_warn(efx, probe, efx->net_dev, 1238 "failed to create MTDs (%d)\n", rc); 1239 1240 if (efx->type->udp_tnl_push_ports) 1241 efx->type->udp_tnl_push_ports(efx); 1242 1243 return 0; 1244 1245 fail3: 1246 efx_cxl_exit(probe_data); 1247 efx_fini_io(efx); 1248 fail2: 1249 efx_fini_struct(efx); 1250 fail1: 1251 WARN_ON(rc > 0); 1252 netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc); 1253 free_netdev(net_dev); 1254 fail0: 1255 kfree(probe_data); 1256 return rc; 1257 } 1258 1259 /* efx_pci_sriov_configure returns the actual number of Virtual Functions 1260 * enabled on success 1261 */ 1262 #ifdef CONFIG_SFC_SRIOV 1263 static int efx_pci_sriov_configure(struct pci_dev *dev, int num_vfs) 1264 { 1265 int rc; 1266 struct efx_nic *efx = pci_get_drvdata(dev); 1267 1268 if (efx->type->sriov_configure) { 1269 rc = efx->type->sriov_configure(efx, num_vfs); 1270 if (rc) 1271 return rc; 1272 else 1273 return num_vfs; 1274 } else 1275 return -EOPNOTSUPP; 1276 } 1277 #endif 1278 1279 static int efx_pm_freeze(struct device *dev) 1280 { 1281 struct efx_nic *efx = dev_get_drvdata(dev); 1282 1283 rtnl_lock(); 1284 1285 if (efx_net_active(efx->state)) { 1286 efx_device_detach_sync(efx); 1287 1288 efx_stop_all(efx); 1289 efx_disable_interrupts(efx); 1290 1291 efx->state = efx_freeze(efx->state); 1292 } 1293 1294 rtnl_unlock(); 1295 1296 return 0; 1297 } 1298 1299 static void efx_pci_shutdown(struct pci_dev *pci_dev) 1300 { 1301 struct efx_nic *efx = pci_get_drvdata(pci_dev); 1302 1303 if (!efx) 1304 return; 1305 1306 efx_pm_freeze(&pci_dev->dev); 1307 pci_disable_device(pci_dev); 1308 } 1309 1310 static int efx_pm_thaw(struct device *dev) 1311 { 1312 int rc; 1313 struct efx_nic *efx = dev_get_drvdata(dev); 1314 1315 rtnl_lock(); 1316 1317 if (efx_frozen(efx->state)) { 1318 rc = efx_enable_interrupts(efx); 1319 if (rc) 1320 goto fail; 1321 1322 mutex_lock(&efx->mac_lock); 1323 efx_mcdi_port_reconfigure(efx); 1324 mutex_unlock(&efx->mac_lock); 1325 1326 efx_start_all(efx); 1327 1328 efx_device_attach_if_not_resetting(efx); 1329 1330 efx->state = efx_thaw(efx->state); 1331 1332 efx->type->resume_wol(efx); 1333 } 1334 1335 rtnl_unlock(); 1336 1337 /* Reschedule any quenched resets scheduled during efx_pm_freeze() */ 1338 efx_queue_reset_work(efx); 1339 1340 return 0; 1341 1342 fail: 1343 rtnl_unlock(); 1344 1345 return rc; 1346 } 1347 1348 static int efx_pm_poweroff(struct device *dev) 1349 { 1350 struct pci_dev *pci_dev = to_pci_dev(dev); 1351 struct efx_nic *efx = pci_get_drvdata(pci_dev); 1352 1353 efx->type->fini(efx); 1354 1355 efx->reset_pending = 0; 1356 1357 pci_save_state(pci_dev); 1358 return pci_set_power_state(pci_dev, PCI_D3hot); 1359 } 1360 1361 /* Used for both resume and restore */ 1362 static int efx_pm_resume(struct device *dev) 1363 { 1364 struct pci_dev *pci_dev = to_pci_dev(dev); 1365 struct efx_nic *efx = pci_get_drvdata(pci_dev); 1366 int rc; 1367 1368 rc = pci_set_power_state(pci_dev, PCI_D0); 1369 if (rc) 1370 return rc; 1371 pci_restore_state(pci_dev); 1372 rc = pci_enable_device(pci_dev); 1373 if (rc) 1374 return rc; 1375 pci_set_master(efx->pci_dev); 1376 rc = efx->type->reset(efx, RESET_TYPE_ALL); 1377 if (rc) 1378 return rc; 1379 down_write(&efx->filter_sem); 1380 rc = efx->type->init(efx); 1381 up_write(&efx->filter_sem); 1382 if (rc) 1383 return rc; 1384 rc = efx_pm_thaw(dev); 1385 return rc; 1386 } 1387 1388 static int efx_pm_suspend(struct device *dev) 1389 { 1390 int rc; 1391 1392 efx_pm_freeze(dev); 1393 rc = efx_pm_poweroff(dev); 1394 if (rc) 1395 efx_pm_resume(dev); 1396 return rc; 1397 } 1398 1399 static const struct dev_pm_ops efx_pm_ops = { 1400 .suspend = efx_pm_suspend, 1401 .resume = efx_pm_resume, 1402 .freeze = efx_pm_freeze, 1403 .thaw = efx_pm_thaw, 1404 .poweroff = efx_pm_poweroff, 1405 .restore = efx_pm_resume, 1406 }; 1407 1408 static struct pci_driver efx_pci_driver = { 1409 .name = KBUILD_MODNAME, 1410 .id_table = efx_pci_table, 1411 .probe = efx_pci_probe, 1412 .remove = efx_pci_remove, 1413 .driver.pm = &efx_pm_ops, 1414 .shutdown = efx_pci_shutdown, 1415 .err_handler = &efx_err_handlers, 1416 #ifdef CONFIG_SFC_SRIOV 1417 .sriov_configure = efx_pci_sriov_configure, 1418 #endif 1419 }; 1420 1421 /************************************************************************** 1422 * 1423 * Kernel module interface 1424 * 1425 *************************************************************************/ 1426 1427 static int __init efx_init_module(void) 1428 { 1429 int rc; 1430 1431 printk(KERN_INFO "Solarflare NET driver\n"); 1432 1433 rc = register_netdevice_notifier(&efx_netdev_notifier); 1434 if (rc) 1435 goto err_notifier; 1436 1437 rc = efx_create_reset_workqueue(); 1438 if (rc) 1439 goto err_reset; 1440 1441 rc = pci_register_driver(&efx_pci_driver); 1442 if (rc < 0) 1443 goto err_pci; 1444 1445 rc = pci_register_driver(&ef100_pci_driver); 1446 if (rc < 0) 1447 goto err_pci_ef100; 1448 1449 return 0; 1450 1451 err_pci_ef100: 1452 pci_unregister_driver(&efx_pci_driver); 1453 err_pci: 1454 efx_destroy_reset_workqueue(); 1455 err_reset: 1456 unregister_netdevice_notifier(&efx_netdev_notifier); 1457 err_notifier: 1458 return rc; 1459 } 1460 1461 static void __exit efx_exit_module(void) 1462 { 1463 printk(KERN_INFO "Solarflare NET driver unloading\n"); 1464 1465 pci_unregister_driver(&ef100_pci_driver); 1466 pci_unregister_driver(&efx_pci_driver); 1467 efx_destroy_reset_workqueue(); 1468 unregister_netdevice_notifier(&efx_netdev_notifier); 1469 1470 } 1471 1472 module_init(efx_init_module); 1473 module_exit(efx_exit_module); 1474 1475 MODULE_AUTHOR("Solarflare Communications and " 1476 "Michael Brown <mbrown@fensystems.co.uk>"); 1477 MODULE_DESCRIPTION("Solarflare network driver"); 1478 MODULE_LICENSE("GPL"); 1479 MODULE_DEVICE_TABLE(pci, efx_pci_table); 1480