1 // SPDX-License-Identifier: GPL-2.0-only 2 /**************************************************************************** 3 * Driver for Solarflare network controllers and boards 4 * Copyright 2005-2006 Fen Systems Ltd. 5 * Copyright 2005-2013 Solarflare Communications Inc. 6 */ 7 8 #include <linux/filter.h> 9 #include <linux/module.h> 10 #include <linux/pci.h> 11 #include <linux/netdevice.h> 12 #include <linux/etherdevice.h> 13 #include <linux/delay.h> 14 #include <linux/notifier.h> 15 #include <linux/ip.h> 16 #include <linux/tcp.h> 17 #include <linux/in.h> 18 #include <linux/ethtool.h> 19 #include <linux/topology.h> 20 #include <linux/gfp.h> 21 #include <linux/interrupt.h> 22 #include "net_driver.h" 23 #include <net/gre.h> 24 #include <net/udp_tunnel.h> 25 #include <net/netdev_queues.h> 26 #include "efx.h" 27 #include "efx_common.h" 28 #include "efx_channels.h" 29 #include "ef100.h" 30 #include "rx_common.h" 31 #include "tx_common.h" 32 #include "nic.h" 33 #include "io.h" 34 #include "selftest.h" 35 #include "sriov.h" 36 #include "efx_devlink.h" 37 #include "efx_cxl.h" 38 39 #include "mcdi_port_common.h" 40 #include "mcdi_pcol.h" 41 #include "workarounds.h" 42 43 /************************************************************************** 44 * 45 * Configurable values 46 * 47 *************************************************************************/ 48 49 module_param_named(interrupt_mode, efx_interrupt_mode, uint, 0444); 50 MODULE_PARM_DESC(interrupt_mode, 51 "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)"); 52 53 module_param(rss_cpus, uint, 0444); 54 MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling"); 55 56 /* 57 * Use separate channels for TX and RX events 58 * 59 * Set this to 1 to use separate channels for TX and RX. It allows us 60 * to control interrupt affinity separately for TX and RX. 61 * 62 * This is only used in MSI-X interrupt mode 63 */ 64 bool efx_separate_tx_channels; 65 module_param(efx_separate_tx_channels, bool, 0444); 66 MODULE_PARM_DESC(efx_separate_tx_channels, 67 "Use separate channels for TX and RX"); 68 69 /* Initial interrupt moderation settings. They can be modified after 70 * module load with ethtool. 71 * 72 * The default for RX should strike a balance between increasing the 73 * round-trip latency and reducing overhead. 74 */ 75 static unsigned int rx_irq_mod_usec = 60; 76 77 /* Initial interrupt moderation settings. They can be modified after 78 * module load with ethtool. 79 * 80 * This default is chosen to ensure that a 10G link does not go idle 81 * while a TX queue is stopped after it has become full. A queue is 82 * restarted when it drops below half full. The time this takes (assuming 83 * worst case 3 descriptors per packet and 1024 descriptors) is 84 * 512 / 3 * 1.2 = 205 usec. 85 */ 86 static unsigned int tx_irq_mod_usec = 150; 87 88 static bool phy_flash_cfg; 89 module_param(phy_flash_cfg, bool, 0644); 90 MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially"); 91 92 static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 93 NETIF_MSG_LINK | NETIF_MSG_IFDOWN | 94 NETIF_MSG_IFUP | NETIF_MSG_RX_ERR | 95 NETIF_MSG_TX_ERR | NETIF_MSG_HW); 96 module_param(debug, uint, 0); 97 MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value"); 98 99 /************************************************************************** 100 * 101 * Utility functions and prototypes 102 * 103 *************************************************************************/ 104 105 static void efx_remove_port(struct efx_nic *efx); 106 static int efx_xdp_setup_prog(struct efx_nic *efx, struct bpf_prog *prog); 107 static int efx_xdp(struct net_device *dev, struct netdev_bpf *xdp); 108 static int efx_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **xdpfs, 109 u32 flags); 110 111 /************************************************************************** 112 * 113 * Port handling 114 * 115 **************************************************************************/ 116 117 static void efx_fini_port(struct efx_nic *efx); 118 119 static int efx_probe_port(struct efx_nic *efx) 120 { 121 int rc; 122 123 netif_dbg(efx, probe, efx->net_dev, "create port\n"); 124 125 if (phy_flash_cfg) 126 efx->phy_mode = PHY_MODE_SPECIAL; 127 128 /* Connect up MAC/PHY operations table */ 129 rc = efx->type->probe_port(efx); 130 if (rc) 131 return rc; 132 133 /* Initialise MAC address to permanent address */ 134 eth_hw_addr_set(efx->net_dev, efx->net_dev->perm_addr); 135 136 return 0; 137 } 138 139 static int efx_init_port(struct efx_nic *efx) 140 { 141 int rc; 142 143 netif_dbg(efx, drv, efx->net_dev, "init port\n"); 144 145 mutex_lock(&efx->mac_lock); 146 147 efx->port_initialized = true; 148 149 /* Ensure the PHY advertises the correct flow control settings */ 150 rc = efx_mcdi_port_reconfigure(efx); 151 if (rc && rc != -EPERM) 152 goto fail; 153 154 mutex_unlock(&efx->mac_lock); 155 return 0; 156 157 fail: 158 mutex_unlock(&efx->mac_lock); 159 return rc; 160 } 161 162 static void efx_fini_port(struct efx_nic *efx) 163 { 164 netif_dbg(efx, drv, efx->net_dev, "shut down port\n"); 165 166 if (!efx->port_initialized) 167 return; 168 169 efx->port_initialized = false; 170 171 efx->link_state.up = false; 172 efx_link_status_changed(efx); 173 } 174 175 static void efx_remove_port(struct efx_nic *efx) 176 { 177 netif_dbg(efx, drv, efx->net_dev, "destroying port\n"); 178 179 efx->type->remove_port(efx); 180 } 181 182 /************************************************************************** 183 * 184 * NIC handling 185 * 186 **************************************************************************/ 187 188 static LIST_HEAD(efx_primary_list); 189 static LIST_HEAD(efx_unassociated_list); 190 191 static bool efx_same_controller(struct efx_nic *left, struct efx_nic *right) 192 { 193 return left->type == right->type && 194 left->vpd_sn && right->vpd_sn && 195 !strcmp(left->vpd_sn, right->vpd_sn); 196 } 197 198 static void efx_associate(struct efx_nic *efx) 199 { 200 struct efx_nic *other, *next; 201 202 if (efx->primary == efx) { 203 /* Adding primary function; look for secondaries */ 204 205 netif_dbg(efx, probe, efx->net_dev, "adding to primary list\n"); 206 list_add_tail(&efx->node, &efx_primary_list); 207 208 list_for_each_entry_safe(other, next, &efx_unassociated_list, 209 node) { 210 if (efx_same_controller(efx, other)) { 211 list_del(&other->node); 212 netif_dbg(other, probe, other->net_dev, 213 "moving to secondary list of %s %s\n", 214 pci_name(efx->pci_dev), 215 efx->net_dev->name); 216 list_add_tail(&other->node, 217 &efx->secondary_list); 218 other->primary = efx; 219 } 220 } 221 } else { 222 /* Adding secondary function; look for primary */ 223 224 list_for_each_entry(other, &efx_primary_list, node) { 225 if (efx_same_controller(efx, other)) { 226 netif_dbg(efx, probe, efx->net_dev, 227 "adding to secondary list of %s %s\n", 228 pci_name(other->pci_dev), 229 other->net_dev->name); 230 list_add_tail(&efx->node, 231 &other->secondary_list); 232 efx->primary = other; 233 return; 234 } 235 } 236 237 netif_dbg(efx, probe, efx->net_dev, 238 "adding to unassociated list\n"); 239 list_add_tail(&efx->node, &efx_unassociated_list); 240 } 241 } 242 243 static void efx_dissociate(struct efx_nic *efx) 244 { 245 struct efx_nic *other, *next; 246 247 list_del(&efx->node); 248 efx->primary = NULL; 249 250 list_for_each_entry_safe(other, next, &efx->secondary_list, node) { 251 list_del(&other->node); 252 netif_dbg(other, probe, other->net_dev, 253 "moving to unassociated list\n"); 254 list_add_tail(&other->node, &efx_unassociated_list); 255 other->primary = NULL; 256 } 257 } 258 259 static int efx_probe_nic(struct efx_nic *efx) 260 { 261 int rc; 262 263 netif_dbg(efx, probe, efx->net_dev, "creating NIC\n"); 264 265 /* Carry out hardware-type specific initialisation */ 266 rc = efx->type->probe(efx); 267 if (rc) 268 return rc; 269 270 do { 271 if (!efx->max_channels || !efx->max_tx_channels) { 272 netif_err(efx, drv, efx->net_dev, 273 "Insufficient resources to allocate" 274 " any channels\n"); 275 rc = -ENOSPC; 276 goto fail1; 277 } 278 279 /* Determine the number of channels and queues by trying 280 * to hook in MSI-X interrupts. 281 */ 282 rc = efx_probe_interrupts(efx); 283 if (rc) 284 goto fail1; 285 286 rc = efx_set_channels(efx); 287 if (rc) 288 goto fail1; 289 290 /* dimension_resources can fail with EAGAIN */ 291 rc = efx->type->dimension_resources(efx); 292 if (rc != 0 && rc != -EAGAIN) 293 goto fail2; 294 295 if (rc == -EAGAIN) 296 /* try again with new max_channels */ 297 efx_remove_interrupts(efx); 298 299 } while (rc == -EAGAIN); 300 301 if (efx->n_channels > 1) 302 netdev_rss_key_fill(efx->rss_context.rx_hash_key, 303 sizeof(efx->rss_context.rx_hash_key)); 304 efx_set_default_rx_indir_table(efx, efx->rss_context.rx_indir_table); 305 306 /* Initialise the interrupt moderation settings */ 307 efx->irq_mod_step_us = DIV_ROUND_UP(efx->timer_quantum_ns, 1000); 308 efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true, 309 true); 310 311 return 0; 312 313 fail2: 314 efx_remove_interrupts(efx); 315 fail1: 316 efx->type->remove(efx); 317 return rc; 318 } 319 320 static void efx_remove_nic(struct efx_nic *efx) 321 { 322 netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n"); 323 324 efx_remove_interrupts(efx); 325 efx->type->remove(efx); 326 } 327 328 /************************************************************************** 329 * 330 * NIC startup/shutdown 331 * 332 *************************************************************************/ 333 334 static int efx_probe_all(struct efx_nic *efx) 335 { 336 int rc; 337 338 rc = efx_probe_nic(efx); 339 if (rc) { 340 netif_err(efx, probe, efx->net_dev, "failed to create NIC\n"); 341 goto fail1; 342 } 343 344 rc = efx_probe_port(efx); 345 if (rc) { 346 netif_err(efx, probe, efx->net_dev, "failed to create port\n"); 347 goto fail2; 348 } 349 350 BUILD_BUG_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_RXQ_MIN_ENT); 351 if (WARN_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_TXQ_MIN_ENT(efx))) { 352 rc = -EINVAL; 353 goto fail3; 354 } 355 356 #ifdef CONFIG_SFC_SRIOV 357 rc = efx->type->vswitching_probe(efx); 358 if (rc) /* not fatal; the PF will still work fine */ 359 netif_warn(efx, probe, efx->net_dev, 360 "failed to setup vswitching rc=%d;" 361 " VFs may not function\n", rc); 362 #endif 363 364 rc = efx_probe_filters(efx); 365 if (rc) { 366 netif_err(efx, probe, efx->net_dev, 367 "failed to create filter tables\n"); 368 goto fail4; 369 } 370 371 rc = efx_probe_channels(efx); 372 if (rc) 373 goto fail5; 374 375 efx->state = STATE_NET_DOWN; 376 377 return 0; 378 379 fail5: 380 efx_remove_filters(efx); 381 fail4: 382 #ifdef CONFIG_SFC_SRIOV 383 efx->type->vswitching_remove(efx); 384 #endif 385 fail3: 386 efx_remove_port(efx); 387 fail2: 388 efx_remove_nic(efx); 389 fail1: 390 return rc; 391 } 392 393 static void efx_remove_all(struct efx_nic *efx) 394 { 395 rtnl_lock(); 396 efx_xdp_setup_prog(efx, NULL); 397 rtnl_unlock(); 398 399 efx_remove_channels(efx); 400 efx_remove_filters(efx); 401 #ifdef CONFIG_SFC_SRIOV 402 efx->type->vswitching_remove(efx); 403 #endif 404 efx_remove_port(efx); 405 efx_remove_nic(efx); 406 } 407 408 /************************************************************************** 409 * 410 * Interrupt moderation 411 * 412 **************************************************************************/ 413 unsigned int efx_usecs_to_ticks(struct efx_nic *efx, unsigned int usecs) 414 { 415 if (usecs == 0) 416 return 0; 417 if (usecs * 1000 < efx->timer_quantum_ns) 418 return 1; /* never round down to 0 */ 419 return usecs * 1000 / efx->timer_quantum_ns; 420 } 421 422 /* Set interrupt moderation parameters */ 423 int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs, 424 unsigned int rx_usecs, bool rx_adaptive, 425 bool rx_may_override_tx) 426 { 427 struct efx_channel *channel; 428 unsigned int timer_max_us; 429 430 EFX_ASSERT_RESET_SERIALISED(efx); 431 432 timer_max_us = efx->timer_max_ns / 1000; 433 434 if (tx_usecs > timer_max_us || rx_usecs > timer_max_us) 435 return -EINVAL; 436 437 if (tx_usecs != rx_usecs && efx->tx_channel_offset == 0 && 438 !rx_may_override_tx) { 439 netif_err(efx, drv, efx->net_dev, "Channels are shared. " 440 "RX and TX IRQ moderation must be equal\n"); 441 return -EINVAL; 442 } 443 444 efx->irq_rx_adaptive = rx_adaptive; 445 efx->irq_rx_moderation_us = rx_usecs; 446 efx_for_each_channel(channel, efx) { 447 if (efx_channel_has_rx_queue(channel)) 448 channel->irq_moderation_us = rx_usecs; 449 else if (efx_channel_has_tx_queues(channel)) 450 channel->irq_moderation_us = tx_usecs; 451 else if (efx_channel_is_xdp_tx(channel)) 452 channel->irq_moderation_us = tx_usecs; 453 } 454 455 return 0; 456 } 457 458 void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs, 459 unsigned int *rx_usecs, bool *rx_adaptive) 460 { 461 *rx_adaptive = efx->irq_rx_adaptive; 462 *rx_usecs = efx->irq_rx_moderation_us; 463 464 /* If channels are shared between RX and TX, so is IRQ 465 * moderation. Otherwise, IRQ moderation is the same for all 466 * TX channels and is not adaptive. 467 */ 468 if (efx->tx_channel_offset == 0) { 469 *tx_usecs = *rx_usecs; 470 } else { 471 struct efx_channel *tx_channel; 472 473 tx_channel = efx->channel[efx->tx_channel_offset]; 474 *tx_usecs = tx_channel->irq_moderation_us; 475 } 476 } 477 478 /************************************************************************** 479 * 480 * Kernel net device interface 481 * 482 *************************************************************************/ 483 484 /* Context: process, rtnl_lock() held. */ 485 int efx_net_open(struct net_device *net_dev) 486 { 487 struct efx_nic *efx = efx_netdev_priv(net_dev); 488 int rc; 489 490 netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n", 491 raw_smp_processor_id()); 492 493 rc = efx_check_disabled(efx); 494 if (rc) 495 return rc; 496 if (efx->phy_mode & PHY_MODE_SPECIAL) 497 return -EBUSY; 498 if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL)) 499 return -EIO; 500 501 /* Notify the kernel of the link state polled during driver load, 502 * before the monitor starts running */ 503 efx_link_status_changed(efx); 504 505 efx_start_all(efx); 506 if (efx->state == STATE_DISABLED || efx->reset_pending) 507 netif_device_detach(efx->net_dev); 508 else 509 efx->state = STATE_NET_UP; 510 511 return 0; 512 } 513 514 /* Context: process, rtnl_lock() held. 515 * Note that the kernel will ignore our return code; this method 516 * should really be a void. 517 */ 518 int efx_net_stop(struct net_device *net_dev) 519 { 520 struct efx_nic *efx = efx_netdev_priv(net_dev); 521 522 netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n", 523 raw_smp_processor_id()); 524 525 /* Stop the device and flush all the channels */ 526 efx_stop_all(efx); 527 528 return 0; 529 } 530 531 static int efx_vlan_rx_add_vid(struct net_device *net_dev, __be16 proto, u16 vid) 532 { 533 struct efx_nic *efx = efx_netdev_priv(net_dev); 534 535 if (efx->type->vlan_rx_add_vid) 536 return efx->type->vlan_rx_add_vid(efx, proto, vid); 537 else 538 return -EOPNOTSUPP; 539 } 540 541 static int efx_vlan_rx_kill_vid(struct net_device *net_dev, __be16 proto, u16 vid) 542 { 543 struct efx_nic *efx = efx_netdev_priv(net_dev); 544 545 if (efx->type->vlan_rx_kill_vid) 546 return efx->type->vlan_rx_kill_vid(efx, proto, vid); 547 else 548 return -EOPNOTSUPP; 549 } 550 551 static int efx_hwtstamp_set(struct net_device *net_dev, 552 struct kernel_hwtstamp_config *config, 553 struct netlink_ext_ack *extack) 554 { 555 struct efx_nic *efx = efx_netdev_priv(net_dev); 556 557 return efx_ptp_set_ts_config(efx, config, extack); 558 } 559 560 static int efx_hwtstamp_get(struct net_device *net_dev, 561 struct kernel_hwtstamp_config *config) 562 { 563 struct efx_nic *efx = efx_netdev_priv(net_dev); 564 565 return efx_ptp_get_ts_config(efx, config); 566 } 567 568 static const struct net_device_ops efx_netdev_ops = { 569 .ndo_open = efx_net_open, 570 .ndo_stop = efx_net_stop, 571 .ndo_get_stats64 = efx_net_stats, 572 .ndo_tx_timeout = efx_watchdog, 573 .ndo_start_xmit = efx_hard_start_xmit, 574 .ndo_validate_addr = eth_validate_addr, 575 .ndo_change_mtu = efx_change_mtu, 576 .ndo_set_mac_address = efx_set_mac_address, 577 .ndo_set_rx_mode = efx_set_rx_mode, 578 .ndo_set_features = efx_set_features, 579 .ndo_features_check = efx_features_check, 580 .ndo_vlan_rx_add_vid = efx_vlan_rx_add_vid, 581 .ndo_vlan_rx_kill_vid = efx_vlan_rx_kill_vid, 582 .ndo_hwtstamp_set = efx_hwtstamp_set, 583 .ndo_hwtstamp_get = efx_hwtstamp_get, 584 #ifdef CONFIG_SFC_SRIOV 585 .ndo_set_vf_mac = efx_sriov_set_vf_mac, 586 .ndo_set_vf_vlan = efx_sriov_set_vf_vlan, 587 .ndo_set_vf_spoofchk = efx_sriov_set_vf_spoofchk, 588 .ndo_get_vf_config = efx_sriov_get_vf_config, 589 .ndo_set_vf_link_state = efx_sriov_set_vf_link_state, 590 #endif 591 .ndo_get_phys_port_id = efx_get_phys_port_id, 592 .ndo_get_phys_port_name = efx_get_phys_port_name, 593 #ifdef CONFIG_RFS_ACCEL 594 .ndo_rx_flow_steer = efx_filter_rfs, 595 #endif 596 .ndo_xdp_xmit = efx_xdp_xmit, 597 .ndo_bpf = efx_xdp 598 }; 599 600 static void efx_get_queue_stats_rx(struct net_device *net_dev, int idx, 601 struct netdev_queue_stats_rx *stats) 602 { 603 struct efx_nic *efx = efx_netdev_priv(net_dev); 604 struct efx_rx_queue *rx_queue; 605 struct efx_channel *channel; 606 607 channel = efx_get_channel(efx, idx); 608 rx_queue = efx_channel_get_rx_queue(channel); 609 /* Count only packets since last time datapath was started */ 610 stats->packets = rx_queue->rx_packets - rx_queue->old_rx_packets; 611 stats->bytes = rx_queue->rx_bytes - rx_queue->old_rx_bytes; 612 stats->hw_drops = efx_get_queue_stat_rx_hw_drops(channel) - 613 channel->old_n_rx_hw_drops; 614 stats->hw_drop_overruns = channel->n_rx_nodesc_trunc - 615 channel->old_n_rx_hw_drop_overruns; 616 } 617 618 static void efx_get_queue_stats_tx(struct net_device *net_dev, int idx, 619 struct netdev_queue_stats_tx *stats) 620 { 621 struct efx_nic *efx = efx_netdev_priv(net_dev); 622 struct efx_tx_queue *tx_queue; 623 struct efx_channel *channel; 624 625 channel = efx_get_tx_channel(efx, idx); 626 stats->packets = 0; 627 stats->bytes = 0; 628 stats->hw_gso_packets = 0; 629 stats->hw_gso_wire_packets = 0; 630 efx_for_each_channel_tx_queue(tx_queue, channel) { 631 stats->packets += tx_queue->complete_packets - 632 tx_queue->old_complete_packets; 633 stats->bytes += tx_queue->complete_bytes - 634 tx_queue->old_complete_bytes; 635 /* Note that, unlike stats->packets and stats->bytes, 636 * these count TXes enqueued, rather than completed, 637 * which may not be what users expect. 638 */ 639 stats->hw_gso_packets += tx_queue->tso_bursts - 640 tx_queue->old_tso_bursts; 641 stats->hw_gso_wire_packets += tx_queue->tso_packets - 642 tx_queue->old_tso_packets; 643 } 644 } 645 646 static void efx_get_base_stats(struct net_device *net_dev, 647 struct netdev_queue_stats_rx *rx, 648 struct netdev_queue_stats_tx *tx) 649 { 650 struct efx_nic *efx = efx_netdev_priv(net_dev); 651 struct efx_tx_queue *tx_queue; 652 struct efx_rx_queue *rx_queue; 653 struct efx_channel *channel; 654 655 rx->packets = 0; 656 rx->bytes = 0; 657 rx->hw_drops = 0; 658 rx->hw_drop_overruns = 0; 659 tx->packets = 0; 660 tx->bytes = 0; 661 tx->hw_gso_packets = 0; 662 tx->hw_gso_wire_packets = 0; 663 664 /* Count all packets on non-core queues, and packets before last 665 * datapath start on core queues. 666 */ 667 efx_for_each_channel(channel, efx) { 668 rx_queue = efx_channel_get_rx_queue(channel); 669 if (channel->channel >= net_dev->real_num_rx_queues) { 670 rx->packets += rx_queue->rx_packets; 671 rx->bytes += rx_queue->rx_bytes; 672 rx->hw_drops += efx_get_queue_stat_rx_hw_drops(channel); 673 rx->hw_drop_overruns += channel->n_rx_nodesc_trunc; 674 } else { 675 rx->packets += rx_queue->old_rx_packets; 676 rx->bytes += rx_queue->old_rx_bytes; 677 rx->hw_drops += channel->old_n_rx_hw_drops; 678 rx->hw_drop_overruns += channel->old_n_rx_hw_drop_overruns; 679 } 680 efx_for_each_channel_tx_queue(tx_queue, channel) { 681 if (channel->channel < efx->tx_channel_offset || 682 channel->channel >= efx->tx_channel_offset + 683 net_dev->real_num_tx_queues) { 684 tx->packets += tx_queue->complete_packets; 685 tx->bytes += tx_queue->complete_bytes; 686 tx->hw_gso_packets += tx_queue->tso_bursts; 687 tx->hw_gso_wire_packets += tx_queue->tso_packets; 688 } else { 689 tx->packets += tx_queue->old_complete_packets; 690 tx->bytes += tx_queue->old_complete_bytes; 691 tx->hw_gso_packets += tx_queue->old_tso_bursts; 692 tx->hw_gso_wire_packets += tx_queue->old_tso_packets; 693 } 694 /* Include XDP TX in device-wide stats */ 695 tx->packets += tx_queue->complete_xdp_packets; 696 tx->bytes += tx_queue->complete_xdp_bytes; 697 } 698 } 699 } 700 701 static const struct netdev_stat_ops efx_stat_ops = { 702 .get_queue_stats_rx = efx_get_queue_stats_rx, 703 .get_queue_stats_tx = efx_get_queue_stats_tx, 704 .get_base_stats = efx_get_base_stats, 705 }; 706 707 static int efx_xdp_setup_prog(struct efx_nic *efx, struct bpf_prog *prog) 708 { 709 struct bpf_prog *old_prog; 710 711 if (efx->xdp_rxq_info_failed) { 712 netif_err(efx, drv, efx->net_dev, 713 "Unable to bind XDP program due to previous failure of rxq_info\n"); 714 return -EINVAL; 715 } 716 717 if (prog && efx->net_dev->mtu > efx_xdp_max_mtu(efx)) { 718 netif_err(efx, drv, efx->net_dev, 719 "Unable to configure XDP with MTU of %d (max: %d)\n", 720 efx->net_dev->mtu, efx_xdp_max_mtu(efx)); 721 return -EINVAL; 722 } 723 724 old_prog = rtnl_dereference(efx->xdp_prog); 725 rcu_assign_pointer(efx->xdp_prog, prog); 726 /* Release the reference that was originally passed by the caller. */ 727 if (old_prog) 728 bpf_prog_put(old_prog); 729 730 return 0; 731 } 732 733 /* Context: process, rtnl_lock() held. */ 734 static int efx_xdp(struct net_device *dev, struct netdev_bpf *xdp) 735 { 736 struct efx_nic *efx = efx_netdev_priv(dev); 737 738 switch (xdp->command) { 739 case XDP_SETUP_PROG: 740 return efx_xdp_setup_prog(efx, xdp->prog); 741 default: 742 return -EINVAL; 743 } 744 } 745 746 static int efx_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **xdpfs, 747 u32 flags) 748 { 749 struct efx_nic *efx = efx_netdev_priv(dev); 750 751 if (!netif_running(dev)) 752 return -EINVAL; 753 754 return efx_xdp_tx_buffers(efx, n, xdpfs, flags & XDP_XMIT_FLUSH); 755 } 756 757 static void efx_update_name(struct efx_nic *efx) 758 { 759 strcpy(efx->name, efx->net_dev->name); 760 efx_mtd_rename(efx); 761 efx_set_channel_names(efx); 762 } 763 764 static int efx_netdev_event(struct notifier_block *this, 765 unsigned long event, void *ptr) 766 { 767 struct net_device *net_dev = netdev_notifier_info_to_dev(ptr); 768 769 if ((net_dev->netdev_ops == &efx_netdev_ops) && 770 event == NETDEV_CHANGENAME) 771 efx_update_name(efx_netdev_priv(net_dev)); 772 773 return NOTIFY_DONE; 774 } 775 776 static struct notifier_block efx_netdev_notifier = { 777 .notifier_call = efx_netdev_event, 778 }; 779 780 static ssize_t phy_type_show(struct device *dev, 781 struct device_attribute *attr, char *buf) 782 { 783 struct efx_nic *efx = dev_get_drvdata(dev); 784 return sprintf(buf, "%d\n", efx->phy_type); 785 } 786 static DEVICE_ATTR_RO(phy_type); 787 788 static int efx_register_netdev(struct efx_nic *efx) 789 { 790 struct net_device *net_dev = efx->net_dev; 791 struct efx_channel *channel; 792 int rc; 793 794 net_dev->watchdog_timeo = 5 * HZ; 795 net_dev->irq = efx->pci_dev->irq; 796 net_dev->netdev_ops = &efx_netdev_ops; 797 net_dev->stat_ops = &efx_stat_ops; 798 if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0) 799 net_dev->priv_flags |= IFF_UNICAST_FLT; 800 net_dev->ethtool_ops = &efx_ethtool_ops; 801 netif_set_tso_max_segs(net_dev, EFX_TSO_MAX_SEGS); 802 net_dev->min_mtu = EFX_MIN_MTU; 803 net_dev->max_mtu = EFX_MAX_MTU; 804 805 rtnl_lock(); 806 807 /* Enable resets to be scheduled and check whether any were 808 * already requested. If so, the NIC is probably hosed so we 809 * abort. 810 */ 811 if (efx->reset_pending) { 812 pci_err(efx->pci_dev, "aborting probe due to scheduled reset\n"); 813 rc = -EIO; 814 goto fail_locked; 815 } 816 817 rc = dev_alloc_name(net_dev, net_dev->name); 818 if (rc < 0) 819 goto fail_locked; 820 efx_update_name(efx); 821 822 /* Always start with carrier off; PHY events will detect the link */ 823 netif_carrier_off(net_dev); 824 825 rc = register_netdevice(net_dev); 826 if (rc) 827 goto fail_locked; 828 829 efx_for_each_channel(channel, efx) { 830 struct efx_tx_queue *tx_queue; 831 efx_for_each_channel_tx_queue(tx_queue, channel) 832 efx_init_tx_queue_core_txq(tx_queue); 833 } 834 835 efx_associate(efx); 836 837 efx->state = STATE_NET_DOWN; 838 839 rtnl_unlock(); 840 841 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type); 842 if (rc) { 843 netif_err(efx, drv, efx->net_dev, 844 "failed to init net dev attributes\n"); 845 goto fail_registered; 846 } 847 848 efx_init_mcdi_logging(efx); 849 850 return 0; 851 852 fail_registered: 853 rtnl_lock(); 854 efx_dissociate(efx); 855 unregister_netdevice(net_dev); 856 fail_locked: 857 efx->state = STATE_UNINIT; 858 rtnl_unlock(); 859 netif_err(efx, drv, efx->net_dev, "could not register net dev\n"); 860 return rc; 861 } 862 863 static void efx_unregister_netdev(struct efx_nic *efx) 864 { 865 if (!efx->net_dev) 866 return; 867 868 if (WARN_ON(efx_netdev_priv(efx->net_dev) != efx)) 869 return; 870 871 if (efx_dev_registered(efx)) { 872 strscpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name)); 873 efx_fini_mcdi_logging(efx); 874 device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 875 unregister_netdev(efx->net_dev); 876 } 877 } 878 879 /************************************************************************** 880 * 881 * List of NICs we support 882 * 883 **************************************************************************/ 884 885 /* PCI device ID table */ 886 static const struct pci_device_id efx_pci_table[] = { 887 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0903), /* SFC9120 PF */ 888 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 889 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1903), /* SFC9120 VF */ 890 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 891 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0923), /* SFC9140 PF */ 892 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 893 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1923), /* SFC9140 VF */ 894 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 895 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0a03), /* SFC9220 PF */ 896 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 897 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1a03), /* SFC9220 VF */ 898 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 899 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0b03), /* SFC9250 PF */ 900 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 901 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1b03), /* SFC9250 VF */ 902 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 903 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0c03), /* X4 PF (FF/LL) */ 904 .driver_data = (unsigned long)&efx_x4_nic_type}, 905 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x2c03), /* X4 PF (FF only) */ 906 .driver_data = (unsigned long)&efx_x4_nic_type}, 907 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x8c03), /* X4D PF (FF/LL) */ 908 .driver_data = (unsigned long)&efx_x4_nic_type}, 909 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0xac03), /* X4D PF (FF only) */ 910 .driver_data = (unsigned long)&efx_x4_nic_type}, 911 {0} /* end of list */ 912 }; 913 914 /************************************************************************** 915 * 916 * Data housekeeping 917 * 918 **************************************************************************/ 919 920 void efx_update_sw_stats(struct efx_nic *efx, u64 *stats) 921 { 922 u64 n_rx_nodesc_trunc = 0; 923 struct efx_channel *channel; 924 925 efx_for_each_channel(channel, efx) 926 n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc; 927 stats[GENERIC_STAT_rx_nodesc_trunc] = n_rx_nodesc_trunc; 928 stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops); 929 } 930 931 /************************************************************************** 932 * 933 * PCI interface 934 * 935 **************************************************************************/ 936 937 /* Main body of final NIC shutdown code 938 * This is called only at module unload (or hotplug removal). 939 */ 940 static void efx_pci_remove_main(struct efx_nic *efx) 941 { 942 /* Flush reset_work. It can no longer be scheduled since we 943 * are not READY. 944 */ 945 WARN_ON(efx_net_active(efx->state)); 946 efx_flush_reset_workqueue(efx); 947 948 efx_disable_interrupts(efx); 949 efx_clear_interrupt_affinity(efx); 950 efx_nic_fini_interrupt(efx); 951 efx_fini_port(efx); 952 efx->type->fini(efx); 953 efx_fini_napi(efx); 954 efx_remove_all(efx); 955 } 956 957 /* Final NIC shutdown 958 * This is called only at module unload (or hotplug removal). A PF can call 959 * this on its VFs to ensure they are unbound first. 960 */ 961 static void efx_pci_remove(struct pci_dev *pci_dev) 962 { 963 struct efx_probe_data *probe_data; 964 struct efx_nic *efx; 965 966 efx = pci_get_drvdata(pci_dev); 967 if (!efx) 968 return; 969 970 /* Mark the NIC as fini, then stop the interface */ 971 rtnl_lock(); 972 efx_dissociate(efx); 973 dev_close(efx->net_dev); 974 efx_disable_interrupts(efx); 975 efx->state = STATE_UNINIT; 976 rtnl_unlock(); 977 978 if (efx->type->sriov_fini) 979 efx->type->sriov_fini(efx); 980 981 efx_fini_devlink_lock(efx); 982 efx_unregister_netdev(efx); 983 984 efx_mtd_remove(efx); 985 986 efx_pci_remove_main(efx); 987 988 efx_fini_io(efx); 989 990 probe_data = container_of(efx, struct efx_probe_data, efx); 991 efx_cxl_exit(probe_data); 992 993 pci_dbg(efx->pci_dev, "shutdown successful\n"); 994 995 efx_fini_devlink_and_unlock(efx); 996 efx_fini_struct(efx); 997 free_netdev(efx->net_dev); 998 kfree(probe_data); 999 }; 1000 1001 /* NIC VPD information 1002 * Called during probe to display the part number of the 1003 * installed NIC. 1004 */ 1005 static void efx_probe_vpd_strings(struct efx_nic *efx) 1006 { 1007 struct pci_dev *dev = efx->pci_dev; 1008 unsigned int vpd_size, kw_len; 1009 u8 *vpd_data; 1010 int start; 1011 1012 vpd_data = pci_vpd_alloc(dev, &vpd_size); 1013 if (IS_ERR(vpd_data)) { 1014 pci_warn(dev, "Unable to read VPD\n"); 1015 return; 1016 } 1017 1018 start = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, 1019 PCI_VPD_RO_KEYWORD_PARTNO, &kw_len); 1020 if (start < 0) 1021 pci_err(dev, "Part number not found or incomplete\n"); 1022 else 1023 pci_info(dev, "Part Number : %.*s\n", kw_len, vpd_data + start); 1024 1025 start = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, 1026 PCI_VPD_RO_KEYWORD_SERIALNO, &kw_len); 1027 if (start < 0) 1028 pci_err(dev, "Serial number not found or incomplete\n"); 1029 else 1030 efx->vpd_sn = kmemdup_nul(vpd_data + start, kw_len, GFP_KERNEL); 1031 1032 kfree(vpd_data); 1033 } 1034 1035 1036 /* Main body of NIC initialisation 1037 * This is called at module load (or hotplug insertion, theoretically). 1038 */ 1039 static int efx_pci_probe_main(struct efx_nic *efx) 1040 { 1041 int rc; 1042 1043 /* Do start-of-day initialisation */ 1044 rc = efx_probe_all(efx); 1045 if (rc) 1046 goto fail1; 1047 1048 efx_init_napi(efx); 1049 1050 down_write(&efx->filter_sem); 1051 rc = efx->type->init(efx); 1052 up_write(&efx->filter_sem); 1053 if (rc) { 1054 pci_err(efx->pci_dev, "failed to initialise NIC\n"); 1055 goto fail3; 1056 } 1057 1058 rc = efx_init_port(efx); 1059 if (rc) { 1060 netif_err(efx, probe, efx->net_dev, 1061 "failed to initialise port\n"); 1062 goto fail4; 1063 } 1064 1065 rc = efx_nic_init_interrupt(efx); 1066 if (rc) 1067 goto fail5; 1068 1069 efx_set_interrupt_affinity(efx); 1070 rc = efx_enable_interrupts(efx); 1071 if (rc) 1072 goto fail6; 1073 1074 return 0; 1075 1076 fail6: 1077 efx_clear_interrupt_affinity(efx); 1078 efx_nic_fini_interrupt(efx); 1079 fail5: 1080 efx_fini_port(efx); 1081 fail4: 1082 efx->type->fini(efx); 1083 fail3: 1084 efx_fini_napi(efx); 1085 efx_remove_all(efx); 1086 fail1: 1087 return rc; 1088 } 1089 1090 static int efx_pci_probe_post_io(struct efx_nic *efx) 1091 { 1092 struct net_device *net_dev = efx->net_dev; 1093 int rc = efx_pci_probe_main(efx); 1094 1095 if (rc) 1096 return rc; 1097 1098 if (efx->type->sriov_init) { 1099 rc = efx->type->sriov_init(efx); 1100 if (rc) 1101 pci_err(efx->pci_dev, "SR-IOV can't be enabled rc %d\n", 1102 rc); 1103 } 1104 1105 /* Determine netdevice features */ 1106 net_dev->features |= efx->type->offload_features; 1107 1108 /* Add TSO features */ 1109 if (efx->type->tso_versions && efx->type->tso_versions(efx)) 1110 net_dev->features |= NETIF_F_TSO | NETIF_F_TSO6; 1111 1112 /* Mask for features that also apply to VLAN devices */ 1113 net_dev->vlan_features |= (NETIF_F_HW_CSUM | NETIF_F_SG | 1114 NETIF_F_HIGHDMA | NETIF_F_ALL_TSO | 1115 NETIF_F_RXCSUM); 1116 1117 /* Determine user configurable features */ 1118 net_dev->hw_features |= net_dev->features & ~efx->fixed_features; 1119 1120 /* Disable receiving frames with bad FCS, by default. */ 1121 net_dev->features &= ~NETIF_F_RXALL; 1122 1123 /* Disable VLAN filtering by default. It may be enforced if 1124 * the feature is fixed (i.e. VLAN filters are required to 1125 * receive VLAN tagged packets due to vPort restrictions). 1126 */ 1127 net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 1128 net_dev->features |= efx->fixed_features; 1129 1130 net_dev->xdp_features = NETDEV_XDP_ACT_BASIC | 1131 NETDEV_XDP_ACT_REDIRECT | 1132 NETDEV_XDP_ACT_NDO_XMIT; 1133 1134 /* devlink creation, registration and lock */ 1135 rc = efx_probe_devlink_and_lock(efx); 1136 if (rc) 1137 pci_err(efx->pci_dev, "devlink registration failed"); 1138 1139 rc = efx_register_netdev(efx); 1140 efx_probe_devlink_unlock(efx); 1141 if (!rc) 1142 return 0; 1143 1144 efx_pci_remove_main(efx); 1145 return rc; 1146 } 1147 1148 /* NIC initialisation 1149 * 1150 * This is called at module load (or hotplug insertion, 1151 * theoretically). It sets up PCI mappings, resets the NIC, 1152 * sets up and registers the network devices with the kernel and hooks 1153 * the interrupt service routine. It does not prepare the device for 1154 * transmission; this is left to the first time one of the network 1155 * interfaces is brought up (i.e. efx_net_open). 1156 */ 1157 static int efx_pci_probe(struct pci_dev *pci_dev, 1158 const struct pci_device_id *entry) 1159 { 1160 struct efx_probe_data *probe_data, **probe_ptr; 1161 struct net_device *net_dev; 1162 struct efx_nic *efx; 1163 int rc; 1164 1165 /* Allocate probe data and struct efx_nic */ 1166 probe_data = kzalloc_obj(*probe_data); 1167 if (!probe_data) 1168 return -ENOMEM; 1169 probe_data->pci_dev = pci_dev; 1170 efx = &probe_data->efx; 1171 1172 /* Allocate and initialise a struct net_device */ 1173 net_dev = alloc_etherdev_mq(sizeof(probe_data), EFX_MAX_CORE_TX_QUEUES); 1174 if (!net_dev) { 1175 rc = -ENOMEM; 1176 goto fail0; 1177 } 1178 probe_ptr = netdev_priv(net_dev); 1179 *probe_ptr = probe_data; 1180 efx->net_dev = net_dev; 1181 efx->type = (const struct efx_nic_type *) entry->driver_data; 1182 efx->fixed_features |= NETIF_F_HIGHDMA; 1183 1184 pci_set_drvdata(pci_dev, efx); 1185 SET_NETDEV_DEV(net_dev, &pci_dev->dev); 1186 rc = efx_init_struct(efx, pci_dev); 1187 if (rc) 1188 goto fail1; 1189 1190 pci_info(pci_dev, "Solarflare NIC detected\n"); 1191 1192 if (!efx->type->is_vf) 1193 efx_probe_vpd_strings(efx); 1194 1195 /* Set up basic I/O (BAR mappings etc) */ 1196 rc = efx_init_io(efx, efx->type->mem_bar(efx), efx->type->max_dma_mask, 1197 efx->type->mem_map_size(efx)); 1198 if (rc) 1199 goto fail2; 1200 1201 /* A successful cxl initialization implies a CXL region created to be 1202 * used for PIO buffers. If there is no CXL support legacy PIO buffers 1203 * defined at specific PCI BAR regions will be used. If there is CXL 1204 * support and the cxl initialization fails, the driver probe fails. 1205 */ 1206 rc = efx_cxl_init(probe_data); 1207 if (rc) { 1208 pci_err(pci_dev, "CXL initialization failed with error %d\n", rc); 1209 goto fail3; 1210 } 1211 1212 rc = efx_pci_probe_post_io(efx); 1213 if (rc) { 1214 /* On failure, retry once immediately. 1215 * If we aborted probe due to a scheduled reset, dismiss it. 1216 */ 1217 efx->reset_pending = 0; 1218 rc = efx_pci_probe_post_io(efx); 1219 if (rc) { 1220 /* On another failure, retry once more 1221 * after a 50-305ms delay. 1222 */ 1223 unsigned char r; 1224 1225 get_random_bytes(&r, 1); 1226 msleep((unsigned int)r + 50); 1227 efx->reset_pending = 0; 1228 rc = efx_pci_probe_post_io(efx); 1229 } 1230 } 1231 if (rc) 1232 goto fail3; 1233 1234 netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n"); 1235 1236 /* Try to create MTDs, but allow this to fail */ 1237 rtnl_lock(); 1238 rc = efx_mtd_probe(efx); 1239 rtnl_unlock(); 1240 if (rc && rc != -EPERM) 1241 netif_warn(efx, probe, efx->net_dev, 1242 "failed to create MTDs (%d)\n", rc); 1243 1244 if (efx->type->udp_tnl_push_ports) 1245 efx->type->udp_tnl_push_ports(efx); 1246 1247 return 0; 1248 1249 fail3: 1250 efx_cxl_exit(probe_data); 1251 efx_fini_io(efx); 1252 fail2: 1253 efx_fini_struct(efx); 1254 fail1: 1255 WARN_ON(rc > 0); 1256 netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc); 1257 free_netdev(net_dev); 1258 fail0: 1259 kfree(probe_data); 1260 return rc; 1261 } 1262 1263 /* efx_pci_sriov_configure returns the actual number of Virtual Functions 1264 * enabled on success 1265 */ 1266 #ifdef CONFIG_SFC_SRIOV 1267 static int efx_pci_sriov_configure(struct pci_dev *dev, int num_vfs) 1268 { 1269 int rc; 1270 struct efx_nic *efx = pci_get_drvdata(dev); 1271 1272 if (efx->type->sriov_configure) { 1273 rc = efx->type->sriov_configure(efx, num_vfs); 1274 if (rc) 1275 return rc; 1276 else 1277 return num_vfs; 1278 } else 1279 return -EOPNOTSUPP; 1280 } 1281 #endif 1282 1283 static int efx_pm_freeze(struct device *dev) 1284 { 1285 struct efx_nic *efx = dev_get_drvdata(dev); 1286 1287 rtnl_lock(); 1288 1289 if (efx_net_active(efx->state)) { 1290 efx_device_detach_sync(efx); 1291 1292 efx_stop_all(efx); 1293 efx_disable_interrupts(efx); 1294 1295 efx->state = efx_freeze(efx->state); 1296 } 1297 1298 rtnl_unlock(); 1299 1300 return 0; 1301 } 1302 1303 static void efx_pci_shutdown(struct pci_dev *pci_dev) 1304 { 1305 struct efx_nic *efx = pci_get_drvdata(pci_dev); 1306 1307 if (!efx) 1308 return; 1309 1310 efx_pm_freeze(&pci_dev->dev); 1311 pci_disable_device(pci_dev); 1312 } 1313 1314 static int efx_pm_thaw(struct device *dev) 1315 { 1316 int rc; 1317 struct efx_nic *efx = dev_get_drvdata(dev); 1318 1319 rtnl_lock(); 1320 1321 if (efx_frozen(efx->state)) { 1322 rc = efx_enable_interrupts(efx); 1323 if (rc) 1324 goto fail; 1325 1326 mutex_lock(&efx->mac_lock); 1327 efx_mcdi_port_reconfigure(efx); 1328 mutex_unlock(&efx->mac_lock); 1329 1330 efx_start_all(efx); 1331 1332 efx_device_attach_if_not_resetting(efx); 1333 1334 efx->state = efx_thaw(efx->state); 1335 1336 efx->type->resume_wol(efx); 1337 } 1338 1339 rtnl_unlock(); 1340 1341 /* Reschedule any quenched resets scheduled during efx_pm_freeze() */ 1342 efx_queue_reset_work(efx); 1343 1344 return 0; 1345 1346 fail: 1347 rtnl_unlock(); 1348 1349 return rc; 1350 } 1351 1352 static int efx_pm_poweroff(struct device *dev) 1353 { 1354 struct pci_dev *pci_dev = to_pci_dev(dev); 1355 struct efx_nic *efx = pci_get_drvdata(pci_dev); 1356 1357 efx->type->fini(efx); 1358 1359 efx->reset_pending = 0; 1360 1361 pci_save_state(pci_dev); 1362 return pci_set_power_state(pci_dev, PCI_D3hot); 1363 } 1364 1365 /* Used for both resume and restore */ 1366 static int efx_pm_resume(struct device *dev) 1367 { 1368 struct pci_dev *pci_dev = to_pci_dev(dev); 1369 struct efx_nic *efx = pci_get_drvdata(pci_dev); 1370 int rc; 1371 1372 rc = pci_set_power_state(pci_dev, PCI_D0); 1373 if (rc) 1374 return rc; 1375 pci_restore_state(pci_dev); 1376 rc = pci_enable_device(pci_dev); 1377 if (rc) 1378 return rc; 1379 pci_set_master(efx->pci_dev); 1380 rc = efx->type->reset(efx, RESET_TYPE_ALL); 1381 if (rc) 1382 return rc; 1383 down_write(&efx->filter_sem); 1384 rc = efx->type->init(efx); 1385 up_write(&efx->filter_sem); 1386 if (rc) 1387 return rc; 1388 rc = efx_pm_thaw(dev); 1389 return rc; 1390 } 1391 1392 static int efx_pm_suspend(struct device *dev) 1393 { 1394 int rc; 1395 1396 efx_pm_freeze(dev); 1397 rc = efx_pm_poweroff(dev); 1398 if (rc) 1399 efx_pm_resume(dev); 1400 return rc; 1401 } 1402 1403 static const struct dev_pm_ops efx_pm_ops = { 1404 .suspend = efx_pm_suspend, 1405 .resume = efx_pm_resume, 1406 .freeze = efx_pm_freeze, 1407 .thaw = efx_pm_thaw, 1408 .poweroff = efx_pm_poweroff, 1409 .restore = efx_pm_resume, 1410 }; 1411 1412 static struct pci_driver efx_pci_driver = { 1413 .name = KBUILD_MODNAME, 1414 .id_table = efx_pci_table, 1415 .probe = efx_pci_probe, 1416 .remove = efx_pci_remove, 1417 .driver.pm = &efx_pm_ops, 1418 .shutdown = efx_pci_shutdown, 1419 .err_handler = &efx_err_handlers, 1420 #ifdef CONFIG_SFC_SRIOV 1421 .sriov_configure = efx_pci_sriov_configure, 1422 #endif 1423 }; 1424 1425 /************************************************************************** 1426 * 1427 * Kernel module interface 1428 * 1429 *************************************************************************/ 1430 1431 static int __init efx_init_module(void) 1432 { 1433 int rc; 1434 1435 printk(KERN_INFO "Solarflare NET driver\n"); 1436 1437 rc = register_netdevice_notifier(&efx_netdev_notifier); 1438 if (rc) 1439 goto err_notifier; 1440 1441 rc = efx_create_reset_workqueue(); 1442 if (rc) 1443 goto err_reset; 1444 1445 rc = pci_register_driver(&efx_pci_driver); 1446 if (rc < 0) 1447 goto err_pci; 1448 1449 rc = pci_register_driver(&ef100_pci_driver); 1450 if (rc < 0) 1451 goto err_pci_ef100; 1452 1453 return 0; 1454 1455 err_pci_ef100: 1456 pci_unregister_driver(&efx_pci_driver); 1457 err_pci: 1458 efx_destroy_reset_workqueue(); 1459 err_reset: 1460 unregister_netdevice_notifier(&efx_netdev_notifier); 1461 err_notifier: 1462 return rc; 1463 } 1464 1465 static void __exit efx_exit_module(void) 1466 { 1467 printk(KERN_INFO "Solarflare NET driver unloading\n"); 1468 1469 pci_unregister_driver(&ef100_pci_driver); 1470 pci_unregister_driver(&efx_pci_driver); 1471 efx_destroy_reset_workqueue(); 1472 unregister_netdevice_notifier(&efx_netdev_notifier); 1473 1474 } 1475 1476 module_init(efx_init_module); 1477 module_exit(efx_exit_module); 1478 1479 MODULE_AUTHOR("Solarflare Communications and " 1480 "Michael Brown <mbrown@fensystems.co.uk>"); 1481 MODULE_DESCRIPTION("Solarflare network driver"); 1482 MODULE_LICENSE("GPL"); 1483 MODULE_DEVICE_TABLE(pci, efx_pci_table); 1484