1 // SPDX-License-Identifier: GPL-2.0-only 2 /**************************************************************************** 3 * Driver for Solarflare network controllers and boards 4 * Copyright 2018 Solarflare Communications Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation, incorporated herein by reference. 9 */ 10 11 #include "net_driver.h" 12 #include <linux/module.h> 13 #include <linux/filter.h> 14 #include "efx_channels.h" 15 #include "efx.h" 16 #include "efx_common.h" 17 #include "tx_common.h" 18 #include "rx_common.h" 19 #include "nic.h" 20 #include "sriov.h" 21 #include "workarounds.h" 22 23 /* This is the first interrupt mode to try out of: 24 * 0 => MSI-X 25 * 1 => MSI 26 * 2 => legacy 27 */ 28 unsigned int efx_siena_interrupt_mode = EFX_INT_MODE_MSIX; 29 30 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS), 31 * i.e. the number of CPUs among which we may distribute simultaneous 32 * interrupt handling. 33 * 34 * Cards without MSI-X will only target one CPU via legacy or MSI interrupt. 35 * The default (0) means to assign an interrupt to each core. 36 */ 37 unsigned int efx_siena_rss_cpus; 38 39 static unsigned int irq_adapt_low_thresh = 8000; 40 module_param(irq_adapt_low_thresh, uint, 0644); 41 MODULE_PARM_DESC(irq_adapt_low_thresh, 42 "Threshold score for reducing IRQ moderation"); 43 44 static unsigned int irq_adapt_high_thresh = 16000; 45 module_param(irq_adapt_high_thresh, uint, 0644); 46 MODULE_PARM_DESC(irq_adapt_high_thresh, 47 "Threshold score for increasing IRQ moderation"); 48 49 static const struct efx_channel_type efx_default_channel_type; 50 51 /************* 52 * INTERRUPTS 53 *************/ 54 55 static unsigned int count_online_cores(struct efx_nic *efx, bool local_node) 56 { 57 cpumask_var_t filter_mask; 58 unsigned int count; 59 int cpu; 60 61 if (unlikely(!zalloc_cpumask_var(&filter_mask, GFP_KERNEL))) { 62 netif_warn(efx, probe, efx->net_dev, 63 "RSS disabled due to allocation failure\n"); 64 return 1; 65 } 66 67 cpumask_copy(filter_mask, cpu_online_mask); 68 if (local_node) 69 cpumask_and(filter_mask, filter_mask, 70 cpumask_of_pcibus(efx->pci_dev->bus)); 71 72 count = 0; 73 for_each_cpu(cpu, filter_mask) { 74 ++count; 75 cpumask_andnot(filter_mask, filter_mask, topology_sibling_cpumask(cpu)); 76 } 77 78 free_cpumask_var(filter_mask); 79 80 return count; 81 } 82 83 static unsigned int efx_wanted_parallelism(struct efx_nic *efx) 84 { 85 unsigned int count; 86 87 if (efx_siena_rss_cpus) { 88 count = efx_siena_rss_cpus; 89 } else { 90 count = count_online_cores(efx, true); 91 92 /* If no online CPUs in local node, fallback to any online CPUs */ 93 if (count == 0) 94 count = count_online_cores(efx, false); 95 } 96 97 if (count > EFX_MAX_RX_QUEUES) { 98 netif_cond_dbg(efx, probe, efx->net_dev, !efx_siena_rss_cpus, 99 warn, 100 "Reducing number of rx queues from %u to %u.\n", 101 count, EFX_MAX_RX_QUEUES); 102 count = EFX_MAX_RX_QUEUES; 103 } 104 105 /* If RSS is requested for the PF *and* VFs then we can't write RSS 106 * table entries that are inaccessible to VFs 107 */ 108 #ifdef CONFIG_SFC_SIENA_SRIOV 109 if (efx->type->sriov_wanted) { 110 if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 && 111 count > efx_vf_size(efx)) { 112 netif_warn(efx, probe, efx->net_dev, 113 "Reducing number of RSS channels from %u to %u for " 114 "VF support. Increase vf-msix-limit to use more " 115 "channels on the PF.\n", 116 count, efx_vf_size(efx)); 117 count = efx_vf_size(efx); 118 } 119 } 120 #endif 121 122 return count; 123 } 124 125 static int efx_allocate_msix_channels(struct efx_nic *efx, 126 unsigned int max_channels, 127 unsigned int extra_channels, 128 unsigned int parallelism) 129 { 130 unsigned int n_channels = parallelism; 131 int vec_count; 132 int tx_per_ev; 133 int n_xdp_tx; 134 int n_xdp_ev; 135 136 if (efx_siena_separate_tx_channels) 137 n_channels *= 2; 138 n_channels += extra_channels; 139 140 /* To allow XDP transmit to happen from arbitrary NAPI contexts 141 * we allocate a TX queue per CPU. We share event queues across 142 * multiple tx queues, assuming tx and ev queues are both 143 * maximum size. 144 */ 145 tx_per_ev = EFX_MAX_EVQ_SIZE / EFX_TXQ_MAX_ENT(efx); 146 tx_per_ev = min(tx_per_ev, EFX_MAX_TXQ_PER_CHANNEL); 147 n_xdp_tx = num_possible_cpus(); 148 n_xdp_ev = DIV_ROUND_UP(n_xdp_tx, tx_per_ev); 149 150 vec_count = pci_msix_vec_count(efx->pci_dev); 151 if (vec_count < 0) 152 return vec_count; 153 154 max_channels = min_t(unsigned int, vec_count, max_channels); 155 156 /* Check resources. 157 * We need a channel per event queue, plus a VI per tx queue. 158 * This may be more pessimistic than it needs to be. 159 */ 160 if (n_channels >= max_channels) { 161 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED; 162 netif_warn(efx, drv, efx->net_dev, 163 "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n", 164 n_xdp_ev, n_channels, max_channels); 165 netif_warn(efx, drv, efx->net_dev, 166 "XDP_TX and XDP_REDIRECT might decrease device's performance\n"); 167 } else if (n_channels + n_xdp_tx > efx->max_vis) { 168 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED; 169 netif_warn(efx, drv, efx->net_dev, 170 "Insufficient resources for %d XDP TX queues (%d other channels, max VIs %d)\n", 171 n_xdp_tx, n_channels, efx->max_vis); 172 netif_warn(efx, drv, efx->net_dev, 173 "XDP_TX and XDP_REDIRECT might decrease device's performance\n"); 174 } else if (n_channels + n_xdp_ev > max_channels) { 175 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_SHARED; 176 netif_warn(efx, drv, efx->net_dev, 177 "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n", 178 n_xdp_ev, n_channels, max_channels); 179 180 n_xdp_ev = max_channels - n_channels; 181 netif_warn(efx, drv, efx->net_dev, 182 "XDP_TX and XDP_REDIRECT will work with reduced performance (%d cpus/tx_queue)\n", 183 DIV_ROUND_UP(n_xdp_tx, tx_per_ev * n_xdp_ev)); 184 } else { 185 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_DEDICATED; 186 } 187 188 if (efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_BORROWED) { 189 efx->n_xdp_channels = n_xdp_ev; 190 efx->xdp_tx_per_channel = tx_per_ev; 191 efx->xdp_tx_queue_count = n_xdp_tx; 192 n_channels += n_xdp_ev; 193 netif_dbg(efx, drv, efx->net_dev, 194 "Allocating %d TX and %d event queues for XDP\n", 195 n_xdp_ev * tx_per_ev, n_xdp_ev); 196 } else { 197 efx->n_xdp_channels = 0; 198 efx->xdp_tx_per_channel = 0; 199 efx->xdp_tx_queue_count = n_xdp_tx; 200 } 201 202 if (vec_count < n_channels) { 203 netif_err(efx, drv, efx->net_dev, 204 "WARNING: Insufficient MSI-X vectors available (%d < %u).\n", 205 vec_count, n_channels); 206 netif_err(efx, drv, efx->net_dev, 207 "WARNING: Performance may be reduced.\n"); 208 n_channels = vec_count; 209 } 210 211 n_channels = min(n_channels, max_channels); 212 213 efx->n_channels = n_channels; 214 215 /* Ignore XDP tx channels when creating rx channels. */ 216 n_channels -= efx->n_xdp_channels; 217 218 if (efx_siena_separate_tx_channels) { 219 efx->n_tx_channels = 220 min(max(n_channels / 2, 1U), 221 efx->max_tx_channels); 222 efx->tx_channel_offset = 223 n_channels - efx->n_tx_channels; 224 efx->n_rx_channels = 225 max(n_channels - 226 efx->n_tx_channels, 1U); 227 } else { 228 efx->n_tx_channels = min(n_channels, efx->max_tx_channels); 229 efx->tx_channel_offset = 0; 230 efx->n_rx_channels = n_channels; 231 } 232 233 efx->n_rx_channels = min(efx->n_rx_channels, parallelism); 234 efx->n_tx_channels = min(efx->n_tx_channels, parallelism); 235 236 efx->xdp_channel_offset = n_channels; 237 238 netif_dbg(efx, drv, efx->net_dev, 239 "Allocating %u RX channels\n", 240 efx->n_rx_channels); 241 242 return efx->n_channels; 243 } 244 245 /* Probe the number and type of interrupts we are able to obtain, and 246 * the resulting numbers of channels and RX queues. 247 */ 248 int efx_siena_probe_interrupts(struct efx_nic *efx) 249 { 250 unsigned int extra_channels = 0; 251 unsigned int rss_spread; 252 unsigned int i, j; 253 int rc; 254 255 for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) 256 if (efx->extra_channel_type[i]) 257 ++extra_channels; 258 259 if (efx->interrupt_mode == EFX_INT_MODE_MSIX) { 260 unsigned int parallelism = efx_wanted_parallelism(efx); 261 struct msix_entry xentries[EFX_MAX_CHANNELS]; 262 unsigned int n_channels; 263 264 rc = efx_allocate_msix_channels(efx, efx->max_channels, 265 extra_channels, parallelism); 266 if (rc >= 0) { 267 n_channels = rc; 268 for (i = 0; i < n_channels; i++) 269 xentries[i].entry = i; 270 rc = pci_enable_msix_range(efx->pci_dev, xentries, 1, 271 n_channels); 272 } 273 if (rc < 0) { 274 /* Fall back to single channel MSI */ 275 netif_err(efx, drv, efx->net_dev, 276 "could not enable MSI-X\n"); 277 if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI) 278 efx->interrupt_mode = EFX_INT_MODE_MSI; 279 else 280 return rc; 281 } else if (rc < n_channels) { 282 netif_err(efx, drv, efx->net_dev, 283 "WARNING: Insufficient MSI-X vectors" 284 " available (%d < %u).\n", rc, n_channels); 285 netif_err(efx, drv, efx->net_dev, 286 "WARNING: Performance may be reduced.\n"); 287 n_channels = rc; 288 } 289 290 if (rc > 0) { 291 for (i = 0; i < efx->n_channels; i++) 292 efx_get_channel(efx, i)->irq = 293 xentries[i].vector; 294 } 295 } 296 297 /* Try single interrupt MSI */ 298 if (efx->interrupt_mode == EFX_INT_MODE_MSI) { 299 efx->n_channels = 1; 300 efx->n_rx_channels = 1; 301 efx->n_tx_channels = 1; 302 efx->tx_channel_offset = 0; 303 efx->n_xdp_channels = 0; 304 efx->xdp_channel_offset = efx->n_channels; 305 rc = pci_enable_msi(efx->pci_dev); 306 if (rc == 0) { 307 efx_get_channel(efx, 0)->irq = efx->pci_dev->irq; 308 } else { 309 netif_err(efx, drv, efx->net_dev, 310 "could not enable MSI\n"); 311 if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY) 312 efx->interrupt_mode = EFX_INT_MODE_LEGACY; 313 else 314 return rc; 315 } 316 } 317 318 /* Assume legacy interrupts */ 319 if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) { 320 efx->n_channels = 1 + (efx_siena_separate_tx_channels ? 1 : 0); 321 efx->n_rx_channels = 1; 322 efx->n_tx_channels = 1; 323 efx->tx_channel_offset = 1; 324 efx->n_xdp_channels = 0; 325 efx->xdp_channel_offset = efx->n_channels; 326 efx->legacy_irq = efx->pci_dev->irq; 327 } 328 329 /* Assign extra channels if possible, before XDP channels */ 330 efx->n_extra_tx_channels = 0; 331 j = efx->xdp_channel_offset; 332 for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) { 333 if (!efx->extra_channel_type[i]) 334 continue; 335 if (j <= efx->tx_channel_offset + efx->n_tx_channels) { 336 efx->extra_channel_type[i]->handle_no_channel(efx); 337 } else { 338 --j; 339 efx_get_channel(efx, j)->type = 340 efx->extra_channel_type[i]; 341 if (efx_channel_has_tx_queues(efx_get_channel(efx, j))) 342 efx->n_extra_tx_channels++; 343 } 344 } 345 346 rss_spread = efx->n_rx_channels; 347 /* RSS might be usable on VFs even if it is disabled on the PF */ 348 #ifdef CONFIG_SFC_SIENA_SRIOV 349 if (efx->type->sriov_wanted) { 350 efx->rss_spread = ((rss_spread > 1 || 351 !efx->type->sriov_wanted(efx)) ? 352 rss_spread : efx_vf_size(efx)); 353 return 0; 354 } 355 #endif 356 efx->rss_spread = rss_spread; 357 358 return 0; 359 } 360 361 #if defined(CONFIG_SMP) 362 void efx_siena_set_interrupt_affinity(struct efx_nic *efx) 363 { 364 const struct cpumask *numa_mask = cpumask_of_pcibus(efx->pci_dev->bus); 365 struct efx_channel *channel; 366 unsigned int cpu; 367 368 /* If no online CPUs in local node, fallback to any online CPU */ 369 if (cpumask_first_and(cpu_online_mask, numa_mask) >= nr_cpu_ids) 370 numa_mask = cpu_online_mask; 371 372 cpu = -1; 373 efx_for_each_channel(channel, efx) { 374 cpu = cpumask_next_and(cpu, cpu_online_mask, numa_mask); 375 if (cpu >= nr_cpu_ids) 376 cpu = cpumask_first_and(cpu_online_mask, numa_mask); 377 irq_set_affinity_hint(channel->irq, cpumask_of(cpu)); 378 } 379 } 380 381 void efx_siena_clear_interrupt_affinity(struct efx_nic *efx) 382 { 383 struct efx_channel *channel; 384 385 efx_for_each_channel(channel, efx) 386 irq_set_affinity_hint(channel->irq, NULL); 387 } 388 #else 389 void 390 efx_siena_set_interrupt_affinity(struct efx_nic *efx __always_unused) 391 { 392 } 393 394 void 395 efx_siena_clear_interrupt_affinity(struct efx_nic *efx __always_unused) 396 { 397 } 398 #endif /* CONFIG_SMP */ 399 400 void efx_siena_remove_interrupts(struct efx_nic *efx) 401 { 402 struct efx_channel *channel; 403 404 /* Remove MSI/MSI-X interrupts */ 405 efx_for_each_channel(channel, efx) 406 channel->irq = 0; 407 pci_disable_msi(efx->pci_dev); 408 pci_disable_msix(efx->pci_dev); 409 410 /* Remove legacy interrupt */ 411 efx->legacy_irq = 0; 412 } 413 414 /*************** 415 * EVENT QUEUES 416 ***************/ 417 418 /* Create event queue 419 * Event queue memory allocations are done only once. If the channel 420 * is reset, the memory buffer will be reused; this guards against 421 * errors during channel reset and also simplifies interrupt handling. 422 */ 423 static int efx_probe_eventq(struct efx_channel *channel) 424 { 425 struct efx_nic *efx = channel->efx; 426 unsigned long entries; 427 428 netif_dbg(efx, probe, efx->net_dev, 429 "chan %d create event queue\n", channel->channel); 430 431 /* Build an event queue with room for one event per tx and rx buffer, 432 * plus some extra for link state events and MCDI completions. 433 */ 434 entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128); 435 EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE); 436 channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1; 437 438 return efx_nic_probe_eventq(channel); 439 } 440 441 /* Prepare channel's event queue */ 442 static int efx_init_eventq(struct efx_channel *channel) 443 { 444 struct efx_nic *efx = channel->efx; 445 int rc; 446 447 EFX_WARN_ON_PARANOID(channel->eventq_init); 448 449 netif_dbg(efx, drv, efx->net_dev, 450 "chan %d init event queue\n", channel->channel); 451 452 rc = efx_nic_init_eventq(channel); 453 if (rc == 0) { 454 efx->type->push_irq_moderation(channel); 455 channel->eventq_read_ptr = 0; 456 channel->eventq_init = true; 457 } 458 return rc; 459 } 460 461 /* Enable event queue processing and NAPI */ 462 void efx_siena_start_eventq(struct efx_channel *channel) 463 { 464 netif_dbg(channel->efx, ifup, channel->efx->net_dev, 465 "chan %d start event queue\n", channel->channel); 466 467 /* Make sure the NAPI handler sees the enabled flag set */ 468 channel->enabled = true; 469 smp_wmb(); 470 471 napi_enable(&channel->napi_str); 472 efx_nic_eventq_read_ack(channel); 473 } 474 475 /* Disable event queue processing and NAPI */ 476 void efx_siena_stop_eventq(struct efx_channel *channel) 477 { 478 if (!channel->enabled) 479 return; 480 481 napi_disable(&channel->napi_str); 482 channel->enabled = false; 483 } 484 485 static void efx_fini_eventq(struct efx_channel *channel) 486 { 487 if (!channel->eventq_init) 488 return; 489 490 netif_dbg(channel->efx, drv, channel->efx->net_dev, 491 "chan %d fini event queue\n", channel->channel); 492 493 efx_nic_fini_eventq(channel); 494 channel->eventq_init = false; 495 } 496 497 static void efx_remove_eventq(struct efx_channel *channel) 498 { 499 netif_dbg(channel->efx, drv, channel->efx->net_dev, 500 "chan %d remove event queue\n", channel->channel); 501 502 efx_nic_remove_eventq(channel); 503 } 504 505 /************************************************************************** 506 * 507 * Channel handling 508 * 509 *************************************************************************/ 510 511 #ifdef CONFIG_RFS_ACCEL 512 static void efx_filter_rfs_expire(struct work_struct *data) 513 { 514 struct delayed_work *dwork = to_delayed_work(data); 515 struct efx_channel *channel; 516 unsigned int time, quota; 517 518 channel = container_of(dwork, struct efx_channel, filter_work); 519 time = jiffies - channel->rfs_last_expiry; 520 quota = channel->rfs_filter_count * time / (30 * HZ); 521 if (quota >= 20 && __efx_siena_filter_rfs_expire(channel, 522 min(channel->rfs_filter_count, quota))) 523 channel->rfs_last_expiry += time; 524 /* Ensure we do more work eventually even if NAPI poll is not happening */ 525 schedule_delayed_work(dwork, 30 * HZ); 526 } 527 #endif 528 529 /* Allocate and initialise a channel structure. */ 530 static struct efx_channel *efx_alloc_channel(struct efx_nic *efx, int i) 531 { 532 struct efx_rx_queue *rx_queue; 533 struct efx_tx_queue *tx_queue; 534 struct efx_channel *channel; 535 int j; 536 537 channel = kzalloc(sizeof(*channel), GFP_KERNEL); 538 if (!channel) 539 return NULL; 540 541 channel->efx = efx; 542 channel->channel = i; 543 channel->type = &efx_default_channel_type; 544 545 for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) { 546 tx_queue = &channel->tx_queue[j]; 547 tx_queue->efx = efx; 548 tx_queue->queue = -1; 549 tx_queue->label = j; 550 tx_queue->channel = channel; 551 } 552 553 #ifdef CONFIG_RFS_ACCEL 554 INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire); 555 #endif 556 557 rx_queue = &channel->rx_queue; 558 rx_queue->efx = efx; 559 timer_setup(&rx_queue->slow_fill, efx_siena_rx_slow_fill, 0); 560 561 return channel; 562 } 563 564 int efx_siena_init_channels(struct efx_nic *efx) 565 { 566 unsigned int i; 567 568 for (i = 0; i < EFX_MAX_CHANNELS; i++) { 569 efx->channel[i] = efx_alloc_channel(efx, i); 570 if (!efx->channel[i]) 571 return -ENOMEM; 572 efx->msi_context[i].efx = efx; 573 efx->msi_context[i].index = i; 574 } 575 576 /* Higher numbered interrupt modes are less capable! */ 577 efx->interrupt_mode = min(efx->type->min_interrupt_mode, 578 efx_siena_interrupt_mode); 579 580 efx->max_channels = EFX_MAX_CHANNELS; 581 efx->max_tx_channels = EFX_MAX_CHANNELS; 582 583 return 0; 584 } 585 586 void efx_siena_fini_channels(struct efx_nic *efx) 587 { 588 unsigned int i; 589 590 for (i = 0; i < EFX_MAX_CHANNELS; i++) 591 if (efx->channel[i]) { 592 kfree(efx->channel[i]); 593 efx->channel[i] = NULL; 594 } 595 } 596 597 /* Allocate and initialise a channel structure, copying parameters 598 * (but not resources) from an old channel structure. 599 */ 600 static 601 struct efx_channel *efx_copy_channel(const struct efx_channel *old_channel) 602 { 603 struct efx_rx_queue *rx_queue; 604 struct efx_tx_queue *tx_queue; 605 struct efx_channel *channel; 606 int j; 607 608 channel = kmalloc(sizeof(*channel), GFP_KERNEL); 609 if (!channel) 610 return NULL; 611 612 *channel = *old_channel; 613 614 channel->napi_dev = NULL; 615 INIT_HLIST_NODE(&channel->napi_str.napi_hash_node); 616 channel->napi_str.napi_id = 0; 617 channel->napi_str.state = 0; 618 memset(&channel->eventq, 0, sizeof(channel->eventq)); 619 620 for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) { 621 tx_queue = &channel->tx_queue[j]; 622 if (tx_queue->channel) 623 tx_queue->channel = channel; 624 tx_queue->buffer = NULL; 625 tx_queue->cb_page = NULL; 626 memset(&tx_queue->txd, 0, sizeof(tx_queue->txd)); 627 } 628 629 rx_queue = &channel->rx_queue; 630 rx_queue->buffer = NULL; 631 memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd)); 632 timer_setup(&rx_queue->slow_fill, efx_siena_rx_slow_fill, 0); 633 #ifdef CONFIG_RFS_ACCEL 634 INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire); 635 #endif 636 637 return channel; 638 } 639 640 static int efx_probe_channel(struct efx_channel *channel) 641 { 642 struct efx_tx_queue *tx_queue; 643 struct efx_rx_queue *rx_queue; 644 int rc; 645 646 netif_dbg(channel->efx, probe, channel->efx->net_dev, 647 "creating channel %d\n", channel->channel); 648 649 rc = channel->type->pre_probe(channel); 650 if (rc) 651 goto fail; 652 653 rc = efx_probe_eventq(channel); 654 if (rc) 655 goto fail; 656 657 efx_for_each_channel_tx_queue(tx_queue, channel) { 658 rc = efx_siena_probe_tx_queue(tx_queue); 659 if (rc) 660 goto fail; 661 } 662 663 efx_for_each_channel_rx_queue(rx_queue, channel) { 664 rc = efx_siena_probe_rx_queue(rx_queue); 665 if (rc) 666 goto fail; 667 } 668 669 channel->rx_list = NULL; 670 671 return 0; 672 673 fail: 674 efx_siena_remove_channel(channel); 675 return rc; 676 } 677 678 static void efx_get_channel_name(struct efx_channel *channel, char *buf, 679 size_t len) 680 { 681 struct efx_nic *efx = channel->efx; 682 const char *type; 683 int number; 684 685 number = channel->channel; 686 687 if (number >= efx->xdp_channel_offset && 688 !WARN_ON_ONCE(!efx->n_xdp_channels)) { 689 type = "-xdp"; 690 number -= efx->xdp_channel_offset; 691 } else if (efx->tx_channel_offset == 0) { 692 type = ""; 693 } else if (number < efx->tx_channel_offset) { 694 type = "-rx"; 695 } else { 696 type = "-tx"; 697 number -= efx->tx_channel_offset; 698 } 699 snprintf(buf, len, "%s%s-%d", efx->name, type, number); 700 } 701 702 void efx_siena_set_channel_names(struct efx_nic *efx) 703 { 704 struct efx_channel *channel; 705 706 efx_for_each_channel(channel, efx) 707 channel->type->get_name(channel, 708 efx->msi_context[channel->channel].name, 709 sizeof(efx->msi_context[0].name)); 710 } 711 712 int efx_siena_probe_channels(struct efx_nic *efx) 713 { 714 struct efx_channel *channel; 715 int rc; 716 717 /* Restart special buffer allocation */ 718 efx->next_buffer_table = 0; 719 720 /* Probe channels in reverse, so that any 'extra' channels 721 * use the start of the buffer table. This allows the traffic 722 * channels to be resized without moving them or wasting the 723 * entries before them. 724 */ 725 efx_for_each_channel_rev(channel, efx) { 726 rc = efx_probe_channel(channel); 727 if (rc) { 728 netif_err(efx, probe, efx->net_dev, 729 "failed to create channel %d\n", 730 channel->channel); 731 goto fail; 732 } 733 } 734 efx_siena_set_channel_names(efx); 735 736 return 0; 737 738 fail: 739 efx_siena_remove_channels(efx); 740 return rc; 741 } 742 743 void efx_siena_remove_channel(struct efx_channel *channel) 744 { 745 struct efx_tx_queue *tx_queue; 746 struct efx_rx_queue *rx_queue; 747 748 netif_dbg(channel->efx, drv, channel->efx->net_dev, 749 "destroy chan %d\n", channel->channel); 750 751 efx_for_each_channel_rx_queue(rx_queue, channel) 752 efx_siena_remove_rx_queue(rx_queue); 753 efx_for_each_channel_tx_queue(tx_queue, channel) 754 efx_siena_remove_tx_queue(tx_queue); 755 efx_remove_eventq(channel); 756 channel->type->post_remove(channel); 757 } 758 759 void efx_siena_remove_channels(struct efx_nic *efx) 760 { 761 struct efx_channel *channel; 762 763 efx_for_each_channel(channel, efx) 764 efx_siena_remove_channel(channel); 765 766 kfree(efx->xdp_tx_queues); 767 } 768 769 static int efx_set_xdp_tx_queue(struct efx_nic *efx, int xdp_queue_number, 770 struct efx_tx_queue *tx_queue) 771 { 772 if (xdp_queue_number >= efx->xdp_tx_queue_count) 773 return -EINVAL; 774 775 netif_dbg(efx, drv, efx->net_dev, 776 "Channel %u TXQ %u is XDP %u, HW %u\n", 777 tx_queue->channel->channel, tx_queue->label, 778 xdp_queue_number, tx_queue->queue); 779 efx->xdp_tx_queues[xdp_queue_number] = tx_queue; 780 return 0; 781 } 782 783 static void efx_set_xdp_channels(struct efx_nic *efx) 784 { 785 struct efx_tx_queue *tx_queue; 786 struct efx_channel *channel; 787 unsigned int next_queue = 0; 788 int xdp_queue_number = 0; 789 int rc; 790 791 /* We need to mark which channels really have RX and TX 792 * queues, and adjust the TX queue numbers if we have separate 793 * RX-only and TX-only channels. 794 */ 795 efx_for_each_channel(channel, efx) { 796 if (channel->channel < efx->tx_channel_offset) 797 continue; 798 799 if (efx_channel_is_xdp_tx(channel)) { 800 efx_for_each_channel_tx_queue(tx_queue, channel) { 801 tx_queue->queue = next_queue++; 802 rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, 803 tx_queue); 804 if (rc == 0) 805 xdp_queue_number++; 806 } 807 } else { 808 efx_for_each_channel_tx_queue(tx_queue, channel) { 809 tx_queue->queue = next_queue++; 810 netif_dbg(efx, drv, efx->net_dev, 811 "Channel %u TXQ %u is HW %u\n", 812 channel->channel, tx_queue->label, 813 tx_queue->queue); 814 } 815 816 /* If XDP is borrowing queues from net stack, it must 817 * use the queue with no csum offload, which is the 818 * first one of the channel 819 * (note: tx_queue_by_type is not initialized yet) 820 */ 821 if (efx->xdp_txq_queues_mode == 822 EFX_XDP_TX_QUEUES_BORROWED) { 823 tx_queue = &channel->tx_queue[0]; 824 rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, 825 tx_queue); 826 if (rc == 0) 827 xdp_queue_number++; 828 } 829 } 830 } 831 WARN_ON(efx->xdp_txq_queues_mode == EFX_XDP_TX_QUEUES_DEDICATED && 832 xdp_queue_number != efx->xdp_tx_queue_count); 833 WARN_ON(efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_DEDICATED && 834 xdp_queue_number > efx->xdp_tx_queue_count); 835 836 /* If we have more CPUs than assigned XDP TX queues, assign the already 837 * existing queues to the exceeding CPUs 838 */ 839 next_queue = 0; 840 while (xdp_queue_number < efx->xdp_tx_queue_count) { 841 tx_queue = efx->xdp_tx_queues[next_queue++]; 842 rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, tx_queue); 843 if (rc == 0) 844 xdp_queue_number++; 845 } 846 } 847 848 static int efx_soft_enable_interrupts(struct efx_nic *efx); 849 static void efx_soft_disable_interrupts(struct efx_nic *efx); 850 static void efx_init_napi_channel(struct efx_channel *channel); 851 static void efx_fini_napi_channel(struct efx_channel *channel); 852 853 int efx_siena_realloc_channels(struct efx_nic *efx, u32 rxq_entries, 854 u32 txq_entries) 855 { 856 struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel; 857 unsigned int i, next_buffer_table = 0; 858 u32 old_rxq_entries, old_txq_entries; 859 int rc, rc2; 860 861 rc = efx_check_disabled(efx); 862 if (rc) 863 return rc; 864 865 /* Not all channels should be reallocated. We must avoid 866 * reallocating their buffer table entries. 867 */ 868 efx_for_each_channel(channel, efx) { 869 struct efx_rx_queue *rx_queue; 870 struct efx_tx_queue *tx_queue; 871 872 if (channel->type->copy) 873 continue; 874 next_buffer_table = max(next_buffer_table, 875 channel->eventq.index + 876 channel->eventq.entries); 877 efx_for_each_channel_rx_queue(rx_queue, channel) 878 next_buffer_table = max(next_buffer_table, 879 rx_queue->rxd.index + 880 rx_queue->rxd.entries); 881 efx_for_each_channel_tx_queue(tx_queue, channel) 882 next_buffer_table = max(next_buffer_table, 883 tx_queue->txd.index + 884 tx_queue->txd.entries); 885 } 886 887 efx_device_detach_sync(efx); 888 efx_siena_stop_all(efx); 889 efx_soft_disable_interrupts(efx); 890 891 /* Clone channels (where possible) */ 892 memset(other_channel, 0, sizeof(other_channel)); 893 for (i = 0; i < efx->n_channels; i++) { 894 channel = efx->channel[i]; 895 if (channel->type->copy) 896 channel = channel->type->copy(channel); 897 if (!channel) { 898 rc = -ENOMEM; 899 goto out; 900 } 901 other_channel[i] = channel; 902 } 903 904 /* Swap entry counts and channel pointers */ 905 old_rxq_entries = efx->rxq_entries; 906 old_txq_entries = efx->txq_entries; 907 efx->rxq_entries = rxq_entries; 908 efx->txq_entries = txq_entries; 909 for (i = 0; i < efx->n_channels; i++) 910 swap(efx->channel[i], other_channel[i]); 911 912 /* Restart buffer table allocation */ 913 efx->next_buffer_table = next_buffer_table; 914 915 for (i = 0; i < efx->n_channels; i++) { 916 channel = efx->channel[i]; 917 if (!channel->type->copy) 918 continue; 919 rc = efx_probe_channel(channel); 920 if (rc) 921 goto rollback; 922 efx_init_napi_channel(efx->channel[i]); 923 } 924 925 efx_set_xdp_channels(efx); 926 out: 927 /* Destroy unused channel structures */ 928 for (i = 0; i < efx->n_channels; i++) { 929 channel = other_channel[i]; 930 if (channel && channel->type->copy) { 931 efx_fini_napi_channel(channel); 932 efx_siena_remove_channel(channel); 933 kfree(channel); 934 } 935 } 936 937 rc2 = efx_soft_enable_interrupts(efx); 938 if (rc2) { 939 rc = rc ? rc : rc2; 940 netif_err(efx, drv, efx->net_dev, 941 "unable to restart interrupts on channel reallocation\n"); 942 efx_siena_schedule_reset(efx, RESET_TYPE_DISABLE); 943 } else { 944 efx_siena_start_all(efx); 945 efx_device_attach_if_not_resetting(efx); 946 } 947 return rc; 948 949 rollback: 950 /* Swap back */ 951 efx->rxq_entries = old_rxq_entries; 952 efx->txq_entries = old_txq_entries; 953 for (i = 0; i < efx->n_channels; i++) 954 swap(efx->channel[i], other_channel[i]); 955 goto out; 956 } 957 958 int efx_siena_set_channels(struct efx_nic *efx) 959 { 960 struct efx_channel *channel; 961 int rc; 962 963 if (efx->xdp_tx_queue_count) { 964 EFX_WARN_ON_PARANOID(efx->xdp_tx_queues); 965 966 /* Allocate array for XDP TX queue lookup. */ 967 efx->xdp_tx_queues = kcalloc(efx->xdp_tx_queue_count, 968 sizeof(*efx->xdp_tx_queues), 969 GFP_KERNEL); 970 if (!efx->xdp_tx_queues) 971 return -ENOMEM; 972 } 973 974 efx_for_each_channel(channel, efx) { 975 if (channel->channel < efx->n_rx_channels) 976 channel->rx_queue.core_index = channel->channel; 977 else 978 channel->rx_queue.core_index = -1; 979 } 980 981 efx_set_xdp_channels(efx); 982 983 rc = netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels); 984 if (rc) 985 return rc; 986 return netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels); 987 } 988 989 static bool efx_default_channel_want_txqs(struct efx_channel *channel) 990 { 991 return channel->channel - channel->efx->tx_channel_offset < 992 channel->efx->n_tx_channels; 993 } 994 995 /************* 996 * START/STOP 997 *************/ 998 999 static int efx_soft_enable_interrupts(struct efx_nic *efx) 1000 { 1001 struct efx_channel *channel, *end_channel; 1002 int rc; 1003 1004 BUG_ON(efx->state == STATE_DISABLED); 1005 1006 efx->irq_soft_enabled = true; 1007 smp_wmb(); 1008 1009 efx_for_each_channel(channel, efx) { 1010 if (!channel->type->keep_eventq) { 1011 rc = efx_init_eventq(channel); 1012 if (rc) 1013 goto fail; 1014 } 1015 efx_siena_start_eventq(channel); 1016 } 1017 1018 efx_siena_mcdi_mode_event(efx); 1019 1020 return 0; 1021 fail: 1022 end_channel = channel; 1023 efx_for_each_channel(channel, efx) { 1024 if (channel == end_channel) 1025 break; 1026 efx_siena_stop_eventq(channel); 1027 if (!channel->type->keep_eventq) 1028 efx_fini_eventq(channel); 1029 } 1030 1031 return rc; 1032 } 1033 1034 static void efx_soft_disable_interrupts(struct efx_nic *efx) 1035 { 1036 struct efx_channel *channel; 1037 1038 if (efx->state == STATE_DISABLED) 1039 return; 1040 1041 efx_siena_mcdi_mode_poll(efx); 1042 1043 efx->irq_soft_enabled = false; 1044 smp_wmb(); 1045 1046 if (efx->legacy_irq) 1047 synchronize_irq(efx->legacy_irq); 1048 1049 efx_for_each_channel(channel, efx) { 1050 if (channel->irq) 1051 synchronize_irq(channel->irq); 1052 1053 efx_siena_stop_eventq(channel); 1054 if (!channel->type->keep_eventq) 1055 efx_fini_eventq(channel); 1056 } 1057 1058 /* Flush the asynchronous MCDI request queue */ 1059 efx_siena_mcdi_flush_async(efx); 1060 } 1061 1062 int efx_siena_enable_interrupts(struct efx_nic *efx) 1063 { 1064 struct efx_channel *channel, *end_channel; 1065 int rc; 1066 1067 /* TODO: Is this really a bug? */ 1068 BUG_ON(efx->state == STATE_DISABLED); 1069 1070 if (efx->eeh_disabled_legacy_irq) { 1071 enable_irq(efx->legacy_irq); 1072 efx->eeh_disabled_legacy_irq = false; 1073 } 1074 1075 efx->type->irq_enable_master(efx); 1076 1077 efx_for_each_channel(channel, efx) { 1078 if (channel->type->keep_eventq) { 1079 rc = efx_init_eventq(channel); 1080 if (rc) 1081 goto fail; 1082 } 1083 } 1084 1085 rc = efx_soft_enable_interrupts(efx); 1086 if (rc) 1087 goto fail; 1088 1089 return 0; 1090 1091 fail: 1092 end_channel = channel; 1093 efx_for_each_channel(channel, efx) { 1094 if (channel == end_channel) 1095 break; 1096 if (channel->type->keep_eventq) 1097 efx_fini_eventq(channel); 1098 } 1099 1100 efx->type->irq_disable_non_ev(efx); 1101 1102 return rc; 1103 } 1104 1105 void efx_siena_disable_interrupts(struct efx_nic *efx) 1106 { 1107 struct efx_channel *channel; 1108 1109 efx_soft_disable_interrupts(efx); 1110 1111 efx_for_each_channel(channel, efx) { 1112 if (channel->type->keep_eventq) 1113 efx_fini_eventq(channel); 1114 } 1115 1116 efx->type->irq_disable_non_ev(efx); 1117 } 1118 1119 void efx_siena_start_channels(struct efx_nic *efx) 1120 { 1121 struct efx_tx_queue *tx_queue; 1122 struct efx_rx_queue *rx_queue; 1123 struct efx_channel *channel; 1124 1125 efx_for_each_channel_rev(channel, efx) { 1126 efx_for_each_channel_tx_queue(tx_queue, channel) { 1127 efx_siena_init_tx_queue(tx_queue); 1128 atomic_inc(&efx->active_queues); 1129 } 1130 1131 efx_for_each_channel_rx_queue(rx_queue, channel) { 1132 efx_siena_init_rx_queue(rx_queue); 1133 atomic_inc(&efx->active_queues); 1134 efx_siena_stop_eventq(channel); 1135 efx_siena_fast_push_rx_descriptors(rx_queue, false); 1136 efx_siena_start_eventq(channel); 1137 } 1138 1139 WARN_ON(channel->rx_pkt_n_frags); 1140 } 1141 } 1142 1143 void efx_siena_stop_channels(struct efx_nic *efx) 1144 { 1145 struct efx_tx_queue *tx_queue; 1146 struct efx_rx_queue *rx_queue; 1147 struct efx_channel *channel; 1148 int rc = 0; 1149 1150 /* Stop RX refill */ 1151 efx_for_each_channel(channel, efx) { 1152 efx_for_each_channel_rx_queue(rx_queue, channel) 1153 rx_queue->refill_enabled = false; 1154 } 1155 1156 efx_for_each_channel(channel, efx) { 1157 /* RX packet processing is pipelined, so wait for the 1158 * NAPI handler to complete. At least event queue 0 1159 * might be kept active by non-data events, so don't 1160 * use napi_synchronize() but actually disable NAPI 1161 * temporarily. 1162 */ 1163 if (efx_channel_has_rx_queue(channel)) { 1164 efx_siena_stop_eventq(channel); 1165 efx_siena_start_eventq(channel); 1166 } 1167 } 1168 1169 if (efx->type->fini_dmaq) 1170 rc = efx->type->fini_dmaq(efx); 1171 1172 if (rc) { 1173 netif_err(efx, drv, efx->net_dev, "failed to flush queues\n"); 1174 } else { 1175 netif_dbg(efx, drv, efx->net_dev, 1176 "successfully flushed all queues\n"); 1177 } 1178 1179 efx_for_each_channel(channel, efx) { 1180 efx_for_each_channel_rx_queue(rx_queue, channel) 1181 efx_siena_fini_rx_queue(rx_queue); 1182 efx_for_each_channel_tx_queue(tx_queue, channel) 1183 efx_siena_fini_tx_queue(tx_queue); 1184 } 1185 } 1186 1187 /************************************************************************** 1188 * 1189 * NAPI interface 1190 * 1191 *************************************************************************/ 1192 1193 /* Process channel's event queue 1194 * 1195 * This function is responsible for processing the event queue of a 1196 * single channel. The caller must guarantee that this function will 1197 * never be concurrently called more than once on the same channel, 1198 * though different channels may be being processed concurrently. 1199 */ 1200 static int efx_process_channel(struct efx_channel *channel, int budget) 1201 { 1202 struct efx_tx_queue *tx_queue; 1203 struct list_head rx_list; 1204 int spent; 1205 1206 if (unlikely(!channel->enabled)) 1207 return 0; 1208 1209 /* Prepare the batch receive list */ 1210 EFX_WARN_ON_PARANOID(channel->rx_list != NULL); 1211 INIT_LIST_HEAD(&rx_list); 1212 channel->rx_list = &rx_list; 1213 1214 efx_for_each_channel_tx_queue(tx_queue, channel) { 1215 tx_queue->pkts_compl = 0; 1216 tx_queue->bytes_compl = 0; 1217 } 1218 1219 spent = efx_nic_process_eventq(channel, budget); 1220 if (spent && efx_channel_has_rx_queue(channel)) { 1221 struct efx_rx_queue *rx_queue = 1222 efx_channel_get_rx_queue(channel); 1223 1224 efx_rx_flush_packet(channel); 1225 efx_siena_fast_push_rx_descriptors(rx_queue, true); 1226 } 1227 1228 /* Update BQL */ 1229 efx_for_each_channel_tx_queue(tx_queue, channel) { 1230 if (tx_queue->bytes_compl) { 1231 netdev_tx_completed_queue(tx_queue->core_txq, 1232 tx_queue->pkts_compl, 1233 tx_queue->bytes_compl); 1234 } 1235 } 1236 1237 /* Receive any packets we queued up */ 1238 netif_receive_skb_list(channel->rx_list); 1239 channel->rx_list = NULL; 1240 1241 return spent; 1242 } 1243 1244 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel) 1245 { 1246 int step = efx->irq_mod_step_us; 1247 1248 if (channel->irq_mod_score < irq_adapt_low_thresh) { 1249 if (channel->irq_moderation_us > step) { 1250 channel->irq_moderation_us -= step; 1251 efx->type->push_irq_moderation(channel); 1252 } 1253 } else if (channel->irq_mod_score > irq_adapt_high_thresh) { 1254 if (channel->irq_moderation_us < 1255 efx->irq_rx_moderation_us) { 1256 channel->irq_moderation_us += step; 1257 efx->type->push_irq_moderation(channel); 1258 } 1259 } 1260 1261 channel->irq_count = 0; 1262 channel->irq_mod_score = 0; 1263 } 1264 1265 /* NAPI poll handler 1266 * 1267 * NAPI guarantees serialisation of polls of the same device, which 1268 * provides the guarantee required by efx_process_channel(). 1269 */ 1270 static int efx_poll(struct napi_struct *napi, int budget) 1271 { 1272 struct efx_channel *channel = 1273 container_of(napi, struct efx_channel, napi_str); 1274 struct efx_nic *efx = channel->efx; 1275 #ifdef CONFIG_RFS_ACCEL 1276 unsigned int time; 1277 #endif 1278 int spent; 1279 1280 netif_vdbg(efx, intr, efx->net_dev, 1281 "channel %d NAPI poll executing on CPU %d\n", 1282 channel->channel, raw_smp_processor_id()); 1283 1284 spent = efx_process_channel(channel, budget); 1285 1286 xdp_do_flush_map(); 1287 1288 if (spent < budget) { 1289 if (efx_channel_has_rx_queue(channel) && 1290 efx->irq_rx_adaptive && 1291 unlikely(++channel->irq_count == 1000)) { 1292 efx_update_irq_mod(efx, channel); 1293 } 1294 1295 #ifdef CONFIG_RFS_ACCEL 1296 /* Perhaps expire some ARFS filters */ 1297 time = jiffies - channel->rfs_last_expiry; 1298 /* Would our quota be >= 20? */ 1299 if (channel->rfs_filter_count * time >= 600 * HZ) 1300 mod_delayed_work(system_wq, &channel->filter_work, 0); 1301 #endif 1302 1303 /* There is no race here; although napi_disable() will 1304 * only wait for napi_complete(), this isn't a problem 1305 * since efx_nic_eventq_read_ack() will have no effect if 1306 * interrupts have already been disabled. 1307 */ 1308 if (napi_complete_done(napi, spent)) 1309 efx_nic_eventq_read_ack(channel); 1310 } 1311 1312 return spent; 1313 } 1314 1315 static void efx_init_napi_channel(struct efx_channel *channel) 1316 { 1317 struct efx_nic *efx = channel->efx; 1318 1319 channel->napi_dev = efx->net_dev; 1320 netif_napi_add(channel->napi_dev, &channel->napi_str, efx_poll, 64); 1321 } 1322 1323 void efx_siena_init_napi(struct efx_nic *efx) 1324 { 1325 struct efx_channel *channel; 1326 1327 efx_for_each_channel(channel, efx) 1328 efx_init_napi_channel(channel); 1329 } 1330 1331 static void efx_fini_napi_channel(struct efx_channel *channel) 1332 { 1333 if (channel->napi_dev) 1334 netif_napi_del(&channel->napi_str); 1335 1336 channel->napi_dev = NULL; 1337 } 1338 1339 void efx_siena_fini_napi(struct efx_nic *efx) 1340 { 1341 struct efx_channel *channel; 1342 1343 efx_for_each_channel(channel, efx) 1344 efx_fini_napi_channel(channel); 1345 } 1346 1347 /*************** 1348 * Housekeeping 1349 ***************/ 1350 1351 static int efx_channel_dummy_op_int(struct efx_channel *channel) 1352 { 1353 return 0; 1354 } 1355 1356 void efx_siena_channel_dummy_op_void(struct efx_channel *channel) 1357 { 1358 } 1359 1360 static const struct efx_channel_type efx_default_channel_type = { 1361 .pre_probe = efx_channel_dummy_op_int, 1362 .post_remove = efx_siena_channel_dummy_op_void, 1363 .get_name = efx_get_channel_name, 1364 .copy = efx_copy_channel, 1365 .want_txqs = efx_default_channel_want_txqs, 1366 .keep_eventq = false, 1367 .want_pio = true, 1368 }; 1369