1 // SPDX-License-Identifier: (GPL-2.0 OR MIT) 2 /* Google virtual Ethernet (gve) driver 3 * 4 * Copyright (C) 2015-2024 Google LLC 5 */ 6 7 #include <linux/bitmap.h> 8 #include <linux/bpf.h> 9 #include <linux/cpumask.h> 10 #include <linux/etherdevice.h> 11 #include <linux/filter.h> 12 #include <linux/interrupt.h> 13 #include <linux/irq.h> 14 #include <linux/math64.h> 15 #include <linux/module.h> 16 #include <linux/pci.h> 17 #include <linux/sched.h> 18 #include <linux/timer.h> 19 #include <linux/workqueue.h> 20 #include <net/netdev_queues.h> 21 #include <net/sch_generic.h> 22 #include <net/xdp_sock_drv.h> 23 #include "gve.h" 24 #include "gve_dqo.h" 25 #include "gve_adminq.h" 26 #include "gve_register.h" 27 #include "gve_utils.h" 28 29 #define GVE_DEFAULT_RX_COPYBREAK (256) 30 31 #define DEFAULT_MSG_LEVEL (NETIF_MSG_DRV | NETIF_MSG_LINK) 32 #define GVE_VERSION "1.0.0" 33 #define GVE_VERSION_PREFIX "GVE-" 34 35 // Minimum amount of time between queue kicks in msec (10 seconds) 36 #define MIN_TX_TIMEOUT_GAP (1000 * 10) 37 38 char gve_driver_name[] = "gve"; 39 const char gve_version_str[] = GVE_VERSION; 40 static const char gve_version_prefix[] = GVE_VERSION_PREFIX; 41 42 static netdev_features_t gve_features_check(struct sk_buff *skb, 43 struct net_device *dev, 44 netdev_features_t features) 45 { 46 struct gve_priv *priv = netdev_priv(dev); 47 48 if (!gve_is_gqi(priv)) 49 return gve_features_check_dqo(skb, dev, features); 50 51 return features; 52 } 53 54 static netdev_tx_t gve_start_xmit(struct sk_buff *skb, struct net_device *dev) 55 { 56 struct gve_priv *priv = netdev_priv(dev); 57 58 if (gve_is_gqi(priv)) 59 return gve_tx(skb, dev); 60 else 61 return gve_tx_dqo(skb, dev); 62 } 63 64 static void gve_get_stats(struct net_device *dev, struct rtnl_link_stats64 *s) 65 { 66 struct gve_priv *priv = netdev_priv(dev); 67 unsigned int start; 68 u64 packets, bytes; 69 int num_tx_queues; 70 int ring; 71 72 num_tx_queues = gve_num_tx_queues(priv); 73 if (priv->rx) { 74 for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) { 75 do { 76 start = 77 u64_stats_fetch_begin(&priv->rx[ring].statss); 78 packets = priv->rx[ring].rpackets; 79 bytes = priv->rx[ring].rbytes; 80 } while (u64_stats_fetch_retry(&priv->rx[ring].statss, 81 start)); 82 s->rx_packets += packets; 83 s->rx_bytes += bytes; 84 } 85 } 86 if (priv->tx) { 87 for (ring = 0; ring < num_tx_queues; ring++) { 88 do { 89 start = 90 u64_stats_fetch_begin(&priv->tx[ring].statss); 91 packets = priv->tx[ring].pkt_done; 92 bytes = priv->tx[ring].bytes_done; 93 } while (u64_stats_fetch_retry(&priv->tx[ring].statss, 94 start)); 95 s->tx_packets += packets; 96 s->tx_bytes += bytes; 97 } 98 } 99 } 100 101 static int gve_alloc_flow_rule_caches(struct gve_priv *priv) 102 { 103 struct gve_flow_rules_cache *flow_rules_cache = &priv->flow_rules_cache; 104 int err = 0; 105 106 if (!priv->max_flow_rules) 107 return 0; 108 109 flow_rules_cache->rules_cache = 110 kvzalloc_objs(*flow_rules_cache->rules_cache, 111 GVE_FLOW_RULES_CACHE_SIZE); 112 if (!flow_rules_cache->rules_cache) { 113 dev_err(&priv->pdev->dev, "Cannot alloc flow rules cache\n"); 114 return -ENOMEM; 115 } 116 117 flow_rules_cache->rule_ids_cache = 118 kvcalloc(GVE_FLOW_RULE_IDS_CACHE_SIZE, sizeof(*flow_rules_cache->rule_ids_cache), 119 GFP_KERNEL); 120 if (!flow_rules_cache->rule_ids_cache) { 121 dev_err(&priv->pdev->dev, "Cannot alloc flow rule ids cache\n"); 122 err = -ENOMEM; 123 goto free_rules_cache; 124 } 125 126 return 0; 127 128 free_rules_cache: 129 kvfree(flow_rules_cache->rules_cache); 130 flow_rules_cache->rules_cache = NULL; 131 return err; 132 } 133 134 static void gve_free_flow_rule_caches(struct gve_priv *priv) 135 { 136 struct gve_flow_rules_cache *flow_rules_cache = &priv->flow_rules_cache; 137 138 kvfree(flow_rules_cache->rule_ids_cache); 139 flow_rules_cache->rule_ids_cache = NULL; 140 kvfree(flow_rules_cache->rules_cache); 141 flow_rules_cache->rules_cache = NULL; 142 } 143 144 static int gve_alloc_rss_config_cache(struct gve_priv *priv) 145 { 146 struct gve_rss_config *rss_config = &priv->rss_config; 147 148 if (!priv->cache_rss_config) 149 return 0; 150 151 rss_config->hash_key = kcalloc(priv->rss_key_size, 152 sizeof(rss_config->hash_key[0]), 153 GFP_KERNEL); 154 if (!rss_config->hash_key) 155 return -ENOMEM; 156 157 rss_config->hash_lut = kcalloc(priv->rss_lut_size, 158 sizeof(rss_config->hash_lut[0]), 159 GFP_KERNEL); 160 if (!rss_config->hash_lut) 161 goto free_rss_key_cache; 162 163 return 0; 164 165 free_rss_key_cache: 166 kfree(rss_config->hash_key); 167 rss_config->hash_key = NULL; 168 return -ENOMEM; 169 } 170 171 static void gve_free_rss_config_cache(struct gve_priv *priv) 172 { 173 struct gve_rss_config *rss_config = &priv->rss_config; 174 175 kfree(rss_config->hash_key); 176 kfree(rss_config->hash_lut); 177 178 memset(rss_config, 0, sizeof(*rss_config)); 179 } 180 181 static int gve_alloc_counter_array(struct gve_priv *priv) 182 { 183 priv->counter_array = 184 dma_alloc_coherent(&priv->pdev->dev, 185 priv->num_event_counters * 186 sizeof(*priv->counter_array), 187 &priv->counter_array_bus, GFP_KERNEL); 188 if (!priv->counter_array) 189 return -ENOMEM; 190 191 return 0; 192 } 193 194 static void gve_free_counter_array(struct gve_priv *priv) 195 { 196 if (!priv->counter_array) 197 return; 198 199 dma_free_coherent(&priv->pdev->dev, 200 priv->num_event_counters * 201 sizeof(*priv->counter_array), 202 priv->counter_array, priv->counter_array_bus); 203 priv->counter_array = NULL; 204 } 205 206 static int gve_alloc_irq_db_indices(struct gve_priv *priv) 207 { 208 priv->irq_db_indices = 209 dma_alloc_coherent(&priv->pdev->dev, 210 priv->num_ntfy_blks * 211 sizeof(*priv->irq_db_indices), 212 &priv->irq_db_indices_bus, GFP_KERNEL); 213 if (!priv->irq_db_indices) 214 return -ENOMEM; 215 216 return 0; 217 } 218 219 static void gve_free_irq_db_indices(struct gve_priv *priv) 220 { 221 if (!priv->irq_db_indices) 222 return; 223 224 dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks * 225 sizeof(*priv->irq_db_indices), 226 priv->irq_db_indices, priv->irq_db_indices_bus); 227 priv->irq_db_indices = NULL; 228 } 229 230 /* NIC requests to report stats */ 231 static void gve_stats_report_task(struct work_struct *work) 232 { 233 struct gve_priv *priv = container_of(work, struct gve_priv, 234 stats_report_task); 235 if (gve_get_do_report_stats(priv)) { 236 gve_handle_report_stats(priv); 237 gve_clear_do_report_stats(priv); 238 } 239 } 240 241 static void gve_stats_report_schedule(struct gve_priv *priv) 242 { 243 if (!gve_get_probe_in_progress(priv) && 244 !gve_get_reset_in_progress(priv)) { 245 gve_set_do_report_stats(priv); 246 queue_work(priv->gve_wq, &priv->stats_report_task); 247 } 248 } 249 250 static void gve_stats_report_timer(struct timer_list *t) 251 { 252 struct gve_priv *priv = timer_container_of(priv, t, 253 stats_report_timer); 254 255 mod_timer(&priv->stats_report_timer, 256 round_jiffies(jiffies + 257 msecs_to_jiffies(priv->stats_report_timer_period))); 258 gve_stats_report_schedule(priv); 259 } 260 261 static int gve_alloc_stats_report(struct gve_priv *priv) 262 { 263 int tx_stats_num, rx_stats_num; 264 265 tx_stats_num = (GVE_TX_STATS_REPORT_NUM + NIC_TX_STATS_REPORT_NUM) * 266 priv->tx_cfg.max_queues; 267 rx_stats_num = (GVE_RX_STATS_REPORT_NUM + NIC_RX_STATS_REPORT_NUM) * 268 priv->rx_cfg.max_queues; 269 priv->stats_report_len = struct_size(priv->stats_report, stats, 270 size_add(tx_stats_num, rx_stats_num)); 271 priv->stats_report = 272 dma_alloc_coherent(&priv->pdev->dev, priv->stats_report_len, 273 &priv->stats_report_bus, GFP_KERNEL); 274 if (!priv->stats_report) 275 return -ENOMEM; 276 /* Set up timer for the report-stats task */ 277 timer_setup(&priv->stats_report_timer, gve_stats_report_timer, 0); 278 priv->stats_report_timer_period = GVE_STATS_REPORT_TIMER_PERIOD; 279 return 0; 280 } 281 282 static void gve_free_stats_report(struct gve_priv *priv) 283 { 284 if (!priv->stats_report) 285 return; 286 287 timer_delete_sync(&priv->stats_report_timer); 288 cancel_work_sync(&priv->stats_report_task); 289 dma_free_coherent(&priv->pdev->dev, priv->stats_report_len, 290 priv->stats_report, priv->stats_report_bus); 291 priv->stats_report = NULL; 292 } 293 294 static irqreturn_t gve_intr(int irq, void *arg) 295 { 296 struct gve_notify_block *block = arg; 297 struct gve_priv *priv = block->priv; 298 299 iowrite32be(GVE_IRQ_MASK, gve_irq_doorbell(priv, block)); 300 napi_schedule_irqoff(&block->napi); 301 return IRQ_HANDLED; 302 } 303 304 static irqreturn_t gve_intr_dqo(int irq, void *arg) 305 { 306 struct gve_notify_block *block = arg; 307 308 /* Interrupts are automatically masked */ 309 napi_schedule_irqoff(&block->napi); 310 return IRQ_HANDLED; 311 } 312 313 static int gve_is_napi_on_home_cpu(struct gve_priv *priv, u32 irq) 314 { 315 int cpu_curr = smp_processor_id(); 316 const struct cpumask *aff_mask; 317 318 aff_mask = irq_get_effective_affinity_mask(irq); 319 if (unlikely(!aff_mask)) 320 return 1; 321 322 return cpumask_test_cpu(cpu_curr, aff_mask); 323 } 324 325 int gve_napi_poll(struct napi_struct *napi, int budget) 326 { 327 struct gve_notify_block *block; 328 __be32 __iomem *irq_doorbell; 329 bool reschedule = false; 330 struct gve_priv *priv; 331 int work_done = 0; 332 333 block = container_of(napi, struct gve_notify_block, napi); 334 priv = block->priv; 335 336 if (block->tx) { 337 if (block->tx->q_num < priv->tx_cfg.num_queues) 338 reschedule |= gve_tx_poll(block, budget); 339 else if (budget) 340 reschedule |= gve_xdp_poll(block, budget); 341 } 342 343 if (!budget) 344 return 0; 345 346 if (block->rx) { 347 work_done = gve_rx_poll(block, budget); 348 349 /* Poll XSK TX as part of RX NAPI. Setup re-poll based on max of 350 * TX and RX work done. 351 */ 352 if (priv->xdp_prog) 353 work_done = max_t(int, work_done, 354 gve_xsk_tx_poll(block, budget)); 355 356 reschedule |= work_done == budget; 357 } 358 359 if (reschedule) 360 return budget; 361 362 /* Complete processing - don't unmask irq if busy polling is enabled */ 363 if (likely(napi_complete_done(napi, work_done))) { 364 irq_doorbell = gve_irq_doorbell(priv, block); 365 iowrite32be(GVE_IRQ_ACK | GVE_IRQ_EVENT, irq_doorbell); 366 367 /* Ensure IRQ ACK is visible before we check pending work. 368 * If queue had issued updates, it would be truly visible. 369 */ 370 mb(); 371 372 if (block->tx) 373 reschedule |= gve_tx_clean_pending(priv, block->tx); 374 if (block->rx) 375 reschedule |= gve_rx_work_pending(block->rx); 376 377 if (reschedule && napi_schedule(napi)) 378 iowrite32be(GVE_IRQ_MASK, irq_doorbell); 379 } 380 return work_done; 381 } 382 383 int gve_napi_poll_dqo(struct napi_struct *napi, int budget) 384 { 385 struct gve_notify_block *block = 386 container_of(napi, struct gve_notify_block, napi); 387 struct gve_priv *priv = block->priv; 388 bool reschedule = false; 389 int work_done = 0; 390 391 if (block->tx) { 392 if (block->tx->q_num < priv->tx_cfg.num_queues) 393 reschedule |= gve_tx_poll_dqo(block, /*do_clean=*/true); 394 else 395 reschedule |= gve_xdp_poll_dqo(block); 396 } 397 398 if (!budget) 399 return 0; 400 401 if (block->rx) { 402 work_done = gve_rx_poll_dqo(block, budget); 403 404 /* Poll XSK TX as part of RX NAPI. Setup re-poll based on if 405 * either datapath has more work to do. 406 */ 407 if (priv->xdp_prog) 408 reschedule |= gve_xsk_tx_poll_dqo(block, budget); 409 reschedule |= work_done == budget; 410 } 411 412 if (reschedule) { 413 /* Reschedule by returning budget only if already on the correct 414 * cpu. 415 */ 416 if (likely(gve_is_napi_on_home_cpu(priv, block->irq))) 417 return budget; 418 419 /* If not on the cpu with which this queue's irq has affinity 420 * with, we avoid rescheduling napi and arm the irq instead so 421 * that napi gets rescheduled back eventually onto the right 422 * cpu. 423 */ 424 if (work_done == budget) 425 work_done--; 426 } 427 428 if (likely(napi_complete_done(napi, work_done))) { 429 /* Enable interrupts again. 430 * 431 * We don't need to repoll afterwards because HW supports the 432 * PCI MSI-X PBA feature. 433 * 434 * Another interrupt would be triggered if a new event came in 435 * since the last one. 436 */ 437 gve_write_irq_doorbell_dqo(priv, block, 438 GVE_ITR_NO_UPDATE_DQO | GVE_ITR_ENABLE_BIT_DQO); 439 } 440 441 return work_done; 442 } 443 444 static void gve_free_notify_blocks(struct gve_priv *priv) 445 { 446 pci_disable_msix(priv->pdev); 447 kvfree(priv->ntfy_blocks); 448 priv->ntfy_blocks = NULL; 449 kvfree(priv->msix_vectors); 450 priv->msix_vectors = NULL; 451 } 452 453 static const struct cpumask *gve_get_node_mask(struct gve_priv *priv) 454 { 455 if (priv->numa_node == NUMA_NO_NODE) 456 return cpu_all_mask; 457 else 458 return cpumask_of_node(priv->numa_node); 459 } 460 461 static int gve_alloc_notify_blocks(struct gve_priv *priv) 462 { 463 int num_vecs_requested = priv->num_ntfy_blks + 1; 464 int vecs_enabled; 465 int err; 466 int i; 467 468 priv->msix_vectors = kvzalloc_objs(*priv->msix_vectors, 469 num_vecs_requested); 470 if (!priv->msix_vectors) 471 return -ENOMEM; 472 for (i = 0; i < num_vecs_requested; i++) 473 priv->msix_vectors[i].entry = i; 474 vecs_enabled = pci_enable_msix_range(priv->pdev, priv->msix_vectors, 475 GVE_MIN_MSIX, num_vecs_requested); 476 if (vecs_enabled < 0) { 477 dev_err(&priv->pdev->dev, "Could not enable min msix %d/%d\n", 478 GVE_MIN_MSIX, vecs_enabled); 479 err = vecs_enabled; 480 goto abort; 481 } 482 if (vecs_enabled != num_vecs_requested) { 483 int new_num_ntfy_blks = (vecs_enabled - 1) & ~0x1; 484 int vecs_per_type = new_num_ntfy_blks / 2; 485 int vecs_left = new_num_ntfy_blks % 2; 486 487 priv->num_ntfy_blks = new_num_ntfy_blks; 488 priv->mgmt_msix_idx = priv->num_ntfy_blks; 489 priv->tx_cfg.max_queues = min_t(int, priv->tx_cfg.max_queues, 490 vecs_per_type); 491 priv->rx_cfg.max_queues = min_t(int, priv->rx_cfg.max_queues, 492 vecs_per_type + vecs_left); 493 dev_err(&priv->pdev->dev, 494 "Could not enable desired msix, only enabled %d, adjusting tx max queues to %d, and rx max queues to %d\n", 495 vecs_enabled, priv->tx_cfg.max_queues, 496 priv->rx_cfg.max_queues); 497 if (priv->tx_cfg.num_queues > priv->tx_cfg.max_queues) 498 priv->tx_cfg.num_queues = priv->tx_cfg.max_queues; 499 if (priv->rx_cfg.num_queues > priv->rx_cfg.max_queues) 500 priv->rx_cfg.num_queues = priv->rx_cfg.max_queues; 501 } 502 503 priv->ntfy_blocks = kvzalloc(priv->num_ntfy_blks * 504 sizeof(*priv->ntfy_blocks), GFP_KERNEL); 505 if (!priv->ntfy_blocks) { 506 err = -ENOMEM; 507 goto abort; 508 } 509 510 return 0; 511 abort: 512 gve_free_notify_blocks(priv); 513 return err; 514 } 515 516 static void gve_teardown_notify_blocks(struct gve_priv *priv) 517 { 518 const struct gve_ctrl_ops *ops = priv->ctrl_ops; 519 int i; 520 521 ops->teardown_mgmt_irq(priv); 522 523 if (!priv->ntfy_blocks) 524 return; 525 526 for (i = 0; i < priv->num_ntfy_blks; i++) { 527 struct gve_notify_block *block = &priv->ntfy_blocks[i]; 528 int msix_idx = gve_ntfy_to_msix_idx(priv, i); 529 530 if (!block->irq_requested) 531 continue; 532 533 irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector, 534 NULL); 535 free_irq(priv->msix_vectors[msix_idx].vector, block); 536 block->irq_requested = false; 537 } 538 } 539 540 static int gve_setup_notify_blocks(struct gve_priv *priv) 541 { 542 const struct gve_ctrl_ops *ops = priv->ctrl_ops; 543 const struct cpumask *node_mask; 544 unsigned int cur_cpu; 545 int i; 546 int err; 547 548 /* Setup Management Vector */ 549 err = ops->setup_mgmt_irq(priv); 550 if (err) { 551 dev_err(&priv->pdev->dev, 552 "Did not receive management vector.\n"); 553 return err; 554 } 555 556 node_mask = gve_get_node_mask(priv); 557 cur_cpu = cpumask_first(node_mask); 558 for (i = 0; i < priv->num_ntfy_blks; i++) { 559 struct gve_notify_block *block = &priv->ntfy_blocks[i]; 560 int msix_idx = gve_ntfy_to_msix_idx(priv, i); 561 562 snprintf(block->name, sizeof(block->name), "gve-ntfy-blk%d@pci:%s", 563 i, pci_name(priv->pdev)); 564 block->priv = priv; 565 err = request_irq(priv->msix_vectors[msix_idx].vector, 566 gve_is_gqi(priv) ? gve_intr : gve_intr_dqo, 567 IRQF_NO_AUTOEN, block->name, block); 568 if (err) { 569 dev_err(&priv->pdev->dev, 570 "Failed to receive msix vector %d\n", msix_idx); 571 goto abort; 572 } 573 block->irq = priv->msix_vectors[msix_idx].vector; 574 block->irq_requested = true; 575 irq_set_affinity_and_hint(block->irq, 576 cpumask_of(cur_cpu)); 577 578 cur_cpu = cpumask_next(cur_cpu, node_mask); 579 /* Wrap once CPUs in the node have been exhausted, or when 580 * starting RX queue affinities. TX and RX queues of the same 581 * index share affinity. 582 */ 583 if (cur_cpu >= nr_cpu_ids || (i + 1) == priv->tx_cfg.max_queues) 584 cur_cpu = cpumask_first(node_mask); 585 } 586 return 0; 587 588 abort: 589 gve_teardown_notify_blocks(priv); 590 return err; 591 } 592 593 static void gve_tx_get_curr_alloc_cfg(struct gve_priv *priv, 594 struct gve_tx_alloc_rings_cfg *cfg) 595 { 596 cfg->qcfg = &priv->tx_cfg; 597 cfg->raw_addressing = !gve_is_qpl(priv); 598 cfg->ring_size = priv->tx_desc_cnt; 599 cfg->pages_per_qpl = priv->tx_pages_per_qpl; 600 cfg->num_xdp_rings = cfg->qcfg->num_xdp_queues; 601 cfg->tx = priv->tx; 602 } 603 604 static void gve_rx_get_curr_alloc_cfg(struct gve_priv *priv, 605 struct gve_rx_alloc_rings_cfg *cfg) 606 { 607 cfg->qcfg_rx = &priv->rx_cfg; 608 cfg->qcfg_tx = &priv->tx_cfg; 609 cfg->raw_addressing = !gve_is_qpl(priv); 610 cfg->enable_header_split = priv->header_split_enabled; 611 cfg->ring_size = priv->rx_desc_cnt; 612 cfg->pages_per_qpl = priv->rx_pages_per_qpl; 613 cfg->packet_buffer_size = priv->rx_cfg.packet_buffer_size; 614 cfg->rx = priv->rx; 615 cfg->xdp = !!cfg->qcfg_tx->num_xdp_queues; 616 } 617 618 void gve_get_curr_alloc_cfgs(struct gve_priv *priv, 619 struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, 620 struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) 621 { 622 gve_tx_get_curr_alloc_cfg(priv, tx_alloc_cfg); 623 gve_rx_get_curr_alloc_cfg(priv, rx_alloc_cfg); 624 } 625 626 static void gve_queues_mem_free(struct gve_priv *priv, 627 struct gve_tx_alloc_rings_cfg *tx_cfg, 628 struct gve_rx_alloc_rings_cfg *rx_cfg) 629 { 630 if (gve_is_gqi(priv)) { 631 gve_tx_free_rings_gqi(priv, tx_cfg); 632 gve_rx_free_rings_gqi(priv, rx_cfg); 633 } else { 634 gve_tx_free_rings_dqo(priv, tx_cfg); 635 gve_rx_free_rings_dqo(priv, rx_cfg); 636 } 637 } 638 639 static void gve_queues_mem_remove(struct gve_priv *priv) 640 { 641 struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; 642 struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; 643 644 gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); 645 gve_queues_mem_free(priv, &tx_alloc_cfg, &rx_alloc_cfg); 646 priv->tx = NULL; 647 priv->rx = NULL; 648 } 649 650 static void gve_free_control_plane_resources(struct gve_priv *priv) 651 { 652 bitmap_free(priv->xsk_pools); 653 priv->xsk_pools = NULL; 654 655 kvfree(priv->ptype_lut_dqo); 656 priv->ptype_lut_dqo = NULL; 657 658 gve_teardown_clock(priv); 659 gve_free_irq_db_indices(priv); 660 gve_free_counter_array(priv); 661 gve_free_notify_blocks(priv); 662 gve_free_stats_report(priv); 663 gve_free_rss_config_cache(priv); 664 gve_free_flow_rule_caches(priv); 665 } 666 667 static int gve_alloc_control_plane_resources(struct gve_priv *priv) 668 { 669 int err; 670 671 err = gve_alloc_flow_rule_caches(priv); 672 if (err) 673 return err; 674 err = gve_alloc_rss_config_cache(priv); 675 if (err) 676 goto abort; 677 err = gve_alloc_notify_blocks(priv); 678 if (err) 679 goto abort; 680 err = gve_alloc_stats_report(priv); 681 if (err) 682 goto abort; 683 err = gve_alloc_counter_array(priv); 684 if (err) 685 goto abort; 686 err = gve_alloc_irq_db_indices(priv); 687 if (err) 688 goto abort; 689 690 if (!gve_is_gqi(priv)) { 691 priv->ptype_lut_dqo = kvzalloc_obj(*priv->ptype_lut_dqo, 692 GFP_KERNEL); 693 if (!priv->ptype_lut_dqo) { 694 err = -ENOMEM; 695 goto abort; 696 } 697 } 698 699 priv->xsk_pools = bitmap_zalloc(priv->rx_cfg.max_queues, GFP_KERNEL); 700 if (!priv->xsk_pools) { 701 err = -ENOMEM; 702 goto abort; 703 } 704 705 return 0; 706 abort: 707 gve_free_control_plane_resources(priv); 708 return err; 709 } 710 711 static int gve_setup_control_plane_resources(struct gve_priv *priv) 712 { 713 const struct gve_ctrl_ops *ops = priv->ctrl_ops; 714 int err; 715 716 err = ops->request_db_info(priv); 717 if (err) { 718 dev_err(&priv->pdev->dev, "Failed to get db info"); 719 return err; 720 } 721 722 if (!gve_is_gqi(priv)) { 723 err = ops->get_ptype_map(priv); 724 if (err) { 725 dev_err(&priv->pdev->dev, 726 "Failed to get ptype map: err=%d\n", err); 727 goto release_db_resources; 728 } 729 } 730 731 if (priv->nic_timestamp_supported) { 732 err = gve_init_clock(priv); 733 if (err) { 734 dev_warn(&priv->pdev->dev, "Failed to init clock, continuing without PTP support"); 735 err = 0; 736 } 737 } 738 739 err = gve_init_rss_config(priv, priv->rx_cfg.num_queues); 740 if (err) { 741 dev_err(&priv->pdev->dev, "Failed to init RSS config"); 742 goto release_db_resources; 743 } 744 745 err = ops->setup_stats_report(priv, priv->stats_report_len, 746 priv->stats_report_bus, 747 GVE_STATS_REPORT_TIMER_PERIOD); 748 if (err) 749 dev_err(&priv->pdev->dev, 750 "Failed to report stats: err=%d\n", err); 751 752 gve_set_device_resources_ok(priv); 753 return 0; 754 755 release_db_resources: 756 ops->release_db_resources(priv); 757 return err; 758 } 759 760 /** 761 * gve_teardown_control_plane_resources() - Request the device to release any 762 * shared allocated resources. 763 * 764 * @priv: Pointer to the GVE private device data structure. 765 * 766 * If any part of the teardown step fails, the failure is documented, but is 767 * otherwise ignored. It is expected that a device reset is triggered 768 * immediately after tearing down device resources, which would clear any 769 * lingering state on the device. 770 */ 771 static void gve_teardown_control_plane_resources(struct gve_priv *priv) 772 { 773 const struct gve_ctrl_ops *ops = priv->ctrl_ops; 774 int err; 775 776 if (priv->ptp) 777 ptp_cancel_worker_sync(priv->ptp->clock); 778 779 /* Tell device its resources are being freed */ 780 if (gve_get_device_resources_ok(priv)) { 781 err = gve_flow_rules_reset(priv); 782 if (err) 783 dev_err(&priv->pdev->dev, 784 "Failed to reset flow rules: err=%d\n", err); 785 /* detach the stats report */ 786 err = ops->setup_stats_report(priv, 0, 0x0, 787 GVE_STATS_REPORT_TIMER_PERIOD); 788 if (err) 789 dev_err(&priv->pdev->dev, 790 "Failed to detach stats report: err=%d\n", err); 791 ops->release_db_resources(priv); 792 } 793 794 gve_clear_device_resources_ok(priv); 795 } 796 797 /** 798 * gve_reset_device() - Reset the device 799 * 800 * @priv: Pointer to the GVE private device data structure. 801 * 802 * Once this returns, the device is guaranteed not to access any memory it 803 * shares with the driver, so it is safe for the caller to recycle it. Device 804 * commands in gve_teardown_control_plane_resources() can fail, in which case 805 * the hardware reset triggered by gve_adminq_free() is the only such 806 * guarantee. 807 */ 808 static void gve_reset_device(struct gve_priv *priv) 809 { 810 gve_teardown_control_plane_resources(priv); 811 gve_adminq_free(priv); 812 } 813 814 static void gve_teardown_device(struct gve_priv *priv) 815 { 816 gve_reset_device(priv); 817 /* Free any resources shared with the device only after we have a 818 * guarantee that the device will not try to access such resources. 819 */ 820 gve_teardown_notify_blocks(priv); 821 gve_free_control_plane_resources(priv); 822 gve_queues_mem_remove(priv); 823 } 824 825 static int gve_unregister_qpl(struct gve_priv *priv, 826 struct gve_queue_page_list *qpl) 827 { 828 int err; 829 830 if (!qpl) 831 return 0; 832 833 err = gve_adminq_unregister_page_list(priv, qpl->id); 834 if (err) { 835 netif_err(priv, drv, priv->dev, 836 "Failed to unregister queue page list %d\n", 837 qpl->id); 838 return err; 839 } 840 841 priv->num_registered_pages -= qpl->num_entries; 842 return 0; 843 } 844 845 static int gve_register_qpl(struct gve_priv *priv, 846 struct gve_queue_page_list *qpl) 847 { 848 int pages; 849 int err; 850 851 if (!qpl) 852 return 0; 853 854 pages = qpl->num_entries; 855 856 if (pages + priv->num_registered_pages > priv->max_registered_pages) { 857 netif_err(priv, drv, priv->dev, 858 "Reached max number of registered pages %llu > %llu\n", 859 pages + priv->num_registered_pages, 860 priv->max_registered_pages); 861 return -EINVAL; 862 } 863 864 err = gve_adminq_register_page_list(priv, qpl); 865 if (err) { 866 netif_err(priv, drv, priv->dev, 867 "failed to register queue page list %d\n", 868 qpl->id); 869 return err; 870 } 871 872 priv->num_registered_pages += pages; 873 return 0; 874 } 875 876 static struct gve_queue_page_list *gve_tx_get_qpl(struct gve_priv *priv, int idx) 877 { 878 struct gve_tx_ring *tx = &priv->tx[idx]; 879 880 if (gve_is_gqi(priv)) 881 return tx->tx_fifo.qpl; 882 else 883 return tx->dqo.qpl; 884 } 885 886 static struct gve_queue_page_list *gve_rx_get_qpl(struct gve_priv *priv, int idx) 887 { 888 struct gve_rx_ring *rx = &priv->rx[idx]; 889 890 if (gve_is_gqi(priv)) 891 return rx->data.qpl; 892 else 893 return rx->dqo.qpl; 894 } 895 896 static int gve_register_qpls(struct gve_priv *priv) 897 { 898 int num_tx_qpls, num_rx_qpls; 899 int err; 900 int i; 901 902 num_tx_qpls = gve_num_tx_qpls(&priv->tx_cfg, gve_is_qpl(priv)); 903 num_rx_qpls = gve_num_rx_qpls(&priv->rx_cfg, gve_is_qpl(priv)); 904 905 for (i = 0; i < num_tx_qpls; i++) { 906 err = gve_register_qpl(priv, gve_tx_get_qpl(priv, i)); 907 if (err) 908 return err; 909 } 910 911 for (i = 0; i < num_rx_qpls; i++) { 912 err = gve_register_qpl(priv, gve_rx_get_qpl(priv, i)); 913 if (err) 914 return err; 915 } 916 917 return 0; 918 } 919 920 static int gve_unregister_qpls(struct gve_priv *priv) 921 { 922 int num_tx_qpls, num_rx_qpls; 923 int err; 924 int i; 925 926 num_tx_qpls = gve_num_tx_qpls(&priv->tx_cfg, gve_is_qpl(priv)); 927 num_rx_qpls = gve_num_rx_qpls(&priv->rx_cfg, gve_is_qpl(priv)); 928 929 for (i = 0; i < num_tx_qpls; i++) { 930 err = gve_unregister_qpl(priv, gve_tx_get_qpl(priv, i)); 931 /* This failure will trigger a reset - no need to clean */ 932 if (err) 933 return err; 934 } 935 936 for (i = 0; i < num_rx_qpls; i++) { 937 err = gve_unregister_qpl(priv, gve_rx_get_qpl(priv, i)); 938 /* This failure will trigger a reset - no need to clean */ 939 if (err) 940 return err; 941 } 942 return 0; 943 } 944 945 static int gve_create_rings(struct gve_priv *priv) 946 { 947 int err; 948 int i; 949 950 err = priv->ctrl_ops->create_queues(priv); 951 if (err) { 952 /* This failure will trigger a reset - no need to clean 953 * up 954 */ 955 return err; 956 } 957 958 if (gve_is_gqi(priv)) { 959 /* Rx data ring has been prefilled with packet buffers at queue 960 * allocation time. 961 * 962 * Write the doorbell to provide descriptor slots and packet 963 * buffers to the NIC. 964 */ 965 for (i = 0; i < priv->rx_cfg.num_queues; i++) 966 gve_rx_write_doorbell(priv, &priv->rx[i]); 967 } else { 968 for (i = 0; i < priv->rx_cfg.num_queues; i++) { 969 /* Post buffers and ring doorbell. */ 970 gve_rx_post_buffers_dqo(&priv->rx[i]); 971 } 972 } 973 974 return 0; 975 } 976 977 static void init_xdp_sync_stats(struct gve_priv *priv) 978 { 979 int start_id = gve_xdp_tx_start_queue_id(priv); 980 int i; 981 982 /* Init stats */ 983 for (i = start_id; i < start_id + priv->tx_cfg.num_xdp_queues; i++) { 984 int ntfy_idx = gve_tx_idx_to_ntfy(priv, i); 985 986 u64_stats_init(&priv->tx[i].statss); 987 priv->tx[i].ntfy_id = ntfy_idx; 988 } 989 } 990 991 static void gve_init_sync_stats(struct gve_priv *priv) 992 { 993 int i; 994 995 for (i = 0; i < priv->tx_cfg.num_queues; i++) 996 u64_stats_init(&priv->tx[i].statss); 997 998 /* Init stats for XDP TX queues */ 999 init_xdp_sync_stats(priv); 1000 1001 for (i = 0; i < priv->rx_cfg.num_queues; i++) 1002 u64_stats_init(&priv->rx[i].statss); 1003 } 1004 1005 static void gve_tx_stop_rings(struct gve_priv *priv, int num_rings) 1006 { 1007 int i; 1008 1009 if (!priv->tx) 1010 return; 1011 1012 for (i = 0; i < num_rings; i++) { 1013 if (gve_is_gqi(priv)) 1014 gve_tx_stop_ring_gqi(priv, i); 1015 else 1016 gve_tx_stop_ring_dqo(priv, i); 1017 } 1018 } 1019 1020 static void gve_tx_start_rings(struct gve_priv *priv, int num_rings) 1021 { 1022 int i; 1023 1024 for (i = 0; i < num_rings; i++) { 1025 if (gve_is_gqi(priv)) 1026 gve_tx_start_ring_gqi(priv, i); 1027 else 1028 gve_tx_start_ring_dqo(priv, i); 1029 } 1030 } 1031 1032 void gve_update_num_qpl_pages(struct gve_priv *priv, 1033 struct gve_rx_alloc_rings_cfg *rx_alloc_cfg, 1034 struct gve_tx_alloc_rings_cfg *tx_alloc_cfg) 1035 { 1036 u64 ideal_tx_pages, ideal_rx_pages; 1037 u16 tx_num_queues, rx_num_queues; 1038 u64 max_pages, tx_pages; 1039 1040 if (priv->queue_format == GVE_GQI_QPL_FORMAT) { 1041 rx_alloc_cfg->pages_per_qpl = rx_alloc_cfg->ring_size; 1042 } else if (priv->queue_format == GVE_DQO_QPL_FORMAT) { 1043 /* 1044 * We want 2 pages per RX descriptor and half a page per TX 1045 * descriptor, which means the fraction ideal_tx_pages / 1046 * (ideal_tx_pages + ideal_rx_pages) of the pages we allocate 1047 * should be for TX. Shrink proportionally as necessary to avoid 1048 * allocating more than max_registered_pages total pages. 1049 */ 1050 tx_num_queues = tx_alloc_cfg->qcfg->num_queues; 1051 rx_num_queues = rx_alloc_cfg->qcfg_rx->num_queues; 1052 1053 ideal_tx_pages = tx_alloc_cfg->ring_size * tx_num_queues / 2; 1054 ideal_rx_pages = rx_alloc_cfg->ring_size * rx_num_queues * 2; 1055 max_pages = min(priv->max_registered_pages, 1056 ideal_tx_pages + ideal_rx_pages); 1057 1058 tx_pages = div64_u64(max_pages * ideal_tx_pages, 1059 ideal_tx_pages + ideal_rx_pages); 1060 tx_alloc_cfg->pages_per_qpl = div_u64(tx_pages, tx_num_queues); 1061 rx_alloc_cfg->pages_per_qpl = div_u64(max_pages - tx_pages, 1062 rx_num_queues); 1063 } 1064 } 1065 1066 static int gve_queues_mem_alloc(struct gve_priv *priv, 1067 struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, 1068 struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) 1069 { 1070 int err; 1071 1072 gve_update_num_qpl_pages(priv, rx_alloc_cfg, tx_alloc_cfg); 1073 1074 if (gve_is_gqi(priv)) 1075 err = gve_tx_alloc_rings_gqi(priv, tx_alloc_cfg); 1076 else 1077 err = gve_tx_alloc_rings_dqo(priv, tx_alloc_cfg); 1078 if (err) 1079 return err; 1080 1081 if (gve_is_gqi(priv)) 1082 err = gve_rx_alloc_rings_gqi(priv, rx_alloc_cfg); 1083 else 1084 err = gve_rx_alloc_rings_dqo(priv, rx_alloc_cfg); 1085 if (err) 1086 goto free_tx; 1087 1088 return 0; 1089 1090 free_tx: 1091 if (gve_is_gqi(priv)) 1092 gve_tx_free_rings_gqi(priv, tx_alloc_cfg); 1093 else 1094 gve_tx_free_rings_dqo(priv, tx_alloc_cfg); 1095 return err; 1096 } 1097 1098 static int gve_destroy_rings(struct gve_priv *priv) 1099 { 1100 int err; 1101 1102 err = priv->ctrl_ops->destroy_queues(priv); 1103 1104 /* A failure will trigger a reset - no need to clean up */ 1105 return err; 1106 } 1107 1108 int gve_alloc_page(struct gve_priv *priv, struct device *dev, 1109 struct page **page, dma_addr_t *dma, 1110 enum dma_data_direction dir, gfp_t gfp_flags) 1111 { 1112 *page = alloc_pages_node(priv->numa_node, gfp_flags, 0); 1113 if (!*page) { 1114 priv->page_alloc_fail++; 1115 return -ENOMEM; 1116 } 1117 *dma = dma_map_page(dev, *page, 0, PAGE_SIZE, dir); 1118 if (dma_mapping_error(dev, *dma)) { 1119 priv->dma_mapping_error++; 1120 put_page(*page); 1121 return -ENOMEM; 1122 } 1123 return 0; 1124 } 1125 1126 struct gve_queue_page_list *gve_alloc_queue_page_list(struct gve_priv *priv, 1127 u32 id, int pages) 1128 { 1129 struct gve_queue_page_list *qpl; 1130 int err; 1131 int i; 1132 1133 qpl = kvzalloc_obj(*qpl); 1134 if (!qpl) 1135 return NULL; 1136 1137 qpl->id = id; 1138 qpl->num_entries = 0; 1139 qpl->pages = kvzalloc_objs(*qpl->pages, pages); 1140 if (!qpl->pages) 1141 goto abort; 1142 1143 qpl->page_buses = kvzalloc_objs(*qpl->page_buses, pages); 1144 if (!qpl->page_buses) 1145 goto abort; 1146 1147 for (i = 0; i < pages; i++) { 1148 err = gve_alloc_page(priv, &priv->pdev->dev, &qpl->pages[i], 1149 &qpl->page_buses[i], 1150 gve_qpl_dma_dir(priv, id), GFP_KERNEL); 1151 if (err) 1152 goto abort; 1153 qpl->num_entries++; 1154 } 1155 1156 return qpl; 1157 1158 abort: 1159 gve_free_queue_page_list(priv, qpl, id); 1160 return NULL; 1161 } 1162 1163 void gve_free_page(struct device *dev, struct page *page, dma_addr_t dma, 1164 enum dma_data_direction dir) 1165 { 1166 if (!dma_mapping_error(dev, dma)) 1167 dma_unmap_page(dev, dma, PAGE_SIZE, dir); 1168 if (page) 1169 put_page(page); 1170 } 1171 1172 void gve_free_queue_page_list(struct gve_priv *priv, 1173 struct gve_queue_page_list *qpl, 1174 u32 id) 1175 { 1176 int i; 1177 1178 if (!qpl) 1179 return; 1180 if (!qpl->pages) 1181 goto free_qpl; 1182 if (!qpl->page_buses) 1183 goto free_pages; 1184 1185 for (i = 0; i < qpl->num_entries; i++) 1186 gve_free_page(&priv->pdev->dev, qpl->pages[i], 1187 qpl->page_buses[i], gve_qpl_dma_dir(priv, id)); 1188 1189 kvfree(qpl->page_buses); 1190 qpl->page_buses = NULL; 1191 free_pages: 1192 kvfree(qpl->pages); 1193 qpl->pages = NULL; 1194 free_qpl: 1195 kvfree(qpl); 1196 } 1197 1198 /* Use this to schedule a reset when the device is capable of continuing 1199 * to handle other requests in its current state. If it is not, do a reset 1200 * in thread instead. 1201 */ 1202 void gve_schedule_reset(struct gve_priv *priv) 1203 { 1204 gve_set_do_reset(priv); 1205 queue_work(priv->gve_wq, &priv->service_task); 1206 } 1207 1208 static void gve_turndown(struct gve_priv *priv); 1209 static void gve_turnup(struct gve_priv *priv); 1210 1211 static void gve_unreg_xsk_pool(struct gve_priv *priv, u16 qid) 1212 { 1213 struct gve_rx_ring *rx; 1214 1215 if (!priv->rx) 1216 return; 1217 1218 rx = &priv->rx[qid]; 1219 rx->xsk_pool = NULL; 1220 if (xdp_rxq_info_is_reg(&rx->xdp_rxq)) 1221 xdp_rxq_info_unreg_mem_model(&rx->xdp_rxq); 1222 1223 if (!priv->tx) 1224 return; 1225 priv->tx[gve_xdp_tx_queue_id(priv, qid)].xsk_pool = NULL; 1226 } 1227 1228 static int gve_reg_xsk_pool(struct gve_priv *priv, struct net_device *dev, 1229 struct xsk_buff_pool *pool, u16 qid) 1230 { 1231 struct gve_rx_ring *rx; 1232 u16 tx_qid; 1233 int err; 1234 1235 rx = &priv->rx[qid]; 1236 err = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq, 1237 MEM_TYPE_XSK_BUFF_POOL, pool); 1238 if (err) { 1239 gve_unreg_xsk_pool(priv, qid); 1240 return err; 1241 } 1242 1243 rx->xsk_pool = pool; 1244 1245 tx_qid = gve_xdp_tx_queue_id(priv, qid); 1246 priv->tx[tx_qid].xsk_pool = pool; 1247 1248 return 0; 1249 } 1250 1251 static void gve_unreg_xdp_info(struct gve_priv *priv) 1252 { 1253 int i; 1254 1255 if (!priv->tx_cfg.num_xdp_queues || !priv->rx) 1256 return; 1257 1258 for (i = 0; i < priv->rx_cfg.num_queues; i++) { 1259 struct gve_rx_ring *rx = &priv->rx[i]; 1260 1261 if (xdp_rxq_info_is_reg(&rx->xdp_rxq)) 1262 xdp_rxq_info_unreg(&rx->xdp_rxq); 1263 1264 gve_unreg_xsk_pool(priv, i); 1265 } 1266 } 1267 1268 static struct xsk_buff_pool *gve_get_xsk_pool(struct gve_priv *priv, int qid) 1269 { 1270 if (!test_bit(qid, priv->xsk_pools)) 1271 return NULL; 1272 1273 return xsk_get_pool_from_qid(priv->dev, qid); 1274 } 1275 1276 static int gve_reg_xdp_info(struct gve_priv *priv, struct net_device *dev) 1277 { 1278 struct napi_struct *napi; 1279 struct gve_rx_ring *rx; 1280 int err = 0; 1281 int i; 1282 1283 if (!priv->tx_cfg.num_xdp_queues) 1284 return 0; 1285 1286 for (i = 0; i < priv->rx_cfg.num_queues; i++) { 1287 struct xsk_buff_pool *xsk_pool; 1288 1289 rx = &priv->rx[i]; 1290 napi = &priv->ntfy_blocks[rx->ntfy_id].napi; 1291 1292 err = xdp_rxq_info_reg(&rx->xdp_rxq, dev, i, 1293 napi->napi_id); 1294 if (err) 1295 goto err; 1296 1297 xsk_pool = gve_get_xsk_pool(priv, i); 1298 if (xsk_pool) 1299 err = gve_reg_xsk_pool(priv, dev, xsk_pool, i); 1300 else if (gve_is_qpl(priv)) 1301 err = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq, 1302 MEM_TYPE_PAGE_SHARED, 1303 NULL); 1304 else 1305 err = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq, 1306 MEM_TYPE_PAGE_POOL, 1307 rx->dqo.page_pool); 1308 if (err) 1309 goto err; 1310 } 1311 return 0; 1312 1313 err: 1314 gve_unreg_xdp_info(priv); 1315 return err; 1316 } 1317 1318 static void gve_drain_page_cache(struct gve_priv *priv) 1319 { 1320 int i; 1321 1322 if (!priv->rx) 1323 return; 1324 for (i = 0; i < priv->rx_cfg.num_queues; i++) 1325 page_frag_cache_drain(&priv->rx[i].page_cache); 1326 } 1327 1328 static void gve_rx_start_ring(struct gve_priv *priv, int i) 1329 { 1330 if (gve_is_gqi(priv)) 1331 gve_rx_start_ring_gqi(priv, i); 1332 else 1333 gve_rx_start_ring_dqo(priv, i); 1334 } 1335 1336 static void gve_rx_start_rings(struct gve_priv *priv, int num_rings) 1337 { 1338 int i; 1339 1340 for (i = 0; i < num_rings; i++) 1341 gve_rx_start_ring(priv, i); 1342 } 1343 1344 static void gve_rx_stop_ring(struct gve_priv *priv, int i) 1345 { 1346 if (gve_is_gqi(priv)) 1347 gve_rx_stop_ring_gqi(priv, i); 1348 else 1349 gve_rx_stop_ring_dqo(priv, i); 1350 } 1351 1352 static void gve_rx_stop_rings(struct gve_priv *priv, int num_rings) 1353 { 1354 int i; 1355 1356 if (!priv->rx) 1357 return; 1358 1359 for (i = 0; i < num_rings; i++) 1360 gve_rx_stop_ring(priv, i); 1361 } 1362 1363 static void gve_queues_stop(struct gve_priv *priv) 1364 { 1365 gve_unreg_xdp_info(priv); 1366 gve_drain_page_cache(priv); 1367 1368 timer_delete_sync(&priv->stats_report_timer); 1369 cancel_work_sync(&priv->stats_report_task); 1370 1371 gve_tx_stop_rings(priv, gve_num_tx_queues(priv)); 1372 gve_rx_stop_rings(priv, priv->rx_cfg.num_queues); 1373 } 1374 1375 /* The passed-in queue memory is stored into priv and the queues are made live. 1376 * No memory is allocated. Passed-in memory is freed on errors. 1377 */ 1378 static int gve_queues_start(struct gve_priv *priv, 1379 struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, 1380 struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) 1381 { 1382 struct net_device *dev = priv->dev; 1383 int err; 1384 1385 /* Record new resources into priv */ 1386 priv->tx = tx_alloc_cfg->tx; 1387 priv->rx = rx_alloc_cfg->rx; 1388 1389 /* Record new configs into priv */ 1390 priv->tx_cfg = *tx_alloc_cfg->qcfg; 1391 priv->tx_cfg.num_xdp_queues = tx_alloc_cfg->num_xdp_rings; 1392 priv->rx_cfg = *rx_alloc_cfg->qcfg_rx; 1393 priv->tx_desc_cnt = tx_alloc_cfg->ring_size; 1394 priv->rx_desc_cnt = rx_alloc_cfg->ring_size; 1395 priv->tx_pages_per_qpl = tx_alloc_cfg->pages_per_qpl; 1396 priv->rx_pages_per_qpl = rx_alloc_cfg->pages_per_qpl; 1397 priv->header_split_enabled = rx_alloc_cfg->enable_header_split; 1398 priv->rx_cfg.packet_buffer_size = rx_alloc_cfg->packet_buffer_size; 1399 1400 gve_tx_start_rings(priv, gve_num_tx_queues(priv)); 1401 gve_rx_start_rings(priv, rx_alloc_cfg->qcfg_rx->num_queues); 1402 gve_init_sync_stats(priv); 1403 1404 err = netif_set_real_num_tx_queues(dev, priv->tx_cfg.num_queues); 1405 if (err) 1406 goto stop_rings; 1407 err = netif_set_real_num_rx_queues(dev, priv->rx_cfg.num_queues); 1408 if (err) 1409 goto stop_rings; 1410 1411 err = gve_reg_xdp_info(priv, dev); 1412 if (err) 1413 goto stop_rings; 1414 1415 if (rx_alloc_cfg->reset_rss) { 1416 err = gve_init_rss_config(priv, priv->rx_cfg.num_queues); 1417 if (err) 1418 goto reset; 1419 } 1420 1421 err = gve_register_qpls(priv); 1422 if (err) 1423 goto reset; 1424 1425 err = gve_create_rings(priv); 1426 if (err) 1427 goto reset; 1428 1429 gve_set_device_rings_ok(priv); 1430 1431 if (gve_get_report_stats(priv)) 1432 mod_timer(&priv->stats_report_timer, 1433 round_jiffies(jiffies + 1434 msecs_to_jiffies(priv->stats_report_timer_period))); 1435 1436 gve_turnup(priv); 1437 queue_work(priv->gve_wq, &priv->service_task); 1438 priv->interface_up_cnt++; 1439 return 0; 1440 1441 reset: 1442 if (gve_get_reset_in_progress(priv)) 1443 goto stop_rings; 1444 1445 /* Attempt to reset. If reset is successful, gve_queues_start was 1446 * successful with the new config. 1447 */ 1448 return gve_reset(priv, false); 1449 stop_rings: 1450 gve_queues_stop(priv); 1451 return err; 1452 } 1453 1454 static int gve_open(struct net_device *dev) 1455 { 1456 struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; 1457 struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; 1458 struct gve_priv *priv = netdev_priv(dev); 1459 int err; 1460 1461 if (!gve_get_device_resources_ok(priv)) { 1462 dev_err(&priv->pdev->dev, 1463 "Attempting to open netdev without resources. Device must be reset."); 1464 return -ENODEV; 1465 } 1466 1467 gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1468 1469 err = gve_queues_mem_alloc(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1470 if (err) 1471 return err; 1472 1473 err = gve_queues_start(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1474 if (err) { 1475 gve_queues_mem_remove(priv); 1476 return err; 1477 } 1478 1479 return 0; 1480 } 1481 1482 static int gve_close(struct net_device *dev) 1483 { 1484 struct gve_priv *priv = netdev_priv(dev); 1485 int err; 1486 1487 gve_turndown(priv); 1488 1489 /* Surrender to reset if the queue destroying adminq cmds fail. Reset 1490 * will not re-enable the interface. 1491 */ 1492 if (gve_get_device_rings_ok(priv)) { 1493 gve_clear_device_rings_ok(priv); 1494 err = gve_destroy_rings(priv); 1495 if (err) 1496 goto reset; 1497 err = gve_unregister_qpls(priv); 1498 if (err) 1499 goto reset; 1500 } 1501 1502 gve_queues_stop(priv); 1503 gve_queues_mem_remove(priv); 1504 priv->interface_down_cnt++; 1505 return 0; 1506 1507 reset: 1508 err = gve_reset(priv, true); 1509 return err; 1510 } 1511 1512 static void gve_handle_link_status(struct gve_priv *priv) 1513 { 1514 bool link_up; 1515 1516 if (!gve_get_napi_enabled(priv)) 1517 return; 1518 1519 link_up = READ_ONCE(priv->link_up); 1520 if (link_up == netif_carrier_ok(priv->dev)) 1521 return; 1522 1523 if (link_up) { 1524 netdev_info(priv->dev, "Device link is up.\n"); 1525 netif_carrier_on(priv->dev); 1526 } else { 1527 netdev_info(priv->dev, "Device link is down.\n"); 1528 netif_carrier_off(priv->dev); 1529 } 1530 } 1531 1532 static int gve_configure_rings_xdp(struct gve_priv *priv, 1533 u16 num_xdp_rings) 1534 { 1535 struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; 1536 struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; 1537 1538 gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1539 tx_alloc_cfg.num_xdp_rings = num_xdp_rings; 1540 1541 rx_alloc_cfg.xdp = !!num_xdp_rings; 1542 return gve_adjust_config(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1543 } 1544 1545 static int gve_set_xdp(struct gve_priv *priv, struct bpf_prog *prog, 1546 struct netlink_ext_ack *extack) 1547 { 1548 struct bpf_prog *old_prog; 1549 int err = 0; 1550 1551 old_prog = READ_ONCE(priv->xdp_prog); 1552 if (!netif_running(priv->dev)) { 1553 WRITE_ONCE(priv->xdp_prog, prog); 1554 if (old_prog) 1555 bpf_prog_put(old_prog); 1556 1557 /* Update priv XDP queue configuration */ 1558 priv->tx_cfg.num_xdp_queues = priv->xdp_prog ? 1559 priv->rx_cfg.num_queues : 0; 1560 return 0; 1561 } 1562 1563 if (!old_prog && prog) 1564 err = gve_configure_rings_xdp(priv, priv->rx_cfg.num_queues); 1565 else if (old_prog && !prog) 1566 err = gve_configure_rings_xdp(priv, 0); 1567 1568 if (err) 1569 goto out; 1570 1571 WRITE_ONCE(priv->xdp_prog, prog); 1572 if (old_prog) 1573 bpf_prog_put(old_prog); 1574 1575 out: 1576 priv->ctrl_ops->report_link_status(priv); 1577 gve_handle_link_status(priv); 1578 return err; 1579 } 1580 1581 static int gve_xdp_xmit(struct net_device *dev, int n, 1582 struct xdp_frame **frames, u32 flags) 1583 { 1584 struct gve_priv *priv = netdev_priv(dev); 1585 1586 if (priv->queue_format == GVE_GQI_QPL_FORMAT) 1587 return gve_xdp_xmit_gqi(dev, n, frames, flags); 1588 else if (priv->queue_format == GVE_DQO_RDA_FORMAT) 1589 return gve_xdp_xmit_dqo(dev, n, frames, flags); 1590 1591 return -EOPNOTSUPP; 1592 } 1593 1594 static int gve_xsk_pool_enable(struct net_device *dev, 1595 struct xsk_buff_pool *pool, 1596 u16 qid) 1597 { 1598 struct gve_priv *priv = netdev_priv(dev); 1599 int err; 1600 1601 if (qid >= priv->rx_cfg.num_queues) { 1602 dev_err(&priv->pdev->dev, "xsk pool invalid qid %d", qid); 1603 return -EINVAL; 1604 } 1605 if (xsk_pool_get_rx_frame_size(pool) < 1606 priv->dev->max_mtu + sizeof(struct ethhdr)) { 1607 dev_err(&priv->pdev->dev, "xsk pool frame_len too small"); 1608 return -EINVAL; 1609 } 1610 1611 err = xsk_pool_dma_map(pool, &priv->pdev->dev, 1612 DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); 1613 if (err) 1614 return err; 1615 1616 set_bit(qid, priv->xsk_pools); 1617 1618 /* If XDP prog is not installed or interface is down, return. */ 1619 if (!priv->xdp_prog || !netif_running(dev)) 1620 return 0; 1621 1622 err = gve_reg_xsk_pool(priv, dev, pool, qid); 1623 if (err) 1624 goto err_xsk_pool_dma_mapped; 1625 1626 /* Stop and start RDA queues to repost buffers. */ 1627 if (!gve_is_qpl(priv)) { 1628 err = gve_configure_rings_xdp(priv, priv->rx_cfg.num_queues); 1629 if (err) 1630 goto err_xsk_pool_registered; 1631 } 1632 return 0; 1633 1634 err_xsk_pool_registered: 1635 gve_unreg_xsk_pool(priv, qid); 1636 err_xsk_pool_dma_mapped: 1637 clear_bit(qid, priv->xsk_pools); 1638 xsk_pool_dma_unmap(pool, 1639 DMA_ATTR_SKIP_CPU_SYNC | 1640 DMA_ATTR_WEAK_ORDERING); 1641 return err; 1642 } 1643 1644 static int gve_xsk_pool_disable(struct net_device *dev, 1645 u16 qid) 1646 { 1647 struct gve_priv *priv = netdev_priv(dev); 1648 struct napi_struct *napi_rx; 1649 struct napi_struct *napi_tx; 1650 struct xsk_buff_pool *pool; 1651 int tx_qid; 1652 int err; 1653 1654 if (qid >= priv->rx_cfg.num_queues) 1655 return -EINVAL; 1656 1657 clear_bit(qid, priv->xsk_pools); 1658 1659 pool = xsk_get_pool_from_qid(dev, qid); 1660 if (pool) 1661 xsk_pool_dma_unmap(pool, 1662 DMA_ATTR_SKIP_CPU_SYNC | 1663 DMA_ATTR_WEAK_ORDERING); 1664 1665 if (!netif_running(dev) || !priv->tx_cfg.num_xdp_queues) 1666 return 0; 1667 1668 /* Stop and start RDA queues to repost buffers. */ 1669 if (!gve_is_qpl(priv) && priv->xdp_prog) { 1670 err = gve_configure_rings_xdp(priv, priv->rx_cfg.num_queues); 1671 if (err) 1672 return err; 1673 } 1674 1675 napi_rx = &priv->ntfy_blocks[priv->rx[qid].ntfy_id].napi; 1676 napi_disable_locked(napi_rx); /* make sure current rx poll is done */ 1677 1678 tx_qid = gve_xdp_tx_queue_id(priv, qid); 1679 napi_tx = &priv->ntfy_blocks[priv->tx[tx_qid].ntfy_id].napi; 1680 napi_disable_locked(napi_tx); /* make sure current tx poll is done */ 1681 1682 gve_unreg_xsk_pool(priv, qid); 1683 smp_mb(); /* Make sure it is visible to the workers on datapath */ 1684 1685 napi_enable_locked(napi_rx); 1686 napi_enable_locked(napi_tx); 1687 if (gve_is_gqi(priv)) { 1688 if (gve_rx_work_pending(&priv->rx[qid])) 1689 napi_schedule(napi_rx); 1690 1691 if (gve_tx_clean_pending(priv, &priv->tx[tx_qid])) 1692 napi_schedule(napi_tx); 1693 } 1694 1695 return 0; 1696 } 1697 1698 static int gve_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) 1699 { 1700 struct gve_priv *priv = netdev_priv(dev); 1701 struct napi_struct *napi; 1702 1703 if (!gve_get_napi_enabled(priv)) 1704 return -ENETDOWN; 1705 1706 if (queue_id >= priv->rx_cfg.num_queues || !priv->xdp_prog) 1707 return -EINVAL; 1708 1709 napi = &priv->ntfy_blocks[gve_rx_idx_to_ntfy(priv, queue_id)].napi; 1710 if (!napi_if_scheduled_mark_missed(napi)) { 1711 /* Call local_bh_enable to trigger SoftIRQ processing */ 1712 local_bh_disable(); 1713 napi_schedule(napi); 1714 local_bh_enable(); 1715 } 1716 1717 return 0; 1718 } 1719 1720 static int gve_verify_xdp_configuration(struct net_device *dev, 1721 struct netlink_ext_ack *extack) 1722 { 1723 struct gve_priv *priv = netdev_priv(dev); 1724 u16 max_xdp_mtu; 1725 1726 if (dev->features & NETIF_F_GRO_HW) { 1727 NL_SET_ERR_MSG_MOD(extack, 1728 "XDP is not supported when HW-GRO is on."); 1729 return -EOPNOTSUPP; 1730 } 1731 1732 if (priv->header_split_enabled) { 1733 NL_SET_ERR_MSG_MOD(extack, 1734 "XDP is not supported when header-data split is enabled."); 1735 return -EOPNOTSUPP; 1736 } 1737 1738 if (priv->rx_cfg.packet_buffer_size != SZ_2K) { 1739 NL_SET_ERR_MSG_FMT_MOD(extack, 1740 "XDP is not supported for Rx buf len %d, only %d supported.", 1741 priv->rx_cfg.packet_buffer_size, SZ_2K); 1742 return -EOPNOTSUPP; 1743 } 1744 1745 max_xdp_mtu = priv->rx_cfg.packet_buffer_size - sizeof(struct ethhdr); 1746 if (priv->queue_format == GVE_GQI_QPL_FORMAT) 1747 max_xdp_mtu -= GVE_RX_PAD; 1748 1749 if (dev->mtu > max_xdp_mtu) { 1750 NL_SET_ERR_MSG_FMT_MOD(extack, 1751 "XDP is not supported for mtu %d.", 1752 dev->mtu); 1753 return -EOPNOTSUPP; 1754 } 1755 1756 if (priv->rx_cfg.num_queues != priv->tx_cfg.num_queues || 1757 (2 * priv->tx_cfg.num_queues > priv->tx_cfg.max_queues)) { 1758 netdev_warn(dev, 1759 "XDP load failed: The number of configured RX queues %d should be equal to the number of configured TX queues %d and the number of configured RX/TX queues should be less than or equal to half the maximum number of RX/TX queues %d.", 1760 priv->rx_cfg.num_queues, priv->tx_cfg.num_queues, 1761 priv->tx_cfg.max_queues); 1762 NL_SET_ERR_MSG_MOD(extack, 1763 "XDP load failed: The number of configured RX queues should be equal to the number of configured TX queues and the number of configured RX/TX queues should be less than or equal to half the maximum number of RX/TX queues"); 1764 return -EINVAL; 1765 } 1766 return 0; 1767 } 1768 1769 static int gve_xdp(struct net_device *dev, struct netdev_bpf *xdp) 1770 { 1771 struct gve_priv *priv = netdev_priv(dev); 1772 int err; 1773 1774 err = gve_verify_xdp_configuration(dev, xdp->extack); 1775 if (err) 1776 return err; 1777 switch (xdp->command) { 1778 case XDP_SETUP_PROG: 1779 return gve_set_xdp(priv, xdp->prog, xdp->extack); 1780 case XDP_SETUP_XSK_POOL: 1781 if (xdp->xsk.pool) 1782 return gve_xsk_pool_enable(dev, xdp->xsk.pool, xdp->xsk.queue_id); 1783 else 1784 return gve_xsk_pool_disable(dev, xdp->xsk.queue_id); 1785 default: 1786 return -EINVAL; 1787 } 1788 } 1789 1790 int gve_init_rss_config(struct gve_priv *priv, u16 num_queues) 1791 { 1792 const struct gve_ctrl_ops *ops = priv->ctrl_ops; 1793 struct gve_rss_config *rss_config = &priv->rss_config; 1794 struct ethtool_rxfh_param rxfh = {0}; 1795 u16 i; 1796 1797 if (!priv->cache_rss_config) 1798 return 0; 1799 1800 for (i = 0; i < priv->rss_lut_size; i++) 1801 rss_config->hash_lut[i] = 1802 ethtool_rxfh_indir_default(i, num_queues); 1803 1804 netdev_rss_key_fill(rss_config->hash_key, priv->rss_key_size); 1805 1806 rxfh.hfunc = ETH_RSS_HASH_TOP; 1807 1808 return ops->configure_rss(priv, &rxfh); 1809 } 1810 1811 int gve_flow_rules_reset(struct gve_priv *priv) 1812 { 1813 const struct gve_ctrl_ops *ops = priv->ctrl_ops; 1814 1815 if (!priv->max_flow_rules) 1816 return 0; 1817 1818 return ops->reset_flow_rules(priv); 1819 } 1820 1821 int gve_adjust_config(struct gve_priv *priv, 1822 struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, 1823 struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) 1824 { 1825 int err; 1826 1827 /* Allocate resources for the new configuration */ 1828 err = gve_queues_mem_alloc(priv, tx_alloc_cfg, rx_alloc_cfg); 1829 if (err) { 1830 netif_err(priv, drv, priv->dev, 1831 "Adjust config failed to alloc new queues"); 1832 return err; 1833 } 1834 1835 /* Teardown the device and free existing resources */ 1836 err = gve_close(priv->dev); 1837 if (err) { 1838 netif_err(priv, drv, priv->dev, 1839 "Adjust config failed to close old queues"); 1840 gve_queues_mem_free(priv, tx_alloc_cfg, rx_alloc_cfg); 1841 return err; 1842 } 1843 1844 /* Bring the device back up again with the new resources. */ 1845 err = gve_queues_start(priv, tx_alloc_cfg, rx_alloc_cfg); 1846 if (err) { 1847 netif_err(priv, drv, priv->dev, 1848 "Adjust config failed to start new queues, !!! DISABLING ALL QUEUES !!!\n"); 1849 gve_queues_mem_remove(priv); 1850 gve_turndown(priv); 1851 return err; 1852 } 1853 1854 return 0; 1855 } 1856 1857 int gve_adjust_queues(struct gve_priv *priv, 1858 struct gve_rx_queue_config new_rx_config, 1859 struct gve_tx_queue_config new_tx_config, 1860 bool reset_rss) 1861 { 1862 struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; 1863 struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; 1864 int err; 1865 1866 gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1867 1868 /* Relay the new config from ethtool */ 1869 tx_alloc_cfg.qcfg = &new_tx_config; 1870 rx_alloc_cfg.qcfg_tx = &new_tx_config; 1871 rx_alloc_cfg.qcfg_rx = &new_rx_config; 1872 rx_alloc_cfg.reset_rss = reset_rss; 1873 1874 if (netif_running(priv->dev)) { 1875 err = gve_adjust_config(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1876 return err; 1877 } 1878 /* Set the config for the next up. */ 1879 if (reset_rss) { 1880 err = gve_init_rss_config(priv, new_rx_config.num_queues); 1881 if (err) 1882 return err; 1883 } 1884 priv->tx_cfg = new_tx_config; 1885 priv->rx_cfg = new_rx_config; 1886 1887 return 0; 1888 } 1889 1890 static void gve_turndown(struct gve_priv *priv) 1891 { 1892 int idx; 1893 1894 if (netif_carrier_ok(priv->dev)) 1895 netif_carrier_off(priv->dev); 1896 1897 if (!gve_get_napi_enabled(priv)) 1898 return; 1899 1900 /* Disable napi to prevent more work from coming in */ 1901 for (idx = 0; idx < gve_num_tx_queues(priv); idx++) { 1902 int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx); 1903 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 1904 1905 if (!gve_tx_was_added_to_block(priv, idx)) 1906 continue; 1907 1908 if (idx < priv->tx_cfg.num_queues) 1909 netif_queue_set_napi(priv->dev, idx, 1910 NETDEV_QUEUE_TYPE_TX, NULL); 1911 1912 napi_disable_locked(&block->napi); 1913 } 1914 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { 1915 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx); 1916 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 1917 1918 if (!gve_rx_was_added_to_block(priv, idx)) 1919 continue; 1920 1921 netif_queue_set_napi(priv->dev, idx, NETDEV_QUEUE_TYPE_RX, 1922 NULL); 1923 napi_disable_locked(&block->napi); 1924 } 1925 1926 /* Stop tx queues */ 1927 netif_tx_disable(priv->dev); 1928 1929 xdp_features_clear_redirect_target_locked(priv->dev); 1930 1931 gve_clear_napi_enabled(priv); 1932 gve_clear_report_stats(priv); 1933 1934 /* Make sure that all traffic is finished processing. */ 1935 synchronize_net(); 1936 } 1937 1938 static void gve_turnup(struct gve_priv *priv) 1939 { 1940 int idx; 1941 1942 /* Start the tx queues */ 1943 netif_tx_start_all_queues(priv->dev); 1944 1945 /* Enable napi and unmask interrupts for all queues */ 1946 for (idx = 0; idx < gve_num_tx_queues(priv); idx++) { 1947 int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx); 1948 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 1949 1950 if (!gve_tx_was_added_to_block(priv, idx)) 1951 continue; 1952 1953 napi_enable_locked(&block->napi); 1954 1955 if (idx < priv->tx_cfg.num_queues) 1956 netif_queue_set_napi(priv->dev, idx, 1957 NETDEV_QUEUE_TYPE_TX, 1958 &block->napi); 1959 1960 if (gve_is_gqi(priv)) { 1961 iowrite32be(0, gve_irq_doorbell(priv, block)); 1962 } else { 1963 gve_set_itr_coalesce_usecs_dqo(priv, block, 1964 priv->tx_coalesce_usecs); 1965 } 1966 1967 /* Any descs written by the NIC before this barrier will be 1968 * handled by the one-off napi schedule below. Whereas any 1969 * descs after the barrier will generate interrupts. 1970 */ 1971 mb(); 1972 napi_schedule(&block->napi); 1973 } 1974 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { 1975 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx); 1976 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 1977 1978 if (!gve_rx_was_added_to_block(priv, idx)) 1979 continue; 1980 1981 napi_enable_locked(&block->napi); 1982 netif_queue_set_napi(priv->dev, idx, NETDEV_QUEUE_TYPE_RX, 1983 &block->napi); 1984 1985 if (gve_is_gqi(priv)) { 1986 iowrite32be(0, gve_irq_doorbell(priv, block)); 1987 } else { 1988 gve_set_itr_coalesce_usecs_dqo(priv, block, 1989 priv->rx_coalesce_usecs); 1990 } 1991 1992 /* Any descs written by the NIC before this barrier will be 1993 * handled by the one-off napi schedule below. Whereas any 1994 * descs after the barrier will generate interrupts. 1995 */ 1996 mb(); 1997 napi_schedule(&block->napi); 1998 } 1999 2000 if (priv->tx_cfg.num_xdp_queues && gve_supports_xdp_xmit(priv)) 2001 xdp_features_set_redirect_target_locked(priv->dev, false); 2002 2003 gve_set_napi_enabled(priv); 2004 } 2005 2006 static void gve_turnup_and_check_status(struct gve_priv *priv) 2007 { 2008 const struct gve_ctrl_ops *ops = priv->ctrl_ops; 2009 2010 gve_turnup(priv); 2011 ops->report_link_status(priv); 2012 gve_handle_link_status(priv); 2013 } 2014 2015 static struct gve_notify_block *gve_get_tx_notify_block(struct gve_priv *priv, 2016 unsigned int txqueue) 2017 { 2018 u32 ntfy_idx; 2019 2020 if (txqueue > priv->tx_cfg.num_queues) 2021 return NULL; 2022 2023 ntfy_idx = gve_tx_idx_to_ntfy(priv, txqueue); 2024 if (ntfy_idx >= priv->num_ntfy_blks) 2025 return NULL; 2026 2027 return &priv->ntfy_blocks[ntfy_idx]; 2028 } 2029 2030 static bool gve_tx_timeout_try_q_kick(struct gve_priv *priv, 2031 unsigned int txqueue) 2032 { 2033 struct gve_notify_block *block; 2034 u32 current_time; 2035 2036 block = gve_get_tx_notify_block(priv, txqueue); 2037 2038 if (!block) 2039 return false; 2040 2041 current_time = jiffies_to_msecs(jiffies); 2042 if (block->tx->last_kick_msec + MIN_TX_TIMEOUT_GAP > current_time) 2043 return false; 2044 2045 netdev_info(priv->dev, "Kicking queue %d", txqueue); 2046 napi_schedule(&block->napi); 2047 block->tx->last_kick_msec = current_time; 2048 return true; 2049 } 2050 2051 static void gve_tx_timeout(struct net_device *dev, unsigned int txqueue) 2052 { 2053 struct gve_notify_block *block; 2054 struct gve_priv *priv; 2055 2056 netdev_info(dev, "Timeout on tx queue, %d", txqueue); 2057 priv = netdev_priv(dev); 2058 2059 if (!gve_tx_timeout_try_q_kick(priv, txqueue)) 2060 gve_schedule_reset(priv); 2061 2062 block = gve_get_tx_notify_block(priv, txqueue); 2063 if (block) 2064 block->tx->queue_timeout++; 2065 priv->tx_timeo_cnt++; 2066 } 2067 2068 /* Header split is only supported on DQ RDA queue format. If XDP is enabled, 2069 * header split is not allowed. 2070 */ 2071 bool gve_header_split_supported(const struct gve_priv *priv) 2072 { 2073 return priv->header_buf_size && 2074 priv->queue_format == GVE_DQO_RDA_FORMAT && !priv->xdp_prog; 2075 } 2076 2077 int gve_set_rx_buf_len_config(struct gve_priv *priv, u32 rx_buf_len, 2078 struct netlink_ext_ack *extack, 2079 struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) 2080 { 2081 u32 old_rx_buf_len = rx_alloc_cfg->packet_buffer_size; 2082 2083 if (rx_buf_len == old_rx_buf_len) 2084 return 0; 2085 2086 /* device options may not always contain support for 4K buffers */ 2087 if (!gve_is_dqo(priv) || priv->max_rx_buffer_size < SZ_4K) { 2088 NL_SET_ERR_MSG_MOD(extack, 2089 "Modifying Rx buf len is not supported"); 2090 return -EOPNOTSUPP; 2091 } 2092 2093 if (priv->xdp_prog && rx_buf_len != SZ_2K) { 2094 NL_SET_ERR_MSG_MOD(extack, 2095 "Rx buf len can only be 2048 when XDP is on"); 2096 return -EINVAL; 2097 } 2098 2099 if (rx_buf_len != SZ_2K && rx_buf_len != SZ_4K) { 2100 NL_SET_ERR_MSG_MOD(extack, 2101 "Rx buf len can only be 2048 or 4096"); 2102 return -EINVAL; 2103 } 2104 rx_alloc_cfg->packet_buffer_size = rx_buf_len; 2105 2106 return 0; 2107 } 2108 2109 int gve_set_hsplit_config(struct gve_priv *priv, u8 tcp_data_split, 2110 struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) 2111 { 2112 bool enable_hdr_split; 2113 2114 if (tcp_data_split == ETHTOOL_TCP_DATA_SPLIT_UNKNOWN) 2115 return 0; 2116 2117 if (!gve_header_split_supported(priv)) { 2118 dev_err(&priv->pdev->dev, "Header-split not supported\n"); 2119 return -EOPNOTSUPP; 2120 } 2121 2122 if (tcp_data_split == ETHTOOL_TCP_DATA_SPLIT_ENABLED) 2123 enable_hdr_split = true; 2124 else 2125 enable_hdr_split = false; 2126 2127 if (enable_hdr_split == priv->header_split_enabled) 2128 return 0; 2129 2130 rx_alloc_cfg->enable_header_split = enable_hdr_split; 2131 2132 return 0; 2133 } 2134 2135 static int gve_set_features(struct net_device *netdev, 2136 netdev_features_t features) 2137 { 2138 const netdev_features_t orig_features = netdev->features; 2139 struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; 2140 struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; 2141 struct gve_priv *priv = netdev_priv(netdev); 2142 int err; 2143 2144 gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); 2145 2146 if ((netdev->features & NETIF_F_GRO_HW) != 2147 (features & NETIF_F_GRO_HW)) { 2148 netdev->features ^= NETIF_F_GRO_HW; 2149 if (priv->xdp_prog && (netdev->features & NETIF_F_GRO_HW)) { 2150 netdev_warn(netdev, 2151 "HW-GRO is not supported when XDP is on."); 2152 err = -EOPNOTSUPP; 2153 goto revert_features; 2154 } 2155 if (netif_running(netdev)) { 2156 err = gve_adjust_config(priv, &tx_alloc_cfg, &rx_alloc_cfg); 2157 if (err) 2158 goto revert_features; 2159 } 2160 } 2161 if ((netdev->features & NETIF_F_NTUPLE) && !(features & NETIF_F_NTUPLE)) { 2162 err = gve_flow_rules_reset(priv); 2163 if (err) { 2164 if (err == -ETIME) 2165 gve_schedule_reset(priv); 2166 goto revert_features; 2167 } 2168 } 2169 2170 return 0; 2171 2172 revert_features: 2173 netdev->features = orig_features; 2174 return err; 2175 } 2176 2177 static int gve_get_ts_config(struct net_device *dev, 2178 struct kernel_hwtstamp_config *kernel_config) 2179 { 2180 struct gve_priv *priv = netdev_priv(dev); 2181 2182 *kernel_config = priv->ts_config; 2183 return 0; 2184 } 2185 2186 static int gve_set_ts_config(struct net_device *dev, 2187 struct kernel_hwtstamp_config *kernel_config, 2188 struct netlink_ext_ack *extack) 2189 { 2190 struct gve_priv *priv = netdev_priv(dev); 2191 2192 if (kernel_config->tx_type != HWTSTAMP_TX_OFF) { 2193 NL_SET_ERR_MSG_MOD(extack, "TX timestamping is not supported"); 2194 return -ERANGE; 2195 } 2196 2197 if (kernel_config->rx_filter != HWTSTAMP_FILTER_NONE) { 2198 if (!gve_is_clock_enabled(priv)) { 2199 NL_SET_ERR_MSG_MOD(extack, 2200 "RX timestamping is not supported"); 2201 kernel_config->rx_filter = HWTSTAMP_FILTER_NONE; 2202 return -EOPNOTSUPP; 2203 } 2204 2205 kernel_config->rx_filter = HWTSTAMP_FILTER_ALL; 2206 } 2207 2208 priv->ts_config.rx_filter = kernel_config->rx_filter; 2209 2210 return 0; 2211 } 2212 2213 static const struct net_device_ops gve_netdev_ops = { 2214 .ndo_start_xmit = gve_start_xmit, 2215 .ndo_features_check = gve_features_check, 2216 .ndo_open = gve_open, 2217 .ndo_stop = gve_close, 2218 .ndo_get_stats64 = gve_get_stats, 2219 .ndo_tx_timeout = gve_tx_timeout, 2220 .ndo_set_features = gve_set_features, 2221 .ndo_bpf = gve_xdp, 2222 .ndo_xdp_xmit = gve_xdp_xmit, 2223 .ndo_xsk_wakeup = gve_xsk_wakeup, 2224 .ndo_hwtstamp_get = gve_get_ts_config, 2225 .ndo_hwtstamp_set = gve_set_ts_config, 2226 }; 2227 2228 static void gve_handle_status(struct gve_priv *priv, u32 status) 2229 { 2230 if (GVE_DEVICE_STATUS_RESET_MASK & status) { 2231 dev_info(&priv->pdev->dev, "Device requested reset.\n"); 2232 gve_set_do_reset(priv); 2233 } 2234 if (GVE_DEVICE_STATUS_REPORT_STATS_MASK & status) { 2235 priv->stats_report_trigger_cnt++; 2236 gve_set_do_report_stats(priv); 2237 } 2238 } 2239 2240 static void gve_handle_reset(struct gve_priv *priv) 2241 { 2242 /* A service task will be scheduled at the end of probe to catch any 2243 * resets that need to happen, and we don't want to reset until 2244 * probe is done. 2245 */ 2246 if (gve_get_probe_in_progress(priv)) 2247 return; 2248 2249 if (gve_get_do_reset(priv)) { 2250 rtnl_lock(); 2251 netdev_lock(priv->dev); 2252 if (gve_get_do_reset(priv)) 2253 gve_reset(priv, false); 2254 netdev_unlock(priv->dev); 2255 rtnl_unlock(); 2256 } 2257 } 2258 2259 void gve_handle_report_stats(struct gve_priv *priv) 2260 { 2261 struct stats *stats = priv->stats_report->stats; 2262 int idx, stats_idx = 0; 2263 unsigned int start = 0; 2264 u64 tx_bytes; 2265 2266 if (!gve_get_report_stats(priv)) 2267 return; 2268 2269 be64_add_cpu(&priv->stats_report->written_count, 1); 2270 /* tx stats */ 2271 if (priv->tx) { 2272 for (idx = 0; idx < gve_num_tx_queues(priv); idx++) { 2273 u32 last_completion = 0; 2274 u32 tx_frames = 0; 2275 2276 /* DQO doesn't currently support these metrics. */ 2277 if (gve_is_gqi(priv)) { 2278 last_completion = priv->tx[idx].done; 2279 tx_frames = priv->tx[idx].req; 2280 } 2281 2282 do { 2283 start = u64_stats_fetch_begin(&priv->tx[idx].statss); 2284 tx_bytes = priv->tx[idx].bytes_done; 2285 } while (u64_stats_fetch_retry(&priv->tx[idx].statss, start)); 2286 stats[stats_idx++] = (struct stats) { 2287 .stat_name = cpu_to_be32(TX_WAKE_CNT), 2288 .value = cpu_to_be64(priv->tx[idx].wake_queue), 2289 .queue_id = cpu_to_be32(idx), 2290 }; 2291 stats[stats_idx++] = (struct stats) { 2292 .stat_name = cpu_to_be32(TX_STOP_CNT), 2293 .value = cpu_to_be64(priv->tx[idx].stop_queue), 2294 .queue_id = cpu_to_be32(idx), 2295 }; 2296 stats[stats_idx++] = (struct stats) { 2297 .stat_name = cpu_to_be32(TX_FRAMES_SENT), 2298 .value = cpu_to_be64(tx_frames), 2299 .queue_id = cpu_to_be32(idx), 2300 }; 2301 stats[stats_idx++] = (struct stats) { 2302 .stat_name = cpu_to_be32(TX_BYTES_SENT), 2303 .value = cpu_to_be64(tx_bytes), 2304 .queue_id = cpu_to_be32(idx), 2305 }; 2306 stats[stats_idx++] = (struct stats) { 2307 .stat_name = cpu_to_be32(TX_LAST_COMPLETION_PROCESSED), 2308 .value = cpu_to_be64(last_completion), 2309 .queue_id = cpu_to_be32(idx), 2310 }; 2311 stats[stats_idx++] = (struct stats) { 2312 .stat_name = cpu_to_be32(TX_TIMEOUT_CNT), 2313 .value = cpu_to_be64(priv->tx[idx].queue_timeout), 2314 .queue_id = cpu_to_be32(idx), 2315 }; 2316 } 2317 } 2318 /* rx stats */ 2319 if (priv->rx) { 2320 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { 2321 stats[stats_idx++] = (struct stats) { 2322 .stat_name = cpu_to_be32(RX_NEXT_EXPECTED_SEQUENCE), 2323 .value = cpu_to_be64(priv->rx[idx].desc.seqno), 2324 .queue_id = cpu_to_be32(idx), 2325 }; 2326 stats[stats_idx++] = (struct stats) { 2327 .stat_name = cpu_to_be32(RX_BUFFERS_POSTED), 2328 .value = cpu_to_be64(priv->rx[idx].fill_cnt), 2329 .queue_id = cpu_to_be32(idx), 2330 }; 2331 } 2332 } 2333 } 2334 2335 /* Handle NIC status register changes, reset requests and report stats */ 2336 static void gve_service_task(struct work_struct *work) 2337 { 2338 struct gve_priv *priv = container_of(work, struct gve_priv, 2339 service_task); 2340 const struct gve_ctrl_ops *ops = priv->ctrl_ops; 2341 u32 status = ioread32be(&priv->reg_bar0->device_status); 2342 2343 gve_handle_status(priv, status); 2344 2345 gve_handle_reset(priv); 2346 ops->report_link_status(priv); 2347 gve_handle_link_status(priv); 2348 } 2349 2350 static void gve_set_netdev_xdp_features(struct gve_priv *priv) 2351 { 2352 xdp_features_t xdp_features; 2353 2354 if (priv->queue_format == GVE_GQI_QPL_FORMAT) { 2355 xdp_features = NETDEV_XDP_ACT_BASIC; 2356 xdp_features |= NETDEV_XDP_ACT_REDIRECT; 2357 xdp_features |= NETDEV_XDP_ACT_XSK_ZEROCOPY; 2358 } else if (priv->queue_format == GVE_DQO_RDA_FORMAT) { 2359 xdp_features = NETDEV_XDP_ACT_BASIC; 2360 xdp_features |= NETDEV_XDP_ACT_REDIRECT; 2361 xdp_features |= NETDEV_XDP_ACT_XSK_ZEROCOPY; 2362 } else { 2363 xdp_features = 0; 2364 } 2365 2366 xdp_set_features_flag_locked(priv->dev, xdp_features); 2367 } 2368 2369 static const struct xdp_metadata_ops gve_xdp_metadata_ops = { 2370 .xmo_rx_timestamp = gve_xdp_rx_timestamp, 2371 }; 2372 2373 static void gve_set_desc_cnt(struct gve_priv *priv) 2374 { 2375 struct gve_device_info *device_info = &priv->device_info; 2376 2377 priv->tx_desc_cnt = device_info->default_tx_ring_size; 2378 priv->rx_desc_cnt = device_info->default_rx_ring_size; 2379 priv->max_tx_desc_cnt = device_info->max_tx_ring_size; 2380 priv->max_rx_desc_cnt = device_info->max_rx_ring_size; 2381 priv->min_tx_desc_cnt = device_info->min_tx_ring_size; 2382 priv->min_rx_desc_cnt = device_info->min_rx_ring_size; 2383 } 2384 2385 static void gve_set_queue_properties(struct gve_priv *priv) 2386 { 2387 struct gve_device_info *device_info = &priv->device_info; 2388 2389 gve_set_desc_cnt(priv); 2390 priv->max_registered_pages = device_info->max_registered_pages; 2391 priv->tx_pages_per_qpl = device_info->tx_pages_per_qpl; 2392 } 2393 2394 static int gve_set_mtu(struct gve_priv *priv) 2395 { 2396 struct gve_device_info *device_info = &priv->device_info; 2397 u16 mtu; 2398 2399 mtu = device_info->max_mtu; 2400 if (mtu < ETH_MIN_MTU) { 2401 dev_err(&priv->pdev->dev, "MTU %d below minimum MTU\n", mtu); 2402 return -EINVAL; 2403 } 2404 priv->dev->max_mtu = mtu; 2405 priv->dev->mtu = mtu; 2406 2407 return 0; 2408 } 2409 2410 static void gve_set_mac(struct gve_priv *priv) 2411 { 2412 struct gve_device_info *device_info = &priv->device_info; 2413 u8 *mac; 2414 2415 mac = device_info->mac; 2416 eth_hw_addr_set(priv->dev, mac); 2417 dev_info(&priv->pdev->dev, "MAC addr: %pM\n", mac); 2418 } 2419 2420 static void gve_set_buf_sizes(struct gve_priv *priv) 2421 { 2422 struct gve_device_info *device_info = &priv->device_info; 2423 2424 if (device_info->max_rx_buffer_size) 2425 priv->max_rx_buffer_size = device_info->max_rx_buffer_size; 2426 2427 if (gve_is_dqo(priv) && 2428 priv->max_rx_buffer_size > GVE_DEFAULT_RX_BUFFER_SIZE) 2429 priv->rx_cfg.packet_buffer_size = priv->max_rx_buffer_size; 2430 2431 if (device_info->header_buf_size) 2432 priv->header_buf_size = device_info->header_buf_size; 2433 } 2434 2435 static int gve_setup_device(struct gve_priv *priv) 2436 { 2437 int err; 2438 2439 priv->num_registered_pages = 0; 2440 2441 gve_set_netdev_xdp_features(priv); 2442 if (!gve_is_gqi(priv)) 2443 priv->dev->xdp_metadata_ops = &gve_xdp_metadata_ops; 2444 2445 err = gve_alloc_control_plane_resources(priv); 2446 if (err) 2447 return err; 2448 2449 err = gve_setup_control_plane_resources(priv); 2450 if (err) 2451 return err; 2452 2453 err = gve_setup_notify_blocks(priv); 2454 if (err) 2455 return err; 2456 return 0; 2457 } 2458 2459 static const struct gve_ctrl_ops gve_adminq_ops = { 2460 .map_db_bar = gve_adminq_map_db_bar, 2461 .unmap_db_bar = gve_adminq_unmap_db_bar, 2462 .set_num_queues = gve_adminq_set_num_queues, 2463 .set_num_ntfy_blks = gve_adminq_set_num_ntfy_blks, 2464 .get_ptype_map = gve_adminq_get_ptype_map_dqo, 2465 .reset_flow_rules = gve_adminq_reset_flow_rules, 2466 .setup_stats_report = gve_adminq_report_stats, 2467 .configure_rss = gve_adminq_configure_rss, 2468 .request_db_info = gve_adminq_request_db_info, 2469 .release_db_resources = gve_adminq_release_db_resources, 2470 .setup_mgmt_irq = gve_adminq_setup_mgmt_irq, 2471 .teardown_mgmt_irq = gve_adminq_teardown_mgmt_irq, 2472 .create_queues = gve_adminq_create_queues, 2473 .destroy_queues = gve_adminq_destroy_queues, 2474 .report_link_status = gve_adminq_report_link_status, 2475 .report_link_speed = gve_adminq_report_link_speed, 2476 }; 2477 2478 static int gve_init_priv(struct gve_priv *priv) 2479 { 2480 struct gve_device_info *device_info = &priv->device_info; 2481 int err; 2482 2483 priv->queue_format = priv->device_info.queue_format; 2484 2485 err = priv->ctrl_ops->set_num_ntfy_blks(priv); 2486 if (err) { 2487 dev_err(&priv->pdev->dev, 2488 "Could not setup notify blocks: err=%d\n", err); 2489 return err; 2490 } 2491 2492 priv->ctrl_ops->set_num_queues(priv); 2493 dev_info(&priv->pdev->dev, "TX queues %d, RX queues %d\n", 2494 priv->tx_cfg.num_queues, priv->rx_cfg.num_queues); 2495 dev_info(&priv->pdev->dev, "Max TX queues %d, Max RX queues %d\n", 2496 priv->tx_cfg.max_queues, priv->rx_cfg.max_queues); 2497 2498 if (gve_is_dqo(priv)) { 2499 /* DQO supports HW-GRO and UDP_GSO */ 2500 u64 additional_features = NETIF_F_GRO_HW | NETIF_F_GSO_UDP_L4; 2501 2502 priv->dev->hw_features |= additional_features; 2503 priv->dev->features |= additional_features; 2504 2505 priv->tx_coalesce_usecs = GVE_TX_IRQ_RATELIMIT_US_DQO; 2506 priv->rx_coalesce_usecs = GVE_RX_IRQ_RATELIMIT_US_DQO; 2507 2508 /* Big TCP is only supported on DQO */ 2509 netif_set_tso_max_size(priv->dev, GVE_DQO_TX_MAX); 2510 } 2511 2512 if (gve_set_mtu(priv)) 2513 return -EINVAL; 2514 2515 priv->num_event_counters = device_info->num_event_counters; 2516 2517 gve_set_mac(priv); 2518 2519 gve_set_queue_properties(priv); 2520 priv->modify_ring_size_enabled = device_info->modify_ring_size_enabled; 2521 2522 gve_set_buf_sizes(priv); 2523 2524 priv->max_flow_rules = device_info->max_flow_rules; 2525 if (priv->max_flow_rules) 2526 priv->dev->hw_features |= NETIF_F_NTUPLE; 2527 2528 priv->rss_key_size = device_info->rss_key_size; 2529 priv->rss_lut_size = device_info->rss_lut_size; 2530 priv->cache_rss_config = device_info->cache_rss_config; 2531 2532 priv->numa_node = dev_to_node(&priv->pdev->dev); 2533 priv->tx_cfg.num_xdp_queues = 0; 2534 priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK; 2535 priv->ts_config.tx_type = HWTSTAMP_TX_OFF; 2536 priv->ts_config.rx_filter = HWTSTAMP_FILTER_NONE; 2537 priv->nic_timestamp_supported = device_info->nic_timestamp_supported; 2538 return 0; 2539 } 2540 2541 static int gve_recover(struct gve_priv *priv, bool setup_queues) 2542 { 2543 int err; 2544 2545 err = gve_adminq_init(priv); 2546 if (err) { 2547 dev_err(&priv->pdev->dev, 2548 "Failed to alloc admin queue: err=%d\n", err); 2549 goto teardown_device; 2550 } 2551 2552 err = gve_adminq_verify_driver_compatibility(priv); 2553 if (err) { 2554 dev_err(&priv->pdev->dev, 2555 "Could not verify driver compatibility: err=%d\n", err); 2556 goto teardown_device; 2557 } 2558 2559 err = gve_setup_device(priv); 2560 if (err) 2561 goto teardown_device; 2562 2563 if (setup_queues) { 2564 /* On failure, hold on to the control plane to give a 2565 * chance for the queues to be brought up later. 2566 */ 2567 err = gve_open(priv->dev); 2568 if (err) { 2569 dev_err(&priv->pdev->dev, 2570 "Failed to start queues: err=%d, !!! DISABLING ALL QUEUES !!!\n", 2571 err); 2572 return err; 2573 } 2574 } 2575 2576 /* undo any detach from an earlier failure */ 2577 netif_device_attach(priv->dev); 2578 2579 return 0; 2580 2581 teardown_device: 2582 dev_err(&priv->pdev->dev, "Recover failed: err=%d, detaching device\n", 2583 err); 2584 netif_device_detach(priv->dev); 2585 gve_teardown_device(priv); 2586 return err; 2587 } 2588 2589 int gve_reset(struct gve_priv *priv, bool skip_queue_setup) 2590 { 2591 bool was_up = netif_running(priv->dev); 2592 int err; 2593 2594 if (gve_get_reset_in_progress(priv)) 2595 return 0; 2596 2597 dev_info(&priv->pdev->dev, "Performing reset\n"); 2598 gve_clear_do_reset(priv); 2599 gve_set_reset_in_progress(priv); 2600 2601 if (was_up) { 2602 gve_turndown(priv); 2603 if (gve_get_device_rings_ok(priv)) { 2604 gve_clear_device_rings_ok(priv); 2605 gve_destroy_rings(priv); 2606 gve_unregister_qpls(priv); 2607 } 2608 } 2609 2610 disable_work(&priv->service_task); 2611 gve_reset_device(priv); 2612 gve_queues_stop(priv); 2613 gve_queues_mem_remove(priv); 2614 gve_teardown_notify_blocks(priv); 2615 gve_free_control_plane_resources(priv); 2616 2617 enable_work(&priv->service_task); 2618 err = gve_recover(priv, was_up && !skip_queue_setup); 2619 if (err) 2620 dev_info(&priv->pdev->dev, 2621 "Failed to recover in reset: %d\n", err); 2622 2623 gve_clear_reset_in_progress(priv); 2624 priv->reset_cnt++; 2625 priv->interface_up_cnt = 0; 2626 priv->interface_down_cnt = 0; 2627 priv->stats_report_trigger_cnt = 0; 2628 return err; 2629 } 2630 2631 void gve_adminq_write_version(u8 __iomem *driver_version_register) 2632 { 2633 const char *c = gve_version_prefix; 2634 2635 while (*c) { 2636 writeb(*c, driver_version_register); 2637 c++; 2638 } 2639 2640 c = gve_version_str; 2641 while (*c) { 2642 writeb(*c, driver_version_register); 2643 c++; 2644 } 2645 writeb('\n', driver_version_register); 2646 } 2647 2648 static int gve_rx_queue_stop(struct net_device *dev, void *per_q_mem, int idx) 2649 { 2650 struct gve_priv *priv = netdev_priv(dev); 2651 struct gve_rx_ring *gve_per_q_mem; 2652 int err; 2653 2654 if (!priv->rx) 2655 return -EAGAIN; 2656 2657 /* Destroying queue 0 while other queues exist is not supported in DQO */ 2658 if (!gve_is_gqi(priv) && idx == 0) 2659 return -ERANGE; 2660 2661 /* Single-queue destruction requires quiescence on all queues */ 2662 gve_turndown(priv); 2663 2664 /* This failure will trigger a reset - no need to clean up */ 2665 err = gve_adminq_destroy_single_rx_queue(priv, idx); 2666 if (err) 2667 return err; 2668 2669 if (gve_is_qpl(priv)) { 2670 /* This failure will trigger a reset - no need to clean up */ 2671 err = gve_unregister_qpl(priv, gve_rx_get_qpl(priv, idx)); 2672 if (err) 2673 return err; 2674 } 2675 2676 gve_rx_stop_ring(priv, idx); 2677 2678 /* Turn the unstopped queues back up */ 2679 gve_turnup_and_check_status(priv); 2680 2681 gve_per_q_mem = (struct gve_rx_ring *)per_q_mem; 2682 *gve_per_q_mem = priv->rx[idx]; 2683 memset(&priv->rx[idx], 0, sizeof(priv->rx[idx])); 2684 return 0; 2685 } 2686 2687 static void gve_rx_queue_mem_free(struct net_device *dev, void *per_q_mem) 2688 { 2689 struct gve_priv *priv = netdev_priv(dev); 2690 struct gve_rx_alloc_rings_cfg cfg = {0}; 2691 struct gve_rx_ring *gve_per_q_mem; 2692 2693 gve_per_q_mem = (struct gve_rx_ring *)per_q_mem; 2694 gve_rx_get_curr_alloc_cfg(priv, &cfg); 2695 2696 if (gve_is_gqi(priv)) 2697 gve_rx_free_ring_gqi(priv, gve_per_q_mem, &cfg); 2698 else 2699 gve_rx_free_ring_dqo(priv, gve_per_q_mem, &cfg); 2700 } 2701 2702 static int gve_rx_queue_mem_alloc(struct net_device *dev, 2703 struct netdev_queue_config *qcfg, 2704 void *per_q_mem, int idx) 2705 { 2706 struct gve_priv *priv = netdev_priv(dev); 2707 struct gve_rx_alloc_rings_cfg cfg = {0}; 2708 struct gve_rx_ring *gve_per_q_mem; 2709 int err; 2710 2711 if (!priv->rx) 2712 return -EAGAIN; 2713 2714 gve_per_q_mem = (struct gve_rx_ring *)per_q_mem; 2715 gve_rx_get_curr_alloc_cfg(priv, &cfg); 2716 2717 if (gve_is_gqi(priv)) 2718 err = gve_rx_alloc_ring_gqi(priv, &cfg, gve_per_q_mem, idx); 2719 else 2720 err = gve_rx_alloc_ring_dqo(priv, &cfg, gve_per_q_mem, idx); 2721 2722 return err; 2723 } 2724 2725 static int gve_rx_queue_start(struct net_device *dev, 2726 struct netdev_queue_config *qcfg, 2727 void *per_q_mem, int idx) 2728 { 2729 struct gve_priv *priv = netdev_priv(dev); 2730 struct gve_rx_ring *gve_per_q_mem; 2731 int err; 2732 2733 if (!priv->rx) 2734 return -EAGAIN; 2735 2736 gve_per_q_mem = (struct gve_rx_ring *)per_q_mem; 2737 priv->rx[idx] = *gve_per_q_mem; 2738 2739 /* Single-queue creation requires quiescence on all queues */ 2740 gve_turndown(priv); 2741 2742 gve_rx_start_ring(priv, idx); 2743 2744 if (gve_is_qpl(priv)) { 2745 /* This failure will trigger a reset - no need to clean up */ 2746 err = gve_register_qpl(priv, gve_rx_get_qpl(priv, idx)); 2747 if (err) 2748 goto abort; 2749 } 2750 2751 /* This failure will trigger a reset - no need to clean up */ 2752 err = gve_adminq_create_single_rx_queue(priv, idx); 2753 if (err) 2754 goto abort; 2755 2756 if (gve_is_gqi(priv)) 2757 gve_rx_write_doorbell(priv, &priv->rx[idx]); 2758 else 2759 gve_rx_post_buffers_dqo(&priv->rx[idx]); 2760 2761 /* Turn the unstopped queues back up */ 2762 gve_turnup_and_check_status(priv); 2763 return 0; 2764 2765 abort: 2766 gve_rx_stop_ring(priv, idx); 2767 2768 /* All failures in this func result in a reset, by clearing the struct 2769 * at idx, we prevent a double free when that reset runs. The reset, 2770 * which needs the rtnl lock, will not run till this func returns and 2771 * its caller gives up the lock. 2772 */ 2773 memset(&priv->rx[idx], 0, sizeof(priv->rx[idx])); 2774 return err; 2775 } 2776 2777 static const struct netdev_queue_mgmt_ops gve_queue_mgmt_ops = { 2778 .ndo_queue_mem_size = sizeof(struct gve_rx_ring), 2779 .ndo_queue_mem_alloc = gve_rx_queue_mem_alloc, 2780 .ndo_queue_mem_free = gve_rx_queue_mem_free, 2781 .ndo_queue_start = gve_rx_queue_start, 2782 .ndo_queue_stop = gve_rx_queue_stop, 2783 }; 2784 2785 static void gve_get_rx_queue_stats(struct net_device *dev, int idx, 2786 struct netdev_queue_stats_rx *rx_stats) 2787 { 2788 struct gve_priv *priv = netdev_priv(dev); 2789 struct gve_rx_ring *rx = &priv->rx[idx]; 2790 unsigned int start; 2791 2792 do { 2793 start = u64_stats_fetch_begin(&rx->statss); 2794 rx_stats->packets = rx->rpackets; 2795 rx_stats->bytes = rx->rbytes; 2796 rx_stats->alloc_fail = rx->rx_skb_alloc_fail + 2797 rx->rx_buf_alloc_fail; 2798 } while (u64_stats_fetch_retry(&rx->statss, start)); 2799 } 2800 2801 static void gve_get_tx_queue_stats(struct net_device *dev, int idx, 2802 struct netdev_queue_stats_tx *tx_stats) 2803 { 2804 struct gve_priv *priv = netdev_priv(dev); 2805 struct gve_tx_ring *tx = &priv->tx[idx]; 2806 unsigned int start; 2807 2808 do { 2809 start = u64_stats_fetch_begin(&tx->statss); 2810 tx_stats->packets = tx->pkt_done; 2811 tx_stats->bytes = tx->bytes_done; 2812 } while (u64_stats_fetch_retry(&tx->statss, start)); 2813 } 2814 2815 static void gve_get_base_stats(struct net_device *dev, 2816 struct netdev_queue_stats_rx *rx, 2817 struct netdev_queue_stats_tx *tx) 2818 { 2819 rx->packets = 0; 2820 rx->bytes = 0; 2821 rx->alloc_fail = 0; 2822 2823 tx->packets = 0; 2824 tx->bytes = 0; 2825 } 2826 2827 static const struct netdev_stat_ops gve_stat_ops = { 2828 .get_queue_stats_rx = gve_get_rx_queue_stats, 2829 .get_queue_stats_tx = gve_get_tx_queue_stats, 2830 .get_base_stats = gve_get_base_stats, 2831 }; 2832 2833 static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 2834 { 2835 int max_tx_queues, max_rx_queues; 2836 struct net_device *dev; 2837 struct gve_registers __iomem *reg_bar; 2838 struct gve_priv *priv; 2839 int err; 2840 2841 err = pci_enable_device(pdev); 2842 if (err) 2843 return err; 2844 2845 err = pci_request_regions(pdev, gve_driver_name); 2846 if (err) 2847 goto abort_with_enabled; 2848 2849 pci_set_master(pdev); 2850 2851 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 2852 if (err) { 2853 dev_err(&pdev->dev, "Failed to set dma mask: err=%d\n", err); 2854 goto abort_with_pci_region; 2855 } 2856 2857 reg_bar = pci_iomap(pdev, GVE_REGISTER_BAR, 0); 2858 if (!reg_bar) { 2859 dev_err(&pdev->dev, "Failed to map pci bar!\n"); 2860 err = -ENOMEM; 2861 goto abort_with_pci_region; 2862 } 2863 2864 /* Get max queues to alloc etherdev */ 2865 max_tx_queues = ioread32be(®_bar->max_tx_queues); 2866 max_rx_queues = ioread32be(®_bar->max_rx_queues); 2867 /* Alloc and setup the netdev and priv */ 2868 dev = alloc_etherdev_mqs(sizeof(*priv), max_tx_queues, max_rx_queues); 2869 if (!dev) { 2870 dev_err(&pdev->dev, "could not allocate netdev\n"); 2871 err = -ENOMEM; 2872 goto abort_with_reg_bar; 2873 } 2874 SET_NETDEV_DEV(dev, &pdev->dev); 2875 pci_set_drvdata(pdev, dev); 2876 dev->ethtool_ops = &gve_ethtool_ops; 2877 dev->netdev_ops = &gve_netdev_ops; 2878 dev->queue_mgmt_ops = &gve_queue_mgmt_ops; 2879 dev->stat_ops = &gve_stat_ops; 2880 2881 /* Set default and supported features. 2882 * 2883 * Features might be set in other locations as well (such as 2884 * `gve_adminq_describe_device`). 2885 */ 2886 dev->hw_features = NETIF_F_HIGHDMA; 2887 dev->hw_features |= NETIF_F_SG; 2888 dev->hw_features |= NETIF_F_HW_CSUM; 2889 dev->hw_features |= NETIF_F_TSO; 2890 dev->hw_features |= NETIF_F_TSO6; 2891 dev->hw_features |= NETIF_F_TSO_ECN; 2892 dev->hw_features |= NETIF_F_RXCSUM; 2893 dev->hw_features |= NETIF_F_RXHASH; 2894 dev->features = dev->hw_features; 2895 dev->watchdog_timeo = 5 * HZ; 2896 dev->min_mtu = ETH_MIN_MTU; 2897 netif_carrier_off(dev); 2898 2899 priv = netdev_priv(dev); 2900 priv->dev = dev; 2901 priv->pdev = pdev; 2902 priv->msg_enable = DEFAULT_MSG_LEVEL; 2903 priv->reg_bar0 = reg_bar; 2904 priv->service_task_flags = 0x0; 2905 priv->state_flags = 0x0; 2906 priv->ethtool_flags = 0x0; 2907 priv->rx_cfg.packet_buffer_size = GVE_DEFAULT_RX_BUFFER_SIZE; 2908 priv->max_rx_buffer_size = GVE_DEFAULT_RX_BUFFER_SIZE; 2909 2910 err = gve_adminq_init(priv); 2911 if (err) { 2912 dev_err(&priv->pdev->dev, 2913 "Failed to alloc admin queue: err=%d\n", err); 2914 goto abort_with_netdev; 2915 } 2916 2917 priv->device_info.queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED; 2918 err = gve_adminq_get_device_properties(priv); 2919 if (err) 2920 goto abort_with_adminq; 2921 2922 /* Set adminq ctrl ops */ 2923 priv->ctrl_ops = &gve_adminq_ops; 2924 2925 err = priv->ctrl_ops->map_db_bar(priv); 2926 if (err) 2927 goto abort_with_adminq; 2928 2929 gve_set_probe_in_progress(priv); 2930 priv->gve_wq = alloc_ordered_workqueue("gve", 0); 2931 if (!priv->gve_wq) { 2932 dev_err(&pdev->dev, "Could not allocate workqueue"); 2933 err = -ENOMEM; 2934 goto abort_with_unmap_db_bar; 2935 } 2936 INIT_WORK(&priv->service_task, gve_service_task); 2937 INIT_WORK(&priv->stats_report_task, gve_stats_report_task); 2938 priv->tx_cfg.max_queues = max_tx_queues; 2939 priv->rx_cfg.max_queues = max_rx_queues; 2940 2941 err = gve_init_priv(priv); 2942 if (err) 2943 goto abort_with_wq; 2944 2945 err = gve_setup_device(priv); 2946 if (err) { 2947 dev_err(&priv->pdev->dev, 2948 "Could not setup device: err=%d\n", err); 2949 goto abort_teardown_device; 2950 } 2951 2952 if (!gve_is_gqi(priv) && !gve_is_qpl(priv)) 2953 dev->netmem_tx = NETMEM_TX_DMA; 2954 2955 err = register_netdev(dev); 2956 if (err) 2957 goto abort_teardown_device; 2958 2959 dev_info(&pdev->dev, "GVE version %s\n", gve_version_str); 2960 dev_info(&pdev->dev, "GVE queue format %d\n", (int)priv->queue_format); 2961 gve_clear_probe_in_progress(priv); 2962 queue_work(priv->gve_wq, &priv->service_task); 2963 return 0; 2964 2965 abort_teardown_device: 2966 disable_work_sync(&priv->service_task); 2967 gve_teardown_device(priv); 2968 2969 abort_with_wq: 2970 destroy_workqueue(priv->gve_wq); 2971 2972 abort_with_unmap_db_bar: 2973 priv->ctrl_ops->unmap_db_bar(priv); 2974 2975 abort_with_adminq: 2976 gve_adminq_free(priv); 2977 2978 abort_with_netdev: 2979 free_netdev(dev); 2980 2981 abort_with_reg_bar: 2982 pci_iounmap(pdev, reg_bar); 2983 2984 abort_with_pci_region: 2985 pci_release_regions(pdev); 2986 2987 abort_with_enabled: 2988 pci_disable_device(pdev); 2989 return err; 2990 } 2991 2992 static void gve_remove(struct pci_dev *pdev) 2993 { 2994 struct net_device *netdev = pci_get_drvdata(pdev); 2995 struct gve_priv *priv = netdev_priv(netdev); 2996 void __iomem *reg_bar = priv->reg_bar0; 2997 2998 unregister_netdev(netdev); 2999 disable_work_sync(&priv->service_task); 3000 gve_teardown_device(priv); 3001 destroy_workqueue(priv->gve_wq); 3002 priv->ctrl_ops->unmap_db_bar(priv); 3003 free_netdev(netdev); 3004 pci_iounmap(pdev, reg_bar); 3005 pci_release_regions(pdev); 3006 pci_disable_device(pdev); 3007 } 3008 3009 static void gve_shutdown(struct pci_dev *pdev) 3010 { 3011 struct net_device *netdev = pci_get_drvdata(pdev); 3012 struct gve_priv *priv = netdev_priv(netdev); 3013 bool was_up = netif_running(priv->dev); 3014 3015 disable_work_sync(&priv->service_task); 3016 3017 rtnl_lock(); 3018 netdev_lock(netdev); 3019 if (was_up) 3020 gve_close(priv->dev); 3021 3022 /* detach here because gve_close() might attach in recovery */ 3023 netif_device_detach(netdev); 3024 gve_teardown_device(priv); 3025 netdev_unlock(netdev); 3026 rtnl_unlock(); 3027 } 3028 3029 static int gve_suspend(struct device *dev) 3030 { 3031 struct pci_dev *pdev = to_pci_dev(dev); 3032 struct net_device *netdev = pci_get_drvdata(pdev); 3033 struct gve_priv *priv = netdev_priv(netdev); 3034 bool was_up = netif_running(priv->dev); 3035 3036 disable_work_sync(&priv->service_task); 3037 3038 priv->suspend_cnt++; 3039 rtnl_lock(); 3040 netdev_lock(netdev); 3041 if (was_up) 3042 gve_close(priv->dev); 3043 gve_teardown_device(priv); 3044 priv->up_before_suspend = was_up; 3045 netdev_unlock(netdev); 3046 rtnl_unlock(); 3047 return 0; 3048 } 3049 3050 static int gve_resume(struct device *dev) 3051 { 3052 struct pci_dev *pdev = to_pci_dev(dev); 3053 struct net_device *netdev = pci_get_drvdata(pdev); 3054 struct gve_priv *priv = netdev_priv(netdev); 3055 int err; 3056 3057 priv->resume_cnt++; 3058 rtnl_lock(); 3059 netdev_lock(netdev); 3060 enable_work(&priv->service_task); 3061 err = gve_recover(priv, priv->up_before_suspend); 3062 netdev_unlock(netdev); 3063 rtnl_unlock(); 3064 return err; 3065 } 3066 3067 static DEFINE_SIMPLE_DEV_PM_OPS(gve_pm_ops, gve_suspend, gve_resume); 3068 3069 static const struct pci_device_id gve_id_table[] = { 3070 { PCI_DEVICE(PCI_VENDOR_ID_GOOGLE, PCI_DEV_ID_GVNIC) }, 3071 { } 3072 }; 3073 3074 static struct pci_driver gve_driver = { 3075 .name = gve_driver_name, 3076 .id_table = gve_id_table, 3077 .probe = gve_probe, 3078 .remove = gve_remove, 3079 .shutdown = gve_shutdown, 3080 .driver.pm = pm_sleep_ptr(&gve_pm_ops), 3081 }; 3082 3083 module_pci_driver(gve_driver); 3084 3085 MODULE_DEVICE_TABLE(pci, gve_id_table); 3086 MODULE_AUTHOR("Google, Inc."); 3087 MODULE_DESCRIPTION("Google Virtual NIC Driver"); 3088 MODULE_LICENSE("Dual MIT/GPL"); 3089 MODULE_VERSION(GVE_VERSION); 3090