1 // SPDX-License-Identifier: (GPL-2.0 OR MIT) 2 /* Google virtual Ethernet (gve) driver 3 * 4 * Copyright (C) 2015-2024 Google LLC 5 */ 6 7 #include <linux/bitmap.h> 8 #include <linux/bpf.h> 9 #include <linux/cpumask.h> 10 #include <linux/etherdevice.h> 11 #include <linux/filter.h> 12 #include <linux/interrupt.h> 13 #include <linux/irq.h> 14 #include <linux/math64.h> 15 #include <linux/module.h> 16 #include <linux/pci.h> 17 #include <linux/sched.h> 18 #include <linux/timer.h> 19 #include <linux/workqueue.h> 20 #include <linux/utsname.h> 21 #include <linux/version.h> 22 #include <net/netdev_queues.h> 23 #include <net/sch_generic.h> 24 #include <net/xdp_sock_drv.h> 25 #include "gve.h" 26 #include "gve_dqo.h" 27 #include "gve_adminq.h" 28 #include "gve_register.h" 29 #include "gve_utils.h" 30 31 #define GVE_DEFAULT_RX_COPYBREAK (256) 32 33 #define DEFAULT_MSG_LEVEL (NETIF_MSG_DRV | NETIF_MSG_LINK) 34 #define GVE_VERSION "1.0.0" 35 #define GVE_VERSION_PREFIX "GVE-" 36 37 // Minimum amount of time between queue kicks in msec (10 seconds) 38 #define MIN_TX_TIMEOUT_GAP (1000 * 10) 39 40 char gve_driver_name[] = "gve"; 41 const char gve_version_str[] = GVE_VERSION; 42 static const char gve_version_prefix[] = GVE_VERSION_PREFIX; 43 44 static int gve_verify_driver_compatibility(struct gve_priv *priv) 45 { 46 int err; 47 struct gve_driver_info *driver_info; 48 dma_addr_t driver_info_bus; 49 50 driver_info = dma_alloc_coherent(&priv->pdev->dev, 51 sizeof(struct gve_driver_info), 52 &driver_info_bus, GFP_KERNEL); 53 if (!driver_info) 54 return -ENOMEM; 55 56 *driver_info = (struct gve_driver_info) { 57 .os_type = 1, /* Linux */ 58 .os_version_major = cpu_to_be32(LINUX_VERSION_MAJOR), 59 .os_version_minor = cpu_to_be32(LINUX_VERSION_SUBLEVEL), 60 .os_version_sub = cpu_to_be32(LINUX_VERSION_PATCHLEVEL), 61 .driver_capability_flags = { 62 cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS1), 63 cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS2), 64 cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS3), 65 cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS4), 66 }, 67 }; 68 strscpy(driver_info->os_version_str1, utsname()->release, 69 sizeof(driver_info->os_version_str1)); 70 strscpy(driver_info->os_version_str2, utsname()->version, 71 sizeof(driver_info->os_version_str2)); 72 73 err = gve_adminq_verify_driver_compatibility(priv, 74 sizeof(struct gve_driver_info), 75 driver_info_bus); 76 77 /* It's ok if the device doesn't support this */ 78 if (err == -EOPNOTSUPP) 79 err = 0; 80 81 dma_free_coherent(&priv->pdev->dev, 82 sizeof(struct gve_driver_info), 83 driver_info, driver_info_bus); 84 return err; 85 } 86 87 static netdev_features_t gve_features_check(struct sk_buff *skb, 88 struct net_device *dev, 89 netdev_features_t features) 90 { 91 struct gve_priv *priv = netdev_priv(dev); 92 93 if (!gve_is_gqi(priv)) 94 return gve_features_check_dqo(skb, dev, features); 95 96 return features; 97 } 98 99 static netdev_tx_t gve_start_xmit(struct sk_buff *skb, struct net_device *dev) 100 { 101 struct gve_priv *priv = netdev_priv(dev); 102 103 if (gve_is_gqi(priv)) 104 return gve_tx(skb, dev); 105 else 106 return gve_tx_dqo(skb, dev); 107 } 108 109 static void gve_get_stats(struct net_device *dev, struct rtnl_link_stats64 *s) 110 { 111 struct gve_priv *priv = netdev_priv(dev); 112 unsigned int start; 113 u64 packets, bytes; 114 int num_tx_queues; 115 int ring; 116 117 num_tx_queues = gve_num_tx_queues(priv); 118 if (priv->rx) { 119 for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) { 120 do { 121 start = 122 u64_stats_fetch_begin(&priv->rx[ring].statss); 123 packets = priv->rx[ring].rpackets; 124 bytes = priv->rx[ring].rbytes; 125 } while (u64_stats_fetch_retry(&priv->rx[ring].statss, 126 start)); 127 s->rx_packets += packets; 128 s->rx_bytes += bytes; 129 } 130 } 131 if (priv->tx) { 132 for (ring = 0; ring < num_tx_queues; ring++) { 133 do { 134 start = 135 u64_stats_fetch_begin(&priv->tx[ring].statss); 136 packets = priv->tx[ring].pkt_done; 137 bytes = priv->tx[ring].bytes_done; 138 } while (u64_stats_fetch_retry(&priv->tx[ring].statss, 139 start)); 140 s->tx_packets += packets; 141 s->tx_bytes += bytes; 142 } 143 } 144 } 145 146 static int gve_alloc_flow_rule_caches(struct gve_priv *priv) 147 { 148 struct gve_flow_rules_cache *flow_rules_cache = &priv->flow_rules_cache; 149 int err = 0; 150 151 if (!priv->max_flow_rules) 152 return 0; 153 154 flow_rules_cache->rules_cache = 155 kvzalloc_objs(*flow_rules_cache->rules_cache, 156 GVE_FLOW_RULES_CACHE_SIZE); 157 if (!flow_rules_cache->rules_cache) { 158 dev_err(&priv->pdev->dev, "Cannot alloc flow rules cache\n"); 159 return -ENOMEM; 160 } 161 162 flow_rules_cache->rule_ids_cache = 163 kvcalloc(GVE_FLOW_RULE_IDS_CACHE_SIZE, sizeof(*flow_rules_cache->rule_ids_cache), 164 GFP_KERNEL); 165 if (!flow_rules_cache->rule_ids_cache) { 166 dev_err(&priv->pdev->dev, "Cannot alloc flow rule ids cache\n"); 167 err = -ENOMEM; 168 goto free_rules_cache; 169 } 170 171 return 0; 172 173 free_rules_cache: 174 kvfree(flow_rules_cache->rules_cache); 175 flow_rules_cache->rules_cache = NULL; 176 return err; 177 } 178 179 static void gve_free_flow_rule_caches(struct gve_priv *priv) 180 { 181 struct gve_flow_rules_cache *flow_rules_cache = &priv->flow_rules_cache; 182 183 kvfree(flow_rules_cache->rule_ids_cache); 184 flow_rules_cache->rule_ids_cache = NULL; 185 kvfree(flow_rules_cache->rules_cache); 186 flow_rules_cache->rules_cache = NULL; 187 } 188 189 static int gve_alloc_rss_config_cache(struct gve_priv *priv) 190 { 191 struct gve_rss_config *rss_config = &priv->rss_config; 192 193 if (!priv->cache_rss_config) 194 return 0; 195 196 rss_config->hash_key = kcalloc(priv->rss_key_size, 197 sizeof(rss_config->hash_key[0]), 198 GFP_KERNEL); 199 if (!rss_config->hash_key) 200 return -ENOMEM; 201 202 rss_config->hash_lut = kcalloc(priv->rss_lut_size, 203 sizeof(rss_config->hash_lut[0]), 204 GFP_KERNEL); 205 if (!rss_config->hash_lut) 206 goto free_rss_key_cache; 207 208 return 0; 209 210 free_rss_key_cache: 211 kfree(rss_config->hash_key); 212 rss_config->hash_key = NULL; 213 return -ENOMEM; 214 } 215 216 static void gve_free_rss_config_cache(struct gve_priv *priv) 217 { 218 struct gve_rss_config *rss_config = &priv->rss_config; 219 220 kfree(rss_config->hash_key); 221 kfree(rss_config->hash_lut); 222 223 memset(rss_config, 0, sizeof(*rss_config)); 224 } 225 226 static int gve_alloc_counter_array(struct gve_priv *priv) 227 { 228 priv->counter_array = 229 dma_alloc_coherent(&priv->pdev->dev, 230 priv->num_event_counters * 231 sizeof(*priv->counter_array), 232 &priv->counter_array_bus, GFP_KERNEL); 233 if (!priv->counter_array) 234 return -ENOMEM; 235 236 return 0; 237 } 238 239 static void gve_free_counter_array(struct gve_priv *priv) 240 { 241 if (!priv->counter_array) 242 return; 243 244 dma_free_coherent(&priv->pdev->dev, 245 priv->num_event_counters * 246 sizeof(*priv->counter_array), 247 priv->counter_array, priv->counter_array_bus); 248 priv->counter_array = NULL; 249 } 250 251 /* NIC requests to report stats */ 252 static void gve_stats_report_task(struct work_struct *work) 253 { 254 struct gve_priv *priv = container_of(work, struct gve_priv, 255 stats_report_task); 256 if (gve_get_do_report_stats(priv)) { 257 gve_handle_report_stats(priv); 258 gve_clear_do_report_stats(priv); 259 } 260 } 261 262 static void gve_stats_report_schedule(struct gve_priv *priv) 263 { 264 if (!gve_get_probe_in_progress(priv) && 265 !gve_get_reset_in_progress(priv)) { 266 gve_set_do_report_stats(priv); 267 queue_work(priv->gve_wq, &priv->stats_report_task); 268 } 269 } 270 271 static void gve_stats_report_timer(struct timer_list *t) 272 { 273 struct gve_priv *priv = timer_container_of(priv, t, 274 stats_report_timer); 275 276 mod_timer(&priv->stats_report_timer, 277 round_jiffies(jiffies + 278 msecs_to_jiffies(priv->stats_report_timer_period))); 279 gve_stats_report_schedule(priv); 280 } 281 282 static int gve_alloc_stats_report(struct gve_priv *priv) 283 { 284 int tx_stats_num, rx_stats_num; 285 286 tx_stats_num = (GVE_TX_STATS_REPORT_NUM + NIC_TX_STATS_REPORT_NUM) * 287 priv->tx_cfg.max_queues; 288 rx_stats_num = (GVE_RX_STATS_REPORT_NUM + NIC_RX_STATS_REPORT_NUM) * 289 priv->rx_cfg.max_queues; 290 priv->stats_report_len = struct_size(priv->stats_report, stats, 291 size_add(tx_stats_num, rx_stats_num)); 292 priv->stats_report = 293 dma_alloc_coherent(&priv->pdev->dev, priv->stats_report_len, 294 &priv->stats_report_bus, GFP_KERNEL); 295 if (!priv->stats_report) 296 return -ENOMEM; 297 /* Set up timer for the report-stats task */ 298 timer_setup(&priv->stats_report_timer, gve_stats_report_timer, 0); 299 priv->stats_report_timer_period = GVE_STATS_REPORT_TIMER_PERIOD; 300 return 0; 301 } 302 303 static void gve_free_stats_report(struct gve_priv *priv) 304 { 305 if (!priv->stats_report) 306 return; 307 308 timer_delete_sync(&priv->stats_report_timer); 309 dma_free_coherent(&priv->pdev->dev, priv->stats_report_len, 310 priv->stats_report, priv->stats_report_bus); 311 priv->stats_report = NULL; 312 } 313 314 static irqreturn_t gve_mgmnt_intr(int irq, void *arg) 315 { 316 struct gve_priv *priv = arg; 317 318 queue_work(priv->gve_wq, &priv->service_task); 319 return IRQ_HANDLED; 320 } 321 322 static irqreturn_t gve_intr(int irq, void *arg) 323 { 324 struct gve_notify_block *block = arg; 325 struct gve_priv *priv = block->priv; 326 327 iowrite32be(GVE_IRQ_MASK, gve_irq_doorbell(priv, block)); 328 napi_schedule_irqoff(&block->napi); 329 return IRQ_HANDLED; 330 } 331 332 static irqreturn_t gve_intr_dqo(int irq, void *arg) 333 { 334 struct gve_notify_block *block = arg; 335 336 /* Interrupts are automatically masked */ 337 napi_schedule_irqoff(&block->napi); 338 return IRQ_HANDLED; 339 } 340 341 static int gve_is_napi_on_home_cpu(struct gve_priv *priv, u32 irq) 342 { 343 int cpu_curr = smp_processor_id(); 344 const struct cpumask *aff_mask; 345 346 aff_mask = irq_get_effective_affinity_mask(irq); 347 if (unlikely(!aff_mask)) 348 return 1; 349 350 return cpumask_test_cpu(cpu_curr, aff_mask); 351 } 352 353 int gve_napi_poll(struct napi_struct *napi, int budget) 354 { 355 struct gve_notify_block *block; 356 __be32 __iomem *irq_doorbell; 357 bool reschedule = false; 358 struct gve_priv *priv; 359 int work_done = 0; 360 361 block = container_of(napi, struct gve_notify_block, napi); 362 priv = block->priv; 363 364 if (block->tx) { 365 if (block->tx->q_num < priv->tx_cfg.num_queues) 366 reschedule |= gve_tx_poll(block, budget); 367 else if (budget) 368 reschedule |= gve_xdp_poll(block, budget); 369 } 370 371 if (!budget) 372 return 0; 373 374 if (block->rx) { 375 work_done = gve_rx_poll(block, budget); 376 377 /* Poll XSK TX as part of RX NAPI. Setup re-poll based on max of 378 * TX and RX work done. 379 */ 380 if (priv->xdp_prog) 381 work_done = max_t(int, work_done, 382 gve_xsk_tx_poll(block, budget)); 383 384 reschedule |= work_done == budget; 385 } 386 387 if (reschedule) 388 return budget; 389 390 /* Complete processing - don't unmask irq if busy polling is enabled */ 391 if (likely(napi_complete_done(napi, work_done))) { 392 irq_doorbell = gve_irq_doorbell(priv, block); 393 iowrite32be(GVE_IRQ_ACK | GVE_IRQ_EVENT, irq_doorbell); 394 395 /* Ensure IRQ ACK is visible before we check pending work. 396 * If queue had issued updates, it would be truly visible. 397 */ 398 mb(); 399 400 if (block->tx) 401 reschedule |= gve_tx_clean_pending(priv, block->tx); 402 if (block->rx) 403 reschedule |= gve_rx_work_pending(block->rx); 404 405 if (reschedule && napi_schedule(napi)) 406 iowrite32be(GVE_IRQ_MASK, irq_doorbell); 407 } 408 return work_done; 409 } 410 411 int gve_napi_poll_dqo(struct napi_struct *napi, int budget) 412 { 413 struct gve_notify_block *block = 414 container_of(napi, struct gve_notify_block, napi); 415 struct gve_priv *priv = block->priv; 416 bool reschedule = false; 417 int work_done = 0; 418 419 if (block->tx) { 420 if (block->tx->q_num < priv->tx_cfg.num_queues) 421 reschedule |= gve_tx_poll_dqo(block, /*do_clean=*/true); 422 else 423 reschedule |= gve_xdp_poll_dqo(block); 424 } 425 426 if (!budget) 427 return 0; 428 429 if (block->rx) { 430 work_done = gve_rx_poll_dqo(block, budget); 431 432 /* Poll XSK TX as part of RX NAPI. Setup re-poll based on if 433 * either datapath has more work to do. 434 */ 435 if (priv->xdp_prog) 436 reschedule |= gve_xsk_tx_poll_dqo(block, budget); 437 reschedule |= work_done == budget; 438 } 439 440 if (reschedule) { 441 /* Reschedule by returning budget only if already on the correct 442 * cpu. 443 */ 444 if (likely(gve_is_napi_on_home_cpu(priv, block->irq))) 445 return budget; 446 447 /* If not on the cpu with which this queue's irq has affinity 448 * with, we avoid rescheduling napi and arm the irq instead so 449 * that napi gets rescheduled back eventually onto the right 450 * cpu. 451 */ 452 if (work_done == budget) 453 work_done--; 454 } 455 456 if (likely(napi_complete_done(napi, work_done))) { 457 /* Enable interrupts again. 458 * 459 * We don't need to repoll afterwards because HW supports the 460 * PCI MSI-X PBA feature. 461 * 462 * Another interrupt would be triggered if a new event came in 463 * since the last one. 464 */ 465 gve_write_irq_doorbell_dqo(priv, block, 466 GVE_ITR_NO_UPDATE_DQO | GVE_ITR_ENABLE_BIT_DQO); 467 } 468 469 return work_done; 470 } 471 472 static const struct cpumask *gve_get_node_mask(struct gve_priv *priv) 473 { 474 if (priv->numa_node == NUMA_NO_NODE) 475 return cpu_all_mask; 476 else 477 return cpumask_of_node(priv->numa_node); 478 } 479 480 static int gve_alloc_notify_blocks(struct gve_priv *priv) 481 { 482 int num_vecs_requested = priv->num_ntfy_blks + 1; 483 const struct cpumask *node_mask; 484 unsigned int cur_cpu; 485 int vecs_enabled; 486 int i, j; 487 int err; 488 489 priv->msix_vectors = kvzalloc_objs(*priv->msix_vectors, 490 num_vecs_requested); 491 if (!priv->msix_vectors) 492 return -ENOMEM; 493 for (i = 0; i < num_vecs_requested; i++) 494 priv->msix_vectors[i].entry = i; 495 vecs_enabled = pci_enable_msix_range(priv->pdev, priv->msix_vectors, 496 GVE_MIN_MSIX, num_vecs_requested); 497 if (vecs_enabled < 0) { 498 dev_err(&priv->pdev->dev, "Could not enable min msix %d/%d\n", 499 GVE_MIN_MSIX, vecs_enabled); 500 err = vecs_enabled; 501 goto abort_with_msix_vectors; 502 } 503 if (vecs_enabled != num_vecs_requested) { 504 int new_num_ntfy_blks = (vecs_enabled - 1) & ~0x1; 505 int vecs_per_type = new_num_ntfy_blks / 2; 506 int vecs_left = new_num_ntfy_blks % 2; 507 508 priv->num_ntfy_blks = new_num_ntfy_blks; 509 priv->mgmt_msix_idx = priv->num_ntfy_blks; 510 priv->tx_cfg.max_queues = min_t(int, priv->tx_cfg.max_queues, 511 vecs_per_type); 512 priv->rx_cfg.max_queues = min_t(int, priv->rx_cfg.max_queues, 513 vecs_per_type + vecs_left); 514 dev_err(&priv->pdev->dev, 515 "Could not enable desired msix, only enabled %d, adjusting tx max queues to %d, and rx max queues to %d\n", 516 vecs_enabled, priv->tx_cfg.max_queues, 517 priv->rx_cfg.max_queues); 518 if (priv->tx_cfg.num_queues > priv->tx_cfg.max_queues) 519 priv->tx_cfg.num_queues = priv->tx_cfg.max_queues; 520 if (priv->rx_cfg.num_queues > priv->rx_cfg.max_queues) 521 priv->rx_cfg.num_queues = priv->rx_cfg.max_queues; 522 } 523 524 /* Setup Management Vector - the last vector */ 525 snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name), "gve-mgmnt@pci:%s", 526 pci_name(priv->pdev)); 527 err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, 528 gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv); 529 if (err) { 530 dev_err(&priv->pdev->dev, "Did not receive management vector.\n"); 531 goto abort_with_msix_enabled; 532 } 533 priv->irq_db_indices = 534 dma_alloc_coherent(&priv->pdev->dev, 535 priv->num_ntfy_blks * 536 sizeof(*priv->irq_db_indices), 537 &priv->irq_db_indices_bus, GFP_KERNEL); 538 if (!priv->irq_db_indices) { 539 err = -ENOMEM; 540 goto abort_with_mgmt_vector; 541 } 542 543 priv->ntfy_blocks = kvzalloc(priv->num_ntfy_blks * 544 sizeof(*priv->ntfy_blocks), GFP_KERNEL); 545 if (!priv->ntfy_blocks) { 546 err = -ENOMEM; 547 goto abort_with_irq_db_indices; 548 } 549 550 /* Setup the other blocks - the first n-1 vectors */ 551 node_mask = gve_get_node_mask(priv); 552 cur_cpu = cpumask_first(node_mask); 553 for (i = 0; i < priv->num_ntfy_blks; i++) { 554 struct gve_notify_block *block = &priv->ntfy_blocks[i]; 555 int msix_idx = i; 556 557 snprintf(block->name, sizeof(block->name), "gve-ntfy-blk%d@pci:%s", 558 i, pci_name(priv->pdev)); 559 block->priv = priv; 560 err = request_irq(priv->msix_vectors[msix_idx].vector, 561 gve_is_gqi(priv) ? gve_intr : gve_intr_dqo, 562 IRQF_NO_AUTOEN, block->name, block); 563 if (err) { 564 dev_err(&priv->pdev->dev, 565 "Failed to receive msix vector %d\n", i); 566 goto abort_with_some_ntfy_blocks; 567 } 568 block->irq = priv->msix_vectors[msix_idx].vector; 569 irq_set_affinity_and_hint(block->irq, 570 cpumask_of(cur_cpu)); 571 block->irq_db_index = &priv->irq_db_indices[i].index; 572 573 cur_cpu = cpumask_next(cur_cpu, node_mask); 574 /* Wrap once CPUs in the node have been exhausted, or when 575 * starting RX queue affinities. TX and RX queues of the same 576 * index share affinity. 577 */ 578 if (cur_cpu >= nr_cpu_ids || (i + 1) == priv->tx_cfg.max_queues) 579 cur_cpu = cpumask_first(node_mask); 580 } 581 return 0; 582 abort_with_some_ntfy_blocks: 583 for (j = 0; j < i; j++) { 584 struct gve_notify_block *block = &priv->ntfy_blocks[j]; 585 int msix_idx = j; 586 587 irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector, 588 NULL); 589 free_irq(priv->msix_vectors[msix_idx].vector, block); 590 block->irq = 0; 591 } 592 kvfree(priv->ntfy_blocks); 593 priv->ntfy_blocks = NULL; 594 abort_with_irq_db_indices: 595 dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks * 596 sizeof(*priv->irq_db_indices), 597 priv->irq_db_indices, priv->irq_db_indices_bus); 598 priv->irq_db_indices = NULL; 599 abort_with_mgmt_vector: 600 free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv); 601 abort_with_msix_enabled: 602 pci_disable_msix(priv->pdev); 603 abort_with_msix_vectors: 604 kvfree(priv->msix_vectors); 605 priv->msix_vectors = NULL; 606 return err; 607 } 608 609 static void gve_free_notify_blocks(struct gve_priv *priv) 610 { 611 int i; 612 613 if (!priv->msix_vectors) 614 return; 615 616 /* Free the irqs */ 617 for (i = 0; i < priv->num_ntfy_blks; i++) { 618 struct gve_notify_block *block = &priv->ntfy_blocks[i]; 619 int msix_idx = i; 620 621 irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector, 622 NULL); 623 free_irq(priv->msix_vectors[msix_idx].vector, block); 624 block->irq = 0; 625 } 626 free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv); 627 kvfree(priv->ntfy_blocks); 628 priv->ntfy_blocks = NULL; 629 dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks * 630 sizeof(*priv->irq_db_indices), 631 priv->irq_db_indices, priv->irq_db_indices_bus); 632 priv->irq_db_indices = NULL; 633 pci_disable_msix(priv->pdev); 634 kvfree(priv->msix_vectors); 635 priv->msix_vectors = NULL; 636 } 637 638 static int gve_setup_device_resources(struct gve_priv *priv) 639 { 640 int err; 641 642 err = gve_alloc_flow_rule_caches(priv); 643 if (err) 644 return err; 645 err = gve_alloc_rss_config_cache(priv); 646 if (err) 647 goto abort_with_flow_rule_caches; 648 err = gve_alloc_counter_array(priv); 649 if (err) 650 goto abort_with_rss_config_cache; 651 err = gve_alloc_notify_blocks(priv); 652 if (err) 653 goto abort_with_counter; 654 err = gve_alloc_stats_report(priv); 655 if (err) 656 goto abort_with_ntfy_blocks; 657 err = gve_adminq_configure_device_resources(priv, 658 priv->counter_array_bus, 659 priv->num_event_counters, 660 priv->irq_db_indices_bus, 661 priv->num_ntfy_blks); 662 if (unlikely(err)) { 663 dev_err(&priv->pdev->dev, 664 "could not setup device_resources: err=%d\n", err); 665 err = -ENXIO; 666 goto abort_with_stats_report; 667 } 668 669 if (!gve_is_gqi(priv)) { 670 priv->ptype_lut_dqo = kvzalloc_obj(*priv->ptype_lut_dqo); 671 if (!priv->ptype_lut_dqo) { 672 err = -ENOMEM; 673 goto abort_with_stats_report; 674 } 675 err = gve_adminq_get_ptype_map_dqo(priv, priv->ptype_lut_dqo); 676 if (err) { 677 dev_err(&priv->pdev->dev, 678 "Failed to get ptype map: err=%d\n", err); 679 goto abort_with_ptype_lut; 680 } 681 } 682 683 if (priv->nic_timestamp_supported) { 684 err = gve_init_clock(priv); 685 if (err) { 686 dev_warn(&priv->pdev->dev, "Failed to init clock, continuing without PTP support"); 687 err = 0; 688 } 689 } 690 691 err = gve_init_rss_config(priv, priv->rx_cfg.num_queues); 692 if (err) { 693 dev_err(&priv->pdev->dev, "Failed to init RSS config"); 694 goto abort_with_clock; 695 } 696 697 err = gve_adminq_report_stats(priv, priv->stats_report_len, 698 priv->stats_report_bus, 699 GVE_STATS_REPORT_TIMER_PERIOD); 700 if (err) 701 dev_err(&priv->pdev->dev, 702 "Failed to report stats: err=%d\n", err); 703 gve_set_device_resources_ok(priv); 704 return 0; 705 706 abort_with_clock: 707 gve_teardown_clock(priv); 708 abort_with_ptype_lut: 709 kvfree(priv->ptype_lut_dqo); 710 priv->ptype_lut_dqo = NULL; 711 abort_with_stats_report: 712 gve_free_stats_report(priv); 713 abort_with_ntfy_blocks: 714 gve_free_notify_blocks(priv); 715 abort_with_counter: 716 gve_free_counter_array(priv); 717 abort_with_rss_config_cache: 718 gve_free_rss_config_cache(priv); 719 abort_with_flow_rule_caches: 720 gve_free_flow_rule_caches(priv); 721 722 return err; 723 } 724 725 static void gve_trigger_reset(struct gve_priv *priv); 726 727 static void gve_teardown_device_resources(struct gve_priv *priv) 728 { 729 int err; 730 731 /* Tell device its resources are being freed */ 732 if (gve_get_device_resources_ok(priv)) { 733 err = gve_flow_rules_reset(priv); 734 if (err) { 735 dev_err(&priv->pdev->dev, 736 "Failed to reset flow rules: err=%d\n", err); 737 gve_trigger_reset(priv); 738 } 739 /* detach the stats report */ 740 err = gve_adminq_report_stats(priv, 0, 0x0, GVE_STATS_REPORT_TIMER_PERIOD); 741 if (err) { 742 dev_err(&priv->pdev->dev, 743 "Failed to detach stats report: err=%d\n", err); 744 gve_trigger_reset(priv); 745 } 746 err = gve_adminq_deconfigure_device_resources(priv); 747 if (err) { 748 dev_err(&priv->pdev->dev, 749 "Could not deconfigure device resources: err=%d\n", 750 err); 751 gve_trigger_reset(priv); 752 } 753 } 754 755 kvfree(priv->ptype_lut_dqo); 756 priv->ptype_lut_dqo = NULL; 757 758 gve_free_flow_rule_caches(priv); 759 gve_free_rss_config_cache(priv); 760 gve_free_counter_array(priv); 761 gve_free_notify_blocks(priv); 762 gve_free_stats_report(priv); 763 gve_teardown_clock(priv); 764 gve_clear_device_resources_ok(priv); 765 } 766 767 static int gve_unregister_qpl(struct gve_priv *priv, 768 struct gve_queue_page_list *qpl) 769 { 770 int err; 771 772 if (!qpl) 773 return 0; 774 775 err = gve_adminq_unregister_page_list(priv, qpl->id); 776 if (err) { 777 netif_err(priv, drv, priv->dev, 778 "Failed to unregister queue page list %d\n", 779 qpl->id); 780 return err; 781 } 782 783 priv->num_registered_pages -= qpl->num_entries; 784 return 0; 785 } 786 787 static int gve_register_qpl(struct gve_priv *priv, 788 struct gve_queue_page_list *qpl) 789 { 790 int pages; 791 int err; 792 793 if (!qpl) 794 return 0; 795 796 pages = qpl->num_entries; 797 798 if (pages + priv->num_registered_pages > priv->max_registered_pages) { 799 netif_err(priv, drv, priv->dev, 800 "Reached max number of registered pages %llu > %llu\n", 801 pages + priv->num_registered_pages, 802 priv->max_registered_pages); 803 return -EINVAL; 804 } 805 806 err = gve_adminq_register_page_list(priv, qpl); 807 if (err) { 808 netif_err(priv, drv, priv->dev, 809 "failed to register queue page list %d\n", 810 qpl->id); 811 return err; 812 } 813 814 priv->num_registered_pages += pages; 815 return 0; 816 } 817 818 static struct gve_queue_page_list *gve_tx_get_qpl(struct gve_priv *priv, int idx) 819 { 820 struct gve_tx_ring *tx = &priv->tx[idx]; 821 822 if (gve_is_gqi(priv)) 823 return tx->tx_fifo.qpl; 824 else 825 return tx->dqo.qpl; 826 } 827 828 static struct gve_queue_page_list *gve_rx_get_qpl(struct gve_priv *priv, int idx) 829 { 830 struct gve_rx_ring *rx = &priv->rx[idx]; 831 832 if (gve_is_gqi(priv)) 833 return rx->data.qpl; 834 else 835 return rx->dqo.qpl; 836 } 837 838 static int gve_register_qpls(struct gve_priv *priv) 839 { 840 int num_tx_qpls, num_rx_qpls; 841 int err; 842 int i; 843 844 num_tx_qpls = gve_num_tx_qpls(&priv->tx_cfg, gve_is_qpl(priv)); 845 num_rx_qpls = gve_num_rx_qpls(&priv->rx_cfg, gve_is_qpl(priv)); 846 847 for (i = 0; i < num_tx_qpls; i++) { 848 err = gve_register_qpl(priv, gve_tx_get_qpl(priv, i)); 849 if (err) 850 return err; 851 } 852 853 for (i = 0; i < num_rx_qpls; i++) { 854 err = gve_register_qpl(priv, gve_rx_get_qpl(priv, i)); 855 if (err) 856 return err; 857 } 858 859 return 0; 860 } 861 862 static int gve_unregister_qpls(struct gve_priv *priv) 863 { 864 int num_tx_qpls, num_rx_qpls; 865 int err; 866 int i; 867 868 num_tx_qpls = gve_num_tx_qpls(&priv->tx_cfg, gve_is_qpl(priv)); 869 num_rx_qpls = gve_num_rx_qpls(&priv->rx_cfg, gve_is_qpl(priv)); 870 871 for (i = 0; i < num_tx_qpls; i++) { 872 err = gve_unregister_qpl(priv, gve_tx_get_qpl(priv, i)); 873 /* This failure will trigger a reset - no need to clean */ 874 if (err) 875 return err; 876 } 877 878 for (i = 0; i < num_rx_qpls; i++) { 879 err = gve_unregister_qpl(priv, gve_rx_get_qpl(priv, i)); 880 /* This failure will trigger a reset - no need to clean */ 881 if (err) 882 return err; 883 } 884 return 0; 885 } 886 887 static int gve_create_rings(struct gve_priv *priv) 888 { 889 int num_tx_queues = gve_num_tx_queues(priv); 890 int err; 891 int i; 892 893 err = gve_adminq_create_tx_queues(priv, 0, num_tx_queues); 894 if (err) { 895 netif_err(priv, drv, priv->dev, "failed to create %d tx queues\n", 896 num_tx_queues); 897 /* This failure will trigger a reset - no need to clean 898 * up 899 */ 900 return err; 901 } 902 netif_dbg(priv, drv, priv->dev, "created %d tx queues\n", 903 num_tx_queues); 904 905 err = gve_adminq_create_rx_queues(priv, priv->rx_cfg.num_queues); 906 if (err) { 907 netif_err(priv, drv, priv->dev, "failed to create %d rx queues\n", 908 priv->rx_cfg.num_queues); 909 /* This failure will trigger a reset - no need to clean 910 * up 911 */ 912 return err; 913 } 914 netif_dbg(priv, drv, priv->dev, "created %d rx queues\n", 915 priv->rx_cfg.num_queues); 916 917 if (gve_is_gqi(priv)) { 918 /* Rx data ring has been prefilled with packet buffers at queue 919 * allocation time. 920 * 921 * Write the doorbell to provide descriptor slots and packet 922 * buffers to the NIC. 923 */ 924 for (i = 0; i < priv->rx_cfg.num_queues; i++) 925 gve_rx_write_doorbell(priv, &priv->rx[i]); 926 } else { 927 for (i = 0; i < priv->rx_cfg.num_queues; i++) { 928 /* Post buffers and ring doorbell. */ 929 gve_rx_post_buffers_dqo(&priv->rx[i]); 930 } 931 } 932 933 return 0; 934 } 935 936 static void init_xdp_sync_stats(struct gve_priv *priv) 937 { 938 int start_id = gve_xdp_tx_start_queue_id(priv); 939 int i; 940 941 /* Init stats */ 942 for (i = start_id; i < start_id + priv->tx_cfg.num_xdp_queues; i++) { 943 int ntfy_idx = gve_tx_idx_to_ntfy(priv, i); 944 945 u64_stats_init(&priv->tx[i].statss); 946 priv->tx[i].ntfy_id = ntfy_idx; 947 } 948 } 949 950 static void gve_init_sync_stats(struct gve_priv *priv) 951 { 952 int i; 953 954 for (i = 0; i < priv->tx_cfg.num_queues; i++) 955 u64_stats_init(&priv->tx[i].statss); 956 957 /* Init stats for XDP TX queues */ 958 init_xdp_sync_stats(priv); 959 960 for (i = 0; i < priv->rx_cfg.num_queues; i++) 961 u64_stats_init(&priv->rx[i].statss); 962 } 963 964 static void gve_tx_get_curr_alloc_cfg(struct gve_priv *priv, 965 struct gve_tx_alloc_rings_cfg *cfg) 966 { 967 cfg->qcfg = &priv->tx_cfg; 968 cfg->raw_addressing = !gve_is_qpl(priv); 969 cfg->ring_size = priv->tx_desc_cnt; 970 cfg->pages_per_qpl = priv->tx_pages_per_qpl; 971 cfg->num_xdp_rings = cfg->qcfg->num_xdp_queues; 972 cfg->tx = priv->tx; 973 } 974 975 static void gve_tx_stop_rings(struct gve_priv *priv, int num_rings) 976 { 977 int i; 978 979 if (!priv->tx) 980 return; 981 982 for (i = 0; i < num_rings; i++) { 983 if (gve_is_gqi(priv)) 984 gve_tx_stop_ring_gqi(priv, i); 985 else 986 gve_tx_stop_ring_dqo(priv, i); 987 } 988 } 989 990 static void gve_tx_start_rings(struct gve_priv *priv, int num_rings) 991 { 992 int i; 993 994 for (i = 0; i < num_rings; i++) { 995 if (gve_is_gqi(priv)) 996 gve_tx_start_ring_gqi(priv, i); 997 else 998 gve_tx_start_ring_dqo(priv, i); 999 } 1000 } 1001 1002 void gve_update_num_qpl_pages(struct gve_priv *priv, 1003 struct gve_rx_alloc_rings_cfg *rx_alloc_cfg, 1004 struct gve_tx_alloc_rings_cfg *tx_alloc_cfg) 1005 { 1006 u64 ideal_tx_pages, ideal_rx_pages; 1007 u16 tx_num_queues, rx_num_queues; 1008 u64 max_pages, tx_pages; 1009 1010 if (priv->queue_format == GVE_GQI_QPL_FORMAT) { 1011 rx_alloc_cfg->pages_per_qpl = rx_alloc_cfg->ring_size; 1012 } else if (priv->queue_format == GVE_DQO_QPL_FORMAT) { 1013 /* 1014 * We want 2 pages per RX descriptor and half a page per TX 1015 * descriptor, which means the fraction ideal_tx_pages / 1016 * (ideal_tx_pages + ideal_rx_pages) of the pages we allocate 1017 * should be for TX. Shrink proportionally as necessary to avoid 1018 * allocating more than max_registered_pages total pages. 1019 */ 1020 tx_num_queues = tx_alloc_cfg->qcfg->num_queues; 1021 rx_num_queues = rx_alloc_cfg->qcfg_rx->num_queues; 1022 1023 ideal_tx_pages = tx_alloc_cfg->ring_size * tx_num_queues / 2; 1024 ideal_rx_pages = rx_alloc_cfg->ring_size * rx_num_queues * 2; 1025 max_pages = min(priv->max_registered_pages, 1026 ideal_tx_pages + ideal_rx_pages); 1027 1028 tx_pages = div64_u64(max_pages * ideal_tx_pages, 1029 ideal_tx_pages + ideal_rx_pages); 1030 tx_alloc_cfg->pages_per_qpl = div_u64(tx_pages, tx_num_queues); 1031 rx_alloc_cfg->pages_per_qpl = div_u64(max_pages - tx_pages, 1032 rx_num_queues); 1033 } 1034 } 1035 1036 static int gve_queues_mem_alloc(struct gve_priv *priv, 1037 struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, 1038 struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) 1039 { 1040 int err; 1041 1042 gve_update_num_qpl_pages(priv, rx_alloc_cfg, tx_alloc_cfg); 1043 1044 if (gve_is_gqi(priv)) 1045 err = gve_tx_alloc_rings_gqi(priv, tx_alloc_cfg); 1046 else 1047 err = gve_tx_alloc_rings_dqo(priv, tx_alloc_cfg); 1048 if (err) 1049 return err; 1050 1051 if (gve_is_gqi(priv)) 1052 err = gve_rx_alloc_rings_gqi(priv, rx_alloc_cfg); 1053 else 1054 err = gve_rx_alloc_rings_dqo(priv, rx_alloc_cfg); 1055 if (err) 1056 goto free_tx; 1057 1058 return 0; 1059 1060 free_tx: 1061 if (gve_is_gqi(priv)) 1062 gve_tx_free_rings_gqi(priv, tx_alloc_cfg); 1063 else 1064 gve_tx_free_rings_dqo(priv, tx_alloc_cfg); 1065 return err; 1066 } 1067 1068 static int gve_destroy_rings(struct gve_priv *priv) 1069 { 1070 int num_tx_queues = gve_num_tx_queues(priv); 1071 int err; 1072 1073 err = gve_adminq_destroy_tx_queues(priv, 0, num_tx_queues); 1074 if (err) { 1075 netif_err(priv, drv, priv->dev, 1076 "failed to destroy tx queues\n"); 1077 /* This failure will trigger a reset - no need to clean up */ 1078 return err; 1079 } 1080 netif_dbg(priv, drv, priv->dev, "destroyed tx queues\n"); 1081 err = gve_adminq_destroy_rx_queues(priv, priv->rx_cfg.num_queues); 1082 if (err) { 1083 netif_err(priv, drv, priv->dev, 1084 "failed to destroy rx queues\n"); 1085 /* This failure will trigger a reset - no need to clean up */ 1086 return err; 1087 } 1088 netif_dbg(priv, drv, priv->dev, "destroyed rx queues\n"); 1089 return 0; 1090 } 1091 1092 static void gve_queues_mem_free(struct gve_priv *priv, 1093 struct gve_tx_alloc_rings_cfg *tx_cfg, 1094 struct gve_rx_alloc_rings_cfg *rx_cfg) 1095 { 1096 if (gve_is_gqi(priv)) { 1097 gve_tx_free_rings_gqi(priv, tx_cfg); 1098 gve_rx_free_rings_gqi(priv, rx_cfg); 1099 } else { 1100 gve_tx_free_rings_dqo(priv, tx_cfg); 1101 gve_rx_free_rings_dqo(priv, rx_cfg); 1102 } 1103 } 1104 1105 int gve_alloc_page(struct gve_priv *priv, struct device *dev, 1106 struct page **page, dma_addr_t *dma, 1107 enum dma_data_direction dir, gfp_t gfp_flags) 1108 { 1109 *page = alloc_pages_node(priv->numa_node, gfp_flags, 0); 1110 if (!*page) { 1111 priv->page_alloc_fail++; 1112 return -ENOMEM; 1113 } 1114 *dma = dma_map_page(dev, *page, 0, PAGE_SIZE, dir); 1115 if (dma_mapping_error(dev, *dma)) { 1116 priv->dma_mapping_error++; 1117 put_page(*page); 1118 return -ENOMEM; 1119 } 1120 return 0; 1121 } 1122 1123 struct gve_queue_page_list *gve_alloc_queue_page_list(struct gve_priv *priv, 1124 u32 id, int pages) 1125 { 1126 struct gve_queue_page_list *qpl; 1127 int err; 1128 int i; 1129 1130 qpl = kvzalloc_obj(*qpl); 1131 if (!qpl) 1132 return NULL; 1133 1134 qpl->id = id; 1135 qpl->num_entries = 0; 1136 qpl->pages = kvzalloc_objs(*qpl->pages, pages); 1137 if (!qpl->pages) 1138 goto abort; 1139 1140 qpl->page_buses = kvzalloc_objs(*qpl->page_buses, pages); 1141 if (!qpl->page_buses) 1142 goto abort; 1143 1144 for (i = 0; i < pages; i++) { 1145 err = gve_alloc_page(priv, &priv->pdev->dev, &qpl->pages[i], 1146 &qpl->page_buses[i], 1147 gve_qpl_dma_dir(priv, id), GFP_KERNEL); 1148 if (err) 1149 goto abort; 1150 qpl->num_entries++; 1151 } 1152 1153 return qpl; 1154 1155 abort: 1156 gve_free_queue_page_list(priv, qpl, id); 1157 return NULL; 1158 } 1159 1160 void gve_free_page(struct device *dev, struct page *page, dma_addr_t dma, 1161 enum dma_data_direction dir) 1162 { 1163 if (!dma_mapping_error(dev, dma)) 1164 dma_unmap_page(dev, dma, PAGE_SIZE, dir); 1165 if (page) 1166 put_page(page); 1167 } 1168 1169 void gve_free_queue_page_list(struct gve_priv *priv, 1170 struct gve_queue_page_list *qpl, 1171 u32 id) 1172 { 1173 int i; 1174 1175 if (!qpl) 1176 return; 1177 if (!qpl->pages) 1178 goto free_qpl; 1179 if (!qpl->page_buses) 1180 goto free_pages; 1181 1182 for (i = 0; i < qpl->num_entries; i++) 1183 gve_free_page(&priv->pdev->dev, qpl->pages[i], 1184 qpl->page_buses[i], gve_qpl_dma_dir(priv, id)); 1185 1186 kvfree(qpl->page_buses); 1187 qpl->page_buses = NULL; 1188 free_pages: 1189 kvfree(qpl->pages); 1190 qpl->pages = NULL; 1191 free_qpl: 1192 kvfree(qpl); 1193 } 1194 1195 /* Use this to schedule a reset when the device is capable of continuing 1196 * to handle other requests in its current state. If it is not, do a reset 1197 * in thread instead. 1198 */ 1199 void gve_schedule_reset(struct gve_priv *priv) 1200 { 1201 gve_set_do_reset(priv); 1202 queue_work(priv->gve_wq, &priv->service_task); 1203 } 1204 1205 static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up); 1206 static int gve_reset_recovery(struct gve_priv *priv, bool was_up); 1207 static void gve_turndown(struct gve_priv *priv); 1208 static void gve_turnup(struct gve_priv *priv); 1209 1210 static void gve_unreg_xsk_pool(struct gve_priv *priv, u16 qid) 1211 { 1212 struct gve_rx_ring *rx; 1213 1214 if (!priv->rx) 1215 return; 1216 1217 rx = &priv->rx[qid]; 1218 rx->xsk_pool = NULL; 1219 if (xdp_rxq_info_is_reg(&rx->xdp_rxq)) 1220 xdp_rxq_info_unreg_mem_model(&rx->xdp_rxq); 1221 1222 if (!priv->tx) 1223 return; 1224 priv->tx[gve_xdp_tx_queue_id(priv, qid)].xsk_pool = NULL; 1225 } 1226 1227 static int gve_reg_xsk_pool(struct gve_priv *priv, struct net_device *dev, 1228 struct xsk_buff_pool *pool, u16 qid) 1229 { 1230 struct gve_rx_ring *rx; 1231 u16 tx_qid; 1232 int err; 1233 1234 rx = &priv->rx[qid]; 1235 err = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq, 1236 MEM_TYPE_XSK_BUFF_POOL, pool); 1237 if (err) { 1238 gve_unreg_xsk_pool(priv, qid); 1239 return err; 1240 } 1241 1242 rx->xsk_pool = pool; 1243 1244 tx_qid = gve_xdp_tx_queue_id(priv, qid); 1245 priv->tx[tx_qid].xsk_pool = pool; 1246 1247 return 0; 1248 } 1249 1250 static void gve_unreg_xdp_info(struct gve_priv *priv) 1251 { 1252 int i; 1253 1254 if (!priv->tx_cfg.num_xdp_queues || !priv->rx) 1255 return; 1256 1257 for (i = 0; i < priv->rx_cfg.num_queues; i++) { 1258 struct gve_rx_ring *rx = &priv->rx[i]; 1259 1260 if (xdp_rxq_info_is_reg(&rx->xdp_rxq)) 1261 xdp_rxq_info_unreg(&rx->xdp_rxq); 1262 1263 gve_unreg_xsk_pool(priv, i); 1264 } 1265 } 1266 1267 static struct xsk_buff_pool *gve_get_xsk_pool(struct gve_priv *priv, int qid) 1268 { 1269 if (!test_bit(qid, priv->xsk_pools)) 1270 return NULL; 1271 1272 return xsk_get_pool_from_qid(priv->dev, qid); 1273 } 1274 1275 static int gve_reg_xdp_info(struct gve_priv *priv, struct net_device *dev) 1276 { 1277 struct napi_struct *napi; 1278 struct gve_rx_ring *rx; 1279 int err = 0; 1280 int i; 1281 1282 if (!priv->tx_cfg.num_xdp_queues) 1283 return 0; 1284 1285 for (i = 0; i < priv->rx_cfg.num_queues; i++) { 1286 struct xsk_buff_pool *xsk_pool; 1287 1288 rx = &priv->rx[i]; 1289 napi = &priv->ntfy_blocks[rx->ntfy_id].napi; 1290 1291 err = xdp_rxq_info_reg(&rx->xdp_rxq, dev, i, 1292 napi->napi_id); 1293 if (err) 1294 goto err; 1295 1296 xsk_pool = gve_get_xsk_pool(priv, i); 1297 if (xsk_pool) 1298 err = gve_reg_xsk_pool(priv, dev, xsk_pool, i); 1299 else if (gve_is_qpl(priv)) 1300 err = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq, 1301 MEM_TYPE_PAGE_SHARED, 1302 NULL); 1303 else 1304 err = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq, 1305 MEM_TYPE_PAGE_POOL, 1306 rx->dqo.page_pool); 1307 if (err) 1308 goto err; 1309 } 1310 return 0; 1311 1312 err: 1313 gve_unreg_xdp_info(priv); 1314 return err; 1315 } 1316 1317 1318 static void gve_drain_page_cache(struct gve_priv *priv) 1319 { 1320 int i; 1321 1322 for (i = 0; i < priv->rx_cfg.num_queues; i++) 1323 page_frag_cache_drain(&priv->rx[i].page_cache); 1324 } 1325 1326 static void gve_rx_get_curr_alloc_cfg(struct gve_priv *priv, 1327 struct gve_rx_alloc_rings_cfg *cfg) 1328 { 1329 cfg->qcfg_rx = &priv->rx_cfg; 1330 cfg->qcfg_tx = &priv->tx_cfg; 1331 cfg->raw_addressing = !gve_is_qpl(priv); 1332 cfg->enable_header_split = priv->header_split_enabled; 1333 cfg->ring_size = priv->rx_desc_cnt; 1334 cfg->pages_per_qpl = priv->rx_pages_per_qpl; 1335 cfg->packet_buffer_size = priv->rx_cfg.packet_buffer_size; 1336 cfg->rx = priv->rx; 1337 cfg->xdp = !!cfg->qcfg_tx->num_xdp_queues; 1338 } 1339 1340 void gve_get_curr_alloc_cfgs(struct gve_priv *priv, 1341 struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, 1342 struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) 1343 { 1344 gve_tx_get_curr_alloc_cfg(priv, tx_alloc_cfg); 1345 gve_rx_get_curr_alloc_cfg(priv, rx_alloc_cfg); 1346 } 1347 1348 static void gve_rx_start_ring(struct gve_priv *priv, int i) 1349 { 1350 if (gve_is_gqi(priv)) 1351 gve_rx_start_ring_gqi(priv, i); 1352 else 1353 gve_rx_start_ring_dqo(priv, i); 1354 } 1355 1356 static void gve_rx_start_rings(struct gve_priv *priv, int num_rings) 1357 { 1358 int i; 1359 1360 for (i = 0; i < num_rings; i++) 1361 gve_rx_start_ring(priv, i); 1362 } 1363 1364 static void gve_rx_stop_ring(struct gve_priv *priv, int i) 1365 { 1366 if (gve_is_gqi(priv)) 1367 gve_rx_stop_ring_gqi(priv, i); 1368 else 1369 gve_rx_stop_ring_dqo(priv, i); 1370 } 1371 1372 static void gve_rx_stop_rings(struct gve_priv *priv, int num_rings) 1373 { 1374 int i; 1375 1376 if (!priv->rx) 1377 return; 1378 1379 for (i = 0; i < num_rings; i++) 1380 gve_rx_stop_ring(priv, i); 1381 } 1382 1383 static void gve_queues_mem_remove(struct gve_priv *priv) 1384 { 1385 struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; 1386 struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; 1387 1388 gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1389 gve_queues_mem_free(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1390 priv->tx = NULL; 1391 priv->rx = NULL; 1392 } 1393 1394 /* The passed-in queue memory is stored into priv and the queues are made live. 1395 * No memory is allocated. Passed-in memory is freed on errors. 1396 */ 1397 static int gve_queues_start(struct gve_priv *priv, 1398 struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, 1399 struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) 1400 { 1401 struct net_device *dev = priv->dev; 1402 int err; 1403 1404 /* Record new resources into priv */ 1405 priv->tx = tx_alloc_cfg->tx; 1406 priv->rx = rx_alloc_cfg->rx; 1407 1408 /* Record new configs into priv */ 1409 priv->tx_cfg = *tx_alloc_cfg->qcfg; 1410 priv->tx_cfg.num_xdp_queues = tx_alloc_cfg->num_xdp_rings; 1411 priv->rx_cfg = *rx_alloc_cfg->qcfg_rx; 1412 priv->tx_desc_cnt = tx_alloc_cfg->ring_size; 1413 priv->rx_desc_cnt = rx_alloc_cfg->ring_size; 1414 priv->tx_pages_per_qpl = tx_alloc_cfg->pages_per_qpl; 1415 priv->rx_pages_per_qpl = rx_alloc_cfg->pages_per_qpl; 1416 1417 gve_tx_start_rings(priv, gve_num_tx_queues(priv)); 1418 gve_rx_start_rings(priv, rx_alloc_cfg->qcfg_rx->num_queues); 1419 gve_init_sync_stats(priv); 1420 1421 err = netif_set_real_num_tx_queues(dev, priv->tx_cfg.num_queues); 1422 if (err) 1423 goto stop_and_free_rings; 1424 err = netif_set_real_num_rx_queues(dev, priv->rx_cfg.num_queues); 1425 if (err) 1426 goto stop_and_free_rings; 1427 1428 err = gve_reg_xdp_info(priv, dev); 1429 if (err) 1430 goto stop_and_free_rings; 1431 1432 if (rx_alloc_cfg->reset_rss) { 1433 err = gve_init_rss_config(priv, priv->rx_cfg.num_queues); 1434 if (err) 1435 goto reset; 1436 } 1437 1438 err = gve_register_qpls(priv); 1439 if (err) 1440 goto reset; 1441 1442 priv->header_split_enabled = rx_alloc_cfg->enable_header_split; 1443 priv->rx_cfg.packet_buffer_size = rx_alloc_cfg->packet_buffer_size; 1444 1445 err = gve_create_rings(priv); 1446 if (err) 1447 goto reset; 1448 1449 gve_set_device_rings_ok(priv); 1450 1451 if (gve_get_report_stats(priv)) 1452 mod_timer(&priv->stats_report_timer, 1453 round_jiffies(jiffies + 1454 msecs_to_jiffies(priv->stats_report_timer_period))); 1455 1456 gve_turnup(priv); 1457 queue_work(priv->gve_wq, &priv->service_task); 1458 priv->interface_up_cnt++; 1459 return 0; 1460 1461 reset: 1462 if (gve_get_reset_in_progress(priv)) 1463 goto stop_and_free_rings; 1464 gve_reset_and_teardown(priv, true); 1465 /* if this fails there is nothing we can do so just ignore the return */ 1466 gve_reset_recovery(priv, false); 1467 /* return the original error */ 1468 return err; 1469 stop_and_free_rings: 1470 gve_tx_stop_rings(priv, gve_num_tx_queues(priv)); 1471 gve_rx_stop_rings(priv, priv->rx_cfg.num_queues); 1472 gve_queues_mem_remove(priv); 1473 return err; 1474 } 1475 1476 static int gve_open(struct net_device *dev) 1477 { 1478 struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; 1479 struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; 1480 struct gve_priv *priv = netdev_priv(dev); 1481 int err; 1482 1483 gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1484 1485 err = gve_queues_mem_alloc(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1486 if (err) 1487 return err; 1488 1489 /* No need to free on error: ownership of resources is lost after 1490 * calling gve_queues_start. 1491 */ 1492 err = gve_queues_start(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1493 if (err) 1494 return err; 1495 1496 return 0; 1497 } 1498 1499 static int gve_queues_stop(struct gve_priv *priv) 1500 { 1501 int err; 1502 1503 netif_carrier_off(priv->dev); 1504 if (gve_get_device_rings_ok(priv)) { 1505 gve_turndown(priv); 1506 gve_drain_page_cache(priv); 1507 err = gve_destroy_rings(priv); 1508 if (err) 1509 goto err; 1510 err = gve_unregister_qpls(priv); 1511 if (err) 1512 goto err; 1513 gve_clear_device_rings_ok(priv); 1514 } 1515 timer_delete_sync(&priv->stats_report_timer); 1516 1517 gve_unreg_xdp_info(priv); 1518 1519 gve_tx_stop_rings(priv, gve_num_tx_queues(priv)); 1520 gve_rx_stop_rings(priv, priv->rx_cfg.num_queues); 1521 1522 priv->interface_down_cnt++; 1523 return 0; 1524 1525 err: 1526 /* This must have been called from a reset due to the rtnl lock 1527 * so just return at this point. 1528 */ 1529 if (gve_get_reset_in_progress(priv)) 1530 return err; 1531 /* Otherwise reset before returning */ 1532 gve_reset_and_teardown(priv, true); 1533 return gve_reset_recovery(priv, false); 1534 } 1535 1536 static int gve_close(struct net_device *dev) 1537 { 1538 struct gve_priv *priv = netdev_priv(dev); 1539 int err; 1540 1541 err = gve_queues_stop(priv); 1542 if (err) 1543 return err; 1544 1545 gve_queues_mem_remove(priv); 1546 return 0; 1547 } 1548 1549 static void gve_handle_link_status(struct gve_priv *priv, bool link_status) 1550 { 1551 if (!gve_get_napi_enabled(priv)) 1552 return; 1553 1554 if (link_status == netif_carrier_ok(priv->dev)) 1555 return; 1556 1557 if (link_status) { 1558 netdev_info(priv->dev, "Device link is up.\n"); 1559 netif_carrier_on(priv->dev); 1560 } else { 1561 netdev_info(priv->dev, "Device link is down.\n"); 1562 netif_carrier_off(priv->dev); 1563 } 1564 } 1565 1566 static int gve_configure_rings_xdp(struct gve_priv *priv, 1567 u16 num_xdp_rings) 1568 { 1569 struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; 1570 struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; 1571 1572 gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1573 tx_alloc_cfg.num_xdp_rings = num_xdp_rings; 1574 1575 rx_alloc_cfg.xdp = !!num_xdp_rings; 1576 return gve_adjust_config(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1577 } 1578 1579 static int gve_set_xdp(struct gve_priv *priv, struct bpf_prog *prog, 1580 struct netlink_ext_ack *extack) 1581 { 1582 struct bpf_prog *old_prog; 1583 int err = 0; 1584 u32 status; 1585 1586 old_prog = READ_ONCE(priv->xdp_prog); 1587 if (!netif_running(priv->dev)) { 1588 WRITE_ONCE(priv->xdp_prog, prog); 1589 if (old_prog) 1590 bpf_prog_put(old_prog); 1591 1592 /* Update priv XDP queue configuration */ 1593 priv->tx_cfg.num_xdp_queues = priv->xdp_prog ? 1594 priv->rx_cfg.num_queues : 0; 1595 return 0; 1596 } 1597 1598 if (!old_prog && prog) 1599 err = gve_configure_rings_xdp(priv, priv->rx_cfg.num_queues); 1600 else if (old_prog && !prog) 1601 err = gve_configure_rings_xdp(priv, 0); 1602 1603 if (err) 1604 goto out; 1605 1606 WRITE_ONCE(priv->xdp_prog, prog); 1607 if (old_prog) 1608 bpf_prog_put(old_prog); 1609 1610 out: 1611 status = ioread32be(&priv->reg_bar0->device_status); 1612 gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status); 1613 return err; 1614 } 1615 1616 static int gve_xdp_xmit(struct net_device *dev, int n, 1617 struct xdp_frame **frames, u32 flags) 1618 { 1619 struct gve_priv *priv = netdev_priv(dev); 1620 1621 if (priv->queue_format == GVE_GQI_QPL_FORMAT) 1622 return gve_xdp_xmit_gqi(dev, n, frames, flags); 1623 else if (priv->queue_format == GVE_DQO_RDA_FORMAT) 1624 return gve_xdp_xmit_dqo(dev, n, frames, flags); 1625 1626 return -EOPNOTSUPP; 1627 } 1628 1629 static int gve_xsk_pool_enable(struct net_device *dev, 1630 struct xsk_buff_pool *pool, 1631 u16 qid) 1632 { 1633 struct gve_priv *priv = netdev_priv(dev); 1634 int err; 1635 1636 if (qid >= priv->rx_cfg.num_queues) { 1637 dev_err(&priv->pdev->dev, "xsk pool invalid qid %d", qid); 1638 return -EINVAL; 1639 } 1640 if (xsk_pool_get_rx_frame_size(pool) < 1641 priv->dev->max_mtu + sizeof(struct ethhdr)) { 1642 dev_err(&priv->pdev->dev, "xsk pool frame_len too small"); 1643 return -EINVAL; 1644 } 1645 1646 err = xsk_pool_dma_map(pool, &priv->pdev->dev, 1647 DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); 1648 if (err) 1649 return err; 1650 1651 set_bit(qid, priv->xsk_pools); 1652 1653 /* If XDP prog is not installed or interface is down, return. */ 1654 if (!priv->xdp_prog || !netif_running(dev)) 1655 return 0; 1656 1657 err = gve_reg_xsk_pool(priv, dev, pool, qid); 1658 if (err) 1659 goto err_xsk_pool_dma_mapped; 1660 1661 /* Stop and start RDA queues to repost buffers. */ 1662 if (!gve_is_qpl(priv)) { 1663 err = gve_configure_rings_xdp(priv, priv->rx_cfg.num_queues); 1664 if (err) 1665 goto err_xsk_pool_registered; 1666 } 1667 return 0; 1668 1669 err_xsk_pool_registered: 1670 gve_unreg_xsk_pool(priv, qid); 1671 err_xsk_pool_dma_mapped: 1672 clear_bit(qid, priv->xsk_pools); 1673 xsk_pool_dma_unmap(pool, 1674 DMA_ATTR_SKIP_CPU_SYNC | 1675 DMA_ATTR_WEAK_ORDERING); 1676 return err; 1677 } 1678 1679 static int gve_xsk_pool_disable(struct net_device *dev, 1680 u16 qid) 1681 { 1682 struct gve_priv *priv = netdev_priv(dev); 1683 struct napi_struct *napi_rx; 1684 struct napi_struct *napi_tx; 1685 struct xsk_buff_pool *pool; 1686 int tx_qid; 1687 int err; 1688 1689 if (qid >= priv->rx_cfg.num_queues) 1690 return -EINVAL; 1691 1692 clear_bit(qid, priv->xsk_pools); 1693 1694 pool = xsk_get_pool_from_qid(dev, qid); 1695 if (pool) 1696 xsk_pool_dma_unmap(pool, 1697 DMA_ATTR_SKIP_CPU_SYNC | 1698 DMA_ATTR_WEAK_ORDERING); 1699 1700 if (!netif_running(dev) || !priv->tx_cfg.num_xdp_queues) 1701 return 0; 1702 1703 /* Stop and start RDA queues to repost buffers. */ 1704 if (!gve_is_qpl(priv) && priv->xdp_prog) { 1705 err = gve_configure_rings_xdp(priv, priv->rx_cfg.num_queues); 1706 if (err) 1707 return err; 1708 } 1709 1710 napi_rx = &priv->ntfy_blocks[priv->rx[qid].ntfy_id].napi; 1711 napi_disable_locked(napi_rx); /* make sure current rx poll is done */ 1712 1713 tx_qid = gve_xdp_tx_queue_id(priv, qid); 1714 napi_tx = &priv->ntfy_blocks[priv->tx[tx_qid].ntfy_id].napi; 1715 napi_disable_locked(napi_tx); /* make sure current tx poll is done */ 1716 1717 gve_unreg_xsk_pool(priv, qid); 1718 smp_mb(); /* Make sure it is visible to the workers on datapath */ 1719 1720 napi_enable_locked(napi_rx); 1721 napi_enable_locked(napi_tx); 1722 if (gve_is_gqi(priv)) { 1723 if (gve_rx_work_pending(&priv->rx[qid])) 1724 napi_schedule(napi_rx); 1725 1726 if (gve_tx_clean_pending(priv, &priv->tx[tx_qid])) 1727 napi_schedule(napi_tx); 1728 } 1729 1730 return 0; 1731 } 1732 1733 static int gve_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) 1734 { 1735 struct gve_priv *priv = netdev_priv(dev); 1736 struct napi_struct *napi; 1737 1738 if (!gve_get_napi_enabled(priv)) 1739 return -ENETDOWN; 1740 1741 if (queue_id >= priv->rx_cfg.num_queues || !priv->xdp_prog) 1742 return -EINVAL; 1743 1744 napi = &priv->ntfy_blocks[gve_rx_idx_to_ntfy(priv, queue_id)].napi; 1745 if (!napi_if_scheduled_mark_missed(napi)) { 1746 /* Call local_bh_enable to trigger SoftIRQ processing */ 1747 local_bh_disable(); 1748 napi_schedule(napi); 1749 local_bh_enable(); 1750 } 1751 1752 return 0; 1753 } 1754 1755 static int gve_verify_xdp_configuration(struct net_device *dev, 1756 struct netlink_ext_ack *extack) 1757 { 1758 struct gve_priv *priv = netdev_priv(dev); 1759 u16 max_xdp_mtu; 1760 1761 if (dev->features & NETIF_F_GRO_HW) { 1762 NL_SET_ERR_MSG_MOD(extack, 1763 "XDP is not supported when HW-GRO is on."); 1764 return -EOPNOTSUPP; 1765 } 1766 1767 if (priv->header_split_enabled) { 1768 NL_SET_ERR_MSG_MOD(extack, 1769 "XDP is not supported when header-data split is enabled."); 1770 return -EOPNOTSUPP; 1771 } 1772 1773 if (priv->rx_cfg.packet_buffer_size != SZ_2K) { 1774 NL_SET_ERR_MSG_FMT_MOD(extack, 1775 "XDP is not supported for Rx buf len %d, only %d supported.", 1776 priv->rx_cfg.packet_buffer_size, SZ_2K); 1777 return -EOPNOTSUPP; 1778 } 1779 1780 max_xdp_mtu = priv->rx_cfg.packet_buffer_size - sizeof(struct ethhdr); 1781 if (priv->queue_format == GVE_GQI_QPL_FORMAT) 1782 max_xdp_mtu -= GVE_RX_PAD; 1783 1784 if (dev->mtu > max_xdp_mtu) { 1785 NL_SET_ERR_MSG_FMT_MOD(extack, 1786 "XDP is not supported for mtu %d.", 1787 dev->mtu); 1788 return -EOPNOTSUPP; 1789 } 1790 1791 if (priv->rx_cfg.num_queues != priv->tx_cfg.num_queues || 1792 (2 * priv->tx_cfg.num_queues > priv->tx_cfg.max_queues)) { 1793 netdev_warn(dev, 1794 "XDP load failed: The number of configured RX queues %d should be equal to the number of configured TX queues %d and the number of configured RX/TX queues should be less than or equal to half the maximum number of RX/TX queues %d.", 1795 priv->rx_cfg.num_queues, priv->tx_cfg.num_queues, 1796 priv->tx_cfg.max_queues); 1797 NL_SET_ERR_MSG_MOD(extack, 1798 "XDP load failed: The number of configured RX queues should be equal to the number of configured TX queues and the number of configured RX/TX queues should be less than or equal to half the maximum number of RX/TX queues"); 1799 return -EINVAL; 1800 } 1801 return 0; 1802 } 1803 1804 static int gve_xdp(struct net_device *dev, struct netdev_bpf *xdp) 1805 { 1806 struct gve_priv *priv = netdev_priv(dev); 1807 int err; 1808 1809 err = gve_verify_xdp_configuration(dev, xdp->extack); 1810 if (err) 1811 return err; 1812 switch (xdp->command) { 1813 case XDP_SETUP_PROG: 1814 return gve_set_xdp(priv, xdp->prog, xdp->extack); 1815 case XDP_SETUP_XSK_POOL: 1816 if (xdp->xsk.pool) 1817 return gve_xsk_pool_enable(dev, xdp->xsk.pool, xdp->xsk.queue_id); 1818 else 1819 return gve_xsk_pool_disable(dev, xdp->xsk.queue_id); 1820 default: 1821 return -EINVAL; 1822 } 1823 } 1824 1825 int gve_init_rss_config(struct gve_priv *priv, u16 num_queues) 1826 { 1827 struct gve_rss_config *rss_config = &priv->rss_config; 1828 struct ethtool_rxfh_param rxfh = {0}; 1829 u16 i; 1830 1831 if (!priv->cache_rss_config) 1832 return 0; 1833 1834 for (i = 0; i < priv->rss_lut_size; i++) 1835 rss_config->hash_lut[i] = 1836 ethtool_rxfh_indir_default(i, num_queues); 1837 1838 netdev_rss_key_fill(rss_config->hash_key, priv->rss_key_size); 1839 1840 rxfh.hfunc = ETH_RSS_HASH_TOP; 1841 1842 return gve_adminq_configure_rss(priv, &rxfh); 1843 } 1844 1845 int gve_flow_rules_reset(struct gve_priv *priv) 1846 { 1847 if (!priv->max_flow_rules) 1848 return 0; 1849 1850 return gve_adminq_reset_flow_rules(priv); 1851 } 1852 1853 int gve_adjust_config(struct gve_priv *priv, 1854 struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, 1855 struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) 1856 { 1857 int err; 1858 1859 /* Allocate resources for the new configuration */ 1860 err = gve_queues_mem_alloc(priv, tx_alloc_cfg, rx_alloc_cfg); 1861 if (err) { 1862 netif_err(priv, drv, priv->dev, 1863 "Adjust config failed to alloc new queues"); 1864 return err; 1865 } 1866 1867 /* Teardown the device and free existing resources */ 1868 err = gve_close(priv->dev); 1869 if (err) { 1870 netif_err(priv, drv, priv->dev, 1871 "Adjust config failed to close old queues"); 1872 gve_queues_mem_free(priv, tx_alloc_cfg, rx_alloc_cfg); 1873 return err; 1874 } 1875 1876 /* Bring the device back up again with the new resources. */ 1877 err = gve_queues_start(priv, tx_alloc_cfg, rx_alloc_cfg); 1878 if (err) { 1879 netif_err(priv, drv, priv->dev, 1880 "Adjust config failed to start new queues, !!! DISABLING ALL QUEUES !!!\n"); 1881 /* No need to free on error: ownership of resources is lost after 1882 * calling gve_queues_start. 1883 */ 1884 gve_turndown(priv); 1885 return err; 1886 } 1887 1888 return 0; 1889 } 1890 1891 int gve_adjust_queues(struct gve_priv *priv, 1892 struct gve_rx_queue_config new_rx_config, 1893 struct gve_tx_queue_config new_tx_config, 1894 bool reset_rss) 1895 { 1896 struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; 1897 struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; 1898 int err; 1899 1900 gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1901 1902 /* Relay the new config from ethtool */ 1903 tx_alloc_cfg.qcfg = &new_tx_config; 1904 rx_alloc_cfg.qcfg_tx = &new_tx_config; 1905 rx_alloc_cfg.qcfg_rx = &new_rx_config; 1906 rx_alloc_cfg.reset_rss = reset_rss; 1907 1908 if (netif_running(priv->dev)) { 1909 err = gve_adjust_config(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1910 return err; 1911 } 1912 /* Set the config for the next up. */ 1913 if (reset_rss) { 1914 err = gve_init_rss_config(priv, new_rx_config.num_queues); 1915 if (err) 1916 return err; 1917 } 1918 priv->tx_cfg = new_tx_config; 1919 priv->rx_cfg = new_rx_config; 1920 1921 return 0; 1922 } 1923 1924 static void gve_turndown(struct gve_priv *priv) 1925 { 1926 int idx; 1927 1928 if (netif_carrier_ok(priv->dev)) 1929 netif_carrier_off(priv->dev); 1930 1931 if (!gve_get_napi_enabled(priv)) 1932 return; 1933 1934 /* Disable napi to prevent more work from coming in */ 1935 for (idx = 0; idx < gve_num_tx_queues(priv); idx++) { 1936 int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx); 1937 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 1938 1939 if (!gve_tx_was_added_to_block(priv, idx)) 1940 continue; 1941 1942 if (idx < priv->tx_cfg.num_queues) 1943 netif_queue_set_napi(priv->dev, idx, 1944 NETDEV_QUEUE_TYPE_TX, NULL); 1945 1946 napi_disable_locked(&block->napi); 1947 } 1948 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { 1949 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx); 1950 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 1951 1952 if (!gve_rx_was_added_to_block(priv, idx)) 1953 continue; 1954 1955 netif_queue_set_napi(priv->dev, idx, NETDEV_QUEUE_TYPE_RX, 1956 NULL); 1957 napi_disable_locked(&block->napi); 1958 } 1959 1960 /* Stop tx queues */ 1961 netif_tx_disable(priv->dev); 1962 1963 xdp_features_clear_redirect_target_locked(priv->dev); 1964 1965 gve_clear_napi_enabled(priv); 1966 gve_clear_report_stats(priv); 1967 1968 /* Make sure that all traffic is finished processing. */ 1969 synchronize_net(); 1970 } 1971 1972 static void gve_turnup(struct gve_priv *priv) 1973 { 1974 int idx; 1975 1976 /* Start the tx queues */ 1977 netif_tx_start_all_queues(priv->dev); 1978 1979 /* Enable napi and unmask interrupts for all queues */ 1980 for (idx = 0; idx < gve_num_tx_queues(priv); idx++) { 1981 int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx); 1982 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 1983 1984 if (!gve_tx_was_added_to_block(priv, idx)) 1985 continue; 1986 1987 napi_enable_locked(&block->napi); 1988 1989 if (idx < priv->tx_cfg.num_queues) 1990 netif_queue_set_napi(priv->dev, idx, 1991 NETDEV_QUEUE_TYPE_TX, 1992 &block->napi); 1993 1994 if (gve_is_gqi(priv)) { 1995 iowrite32be(0, gve_irq_doorbell(priv, block)); 1996 } else { 1997 gve_set_itr_coalesce_usecs_dqo(priv, block, 1998 priv->tx_coalesce_usecs); 1999 } 2000 2001 /* Any descs written by the NIC before this barrier will be 2002 * handled by the one-off napi schedule below. Whereas any 2003 * descs after the barrier will generate interrupts. 2004 */ 2005 mb(); 2006 napi_schedule(&block->napi); 2007 } 2008 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { 2009 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx); 2010 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 2011 2012 if (!gve_rx_was_added_to_block(priv, idx)) 2013 continue; 2014 2015 napi_enable_locked(&block->napi); 2016 netif_queue_set_napi(priv->dev, idx, NETDEV_QUEUE_TYPE_RX, 2017 &block->napi); 2018 2019 if (gve_is_gqi(priv)) { 2020 iowrite32be(0, gve_irq_doorbell(priv, block)); 2021 } else { 2022 gve_set_itr_coalesce_usecs_dqo(priv, block, 2023 priv->rx_coalesce_usecs); 2024 } 2025 2026 /* Any descs written by the NIC before this barrier will be 2027 * handled by the one-off napi schedule below. Whereas any 2028 * descs after the barrier will generate interrupts. 2029 */ 2030 mb(); 2031 napi_schedule(&block->napi); 2032 } 2033 2034 if (priv->tx_cfg.num_xdp_queues && gve_supports_xdp_xmit(priv)) 2035 xdp_features_set_redirect_target_locked(priv->dev, false); 2036 2037 gve_set_napi_enabled(priv); 2038 } 2039 2040 static void gve_turnup_and_check_status(struct gve_priv *priv) 2041 { 2042 u32 status; 2043 2044 gve_turnup(priv); 2045 status = ioread32be(&priv->reg_bar0->device_status); 2046 gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status); 2047 } 2048 2049 static struct gve_notify_block *gve_get_tx_notify_block(struct gve_priv *priv, 2050 unsigned int txqueue) 2051 { 2052 u32 ntfy_idx; 2053 2054 if (txqueue > priv->tx_cfg.num_queues) 2055 return NULL; 2056 2057 ntfy_idx = gve_tx_idx_to_ntfy(priv, txqueue); 2058 if (ntfy_idx >= priv->num_ntfy_blks) 2059 return NULL; 2060 2061 return &priv->ntfy_blocks[ntfy_idx]; 2062 } 2063 2064 static bool gve_tx_timeout_try_q_kick(struct gve_priv *priv, 2065 unsigned int txqueue) 2066 { 2067 struct gve_notify_block *block; 2068 u32 current_time; 2069 2070 block = gve_get_tx_notify_block(priv, txqueue); 2071 2072 if (!block) 2073 return false; 2074 2075 current_time = jiffies_to_msecs(jiffies); 2076 if (block->tx->last_kick_msec + MIN_TX_TIMEOUT_GAP > current_time) 2077 return false; 2078 2079 netdev_info(priv->dev, "Kicking queue %d", txqueue); 2080 napi_schedule(&block->napi); 2081 block->tx->last_kick_msec = current_time; 2082 return true; 2083 } 2084 2085 static void gve_tx_timeout(struct net_device *dev, unsigned int txqueue) 2086 { 2087 struct gve_notify_block *block; 2088 struct gve_priv *priv; 2089 2090 netdev_info(dev, "Timeout on tx queue, %d", txqueue); 2091 priv = netdev_priv(dev); 2092 2093 if (!gve_tx_timeout_try_q_kick(priv, txqueue)) 2094 gve_schedule_reset(priv); 2095 2096 block = gve_get_tx_notify_block(priv, txqueue); 2097 if (block) 2098 block->tx->queue_timeout++; 2099 priv->tx_timeo_cnt++; 2100 } 2101 2102 /* Header split is only supported on DQ RDA queue format. If XDP is enabled, 2103 * header split is not allowed. 2104 */ 2105 bool gve_header_split_supported(const struct gve_priv *priv) 2106 { 2107 return priv->header_buf_size && 2108 priv->queue_format == GVE_DQO_RDA_FORMAT && !priv->xdp_prog; 2109 } 2110 2111 int gve_set_rx_buf_len_config(struct gve_priv *priv, u32 rx_buf_len, 2112 struct netlink_ext_ack *extack, 2113 struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) 2114 { 2115 u32 old_rx_buf_len = rx_alloc_cfg->packet_buffer_size; 2116 2117 if (rx_buf_len == old_rx_buf_len) 2118 return 0; 2119 2120 /* device options may not always contain support for 4K buffers */ 2121 if (!gve_is_dqo(priv) || priv->max_rx_buffer_size < SZ_4K) { 2122 NL_SET_ERR_MSG_MOD(extack, 2123 "Modifying Rx buf len is not supported"); 2124 return -EOPNOTSUPP; 2125 } 2126 2127 if (priv->xdp_prog && rx_buf_len != SZ_2K) { 2128 NL_SET_ERR_MSG_MOD(extack, 2129 "Rx buf len can only be 2048 when XDP is on"); 2130 return -EINVAL; 2131 } 2132 2133 if (rx_buf_len != SZ_2K && rx_buf_len != SZ_4K) { 2134 NL_SET_ERR_MSG_MOD(extack, 2135 "Rx buf len can only be 2048 or 4096"); 2136 return -EINVAL; 2137 } 2138 rx_alloc_cfg->packet_buffer_size = rx_buf_len; 2139 2140 return 0; 2141 } 2142 2143 int gve_set_hsplit_config(struct gve_priv *priv, u8 tcp_data_split, 2144 struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) 2145 { 2146 bool enable_hdr_split; 2147 2148 if (tcp_data_split == ETHTOOL_TCP_DATA_SPLIT_UNKNOWN) 2149 return 0; 2150 2151 if (!gve_header_split_supported(priv)) { 2152 dev_err(&priv->pdev->dev, "Header-split not supported\n"); 2153 return -EOPNOTSUPP; 2154 } 2155 2156 if (tcp_data_split == ETHTOOL_TCP_DATA_SPLIT_ENABLED) 2157 enable_hdr_split = true; 2158 else 2159 enable_hdr_split = false; 2160 2161 if (enable_hdr_split == priv->header_split_enabled) 2162 return 0; 2163 2164 rx_alloc_cfg->enable_header_split = enable_hdr_split; 2165 2166 return 0; 2167 } 2168 2169 static int gve_set_features(struct net_device *netdev, 2170 netdev_features_t features) 2171 { 2172 const netdev_features_t orig_features = netdev->features; 2173 struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; 2174 struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; 2175 struct gve_priv *priv = netdev_priv(netdev); 2176 int err; 2177 2178 gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); 2179 2180 if ((netdev->features & NETIF_F_GRO_HW) != 2181 (features & NETIF_F_GRO_HW)) { 2182 netdev->features ^= NETIF_F_GRO_HW; 2183 if (priv->xdp_prog && (netdev->features & NETIF_F_GRO_HW)) { 2184 netdev_warn(netdev, 2185 "HW-GRO is not supported when XDP is on."); 2186 err = -EOPNOTSUPP; 2187 goto revert_features; 2188 } 2189 if (netif_running(netdev)) { 2190 err = gve_adjust_config(priv, &tx_alloc_cfg, &rx_alloc_cfg); 2191 if (err) 2192 goto revert_features; 2193 } 2194 } 2195 if ((netdev->features & NETIF_F_NTUPLE) && !(features & NETIF_F_NTUPLE)) { 2196 err = gve_flow_rules_reset(priv); 2197 if (err) 2198 goto revert_features; 2199 } 2200 2201 return 0; 2202 2203 revert_features: 2204 netdev->features = orig_features; 2205 return err; 2206 } 2207 2208 static int gve_get_ts_config(struct net_device *dev, 2209 struct kernel_hwtstamp_config *kernel_config) 2210 { 2211 struct gve_priv *priv = netdev_priv(dev); 2212 2213 *kernel_config = priv->ts_config; 2214 return 0; 2215 } 2216 2217 static int gve_set_ts_config(struct net_device *dev, 2218 struct kernel_hwtstamp_config *kernel_config, 2219 struct netlink_ext_ack *extack) 2220 { 2221 struct gve_priv *priv = netdev_priv(dev); 2222 2223 if (kernel_config->tx_type != HWTSTAMP_TX_OFF) { 2224 NL_SET_ERR_MSG_MOD(extack, "TX timestamping is not supported"); 2225 return -ERANGE; 2226 } 2227 2228 if (kernel_config->rx_filter != HWTSTAMP_FILTER_NONE) { 2229 if (!gve_is_clock_enabled(priv)) { 2230 NL_SET_ERR_MSG_MOD(extack, 2231 "RX timestamping is not supported"); 2232 kernel_config->rx_filter = HWTSTAMP_FILTER_NONE; 2233 return -EOPNOTSUPP; 2234 } 2235 2236 kernel_config->rx_filter = HWTSTAMP_FILTER_ALL; 2237 } 2238 2239 priv->ts_config.rx_filter = kernel_config->rx_filter; 2240 2241 return 0; 2242 } 2243 2244 static const struct net_device_ops gve_netdev_ops = { 2245 .ndo_start_xmit = gve_start_xmit, 2246 .ndo_features_check = gve_features_check, 2247 .ndo_open = gve_open, 2248 .ndo_stop = gve_close, 2249 .ndo_get_stats64 = gve_get_stats, 2250 .ndo_tx_timeout = gve_tx_timeout, 2251 .ndo_set_features = gve_set_features, 2252 .ndo_bpf = gve_xdp, 2253 .ndo_xdp_xmit = gve_xdp_xmit, 2254 .ndo_xsk_wakeup = gve_xsk_wakeup, 2255 .ndo_hwtstamp_get = gve_get_ts_config, 2256 .ndo_hwtstamp_set = gve_set_ts_config, 2257 }; 2258 2259 static void gve_handle_status(struct gve_priv *priv, u32 status) 2260 { 2261 if (GVE_DEVICE_STATUS_RESET_MASK & status) { 2262 dev_info(&priv->pdev->dev, "Device requested reset.\n"); 2263 gve_set_do_reset(priv); 2264 } 2265 if (GVE_DEVICE_STATUS_REPORT_STATS_MASK & status) { 2266 priv->stats_report_trigger_cnt++; 2267 gve_set_do_report_stats(priv); 2268 } 2269 } 2270 2271 static void gve_handle_reset(struct gve_priv *priv) 2272 { 2273 /* A service task will be scheduled at the end of probe to catch any 2274 * resets that need to happen, and we don't want to reset until 2275 * probe is done. 2276 */ 2277 if (gve_get_probe_in_progress(priv)) 2278 return; 2279 2280 if (gve_get_do_reset(priv)) { 2281 rtnl_lock(); 2282 netdev_lock(priv->dev); 2283 gve_reset(priv, false); 2284 netdev_unlock(priv->dev); 2285 rtnl_unlock(); 2286 } 2287 } 2288 2289 void gve_handle_report_stats(struct gve_priv *priv) 2290 { 2291 struct stats *stats = priv->stats_report->stats; 2292 int idx, stats_idx = 0; 2293 unsigned int start = 0; 2294 u64 tx_bytes; 2295 2296 if (!gve_get_report_stats(priv)) 2297 return; 2298 2299 be64_add_cpu(&priv->stats_report->written_count, 1); 2300 /* tx stats */ 2301 if (priv->tx) { 2302 for (idx = 0; idx < gve_num_tx_queues(priv); idx++) { 2303 u32 last_completion = 0; 2304 u32 tx_frames = 0; 2305 2306 /* DQO doesn't currently support these metrics. */ 2307 if (gve_is_gqi(priv)) { 2308 last_completion = priv->tx[idx].done; 2309 tx_frames = priv->tx[idx].req; 2310 } 2311 2312 do { 2313 start = u64_stats_fetch_begin(&priv->tx[idx].statss); 2314 tx_bytes = priv->tx[idx].bytes_done; 2315 } while (u64_stats_fetch_retry(&priv->tx[idx].statss, start)); 2316 stats[stats_idx++] = (struct stats) { 2317 .stat_name = cpu_to_be32(TX_WAKE_CNT), 2318 .value = cpu_to_be64(priv->tx[idx].wake_queue), 2319 .queue_id = cpu_to_be32(idx), 2320 }; 2321 stats[stats_idx++] = (struct stats) { 2322 .stat_name = cpu_to_be32(TX_STOP_CNT), 2323 .value = cpu_to_be64(priv->tx[idx].stop_queue), 2324 .queue_id = cpu_to_be32(idx), 2325 }; 2326 stats[stats_idx++] = (struct stats) { 2327 .stat_name = cpu_to_be32(TX_FRAMES_SENT), 2328 .value = cpu_to_be64(tx_frames), 2329 .queue_id = cpu_to_be32(idx), 2330 }; 2331 stats[stats_idx++] = (struct stats) { 2332 .stat_name = cpu_to_be32(TX_BYTES_SENT), 2333 .value = cpu_to_be64(tx_bytes), 2334 .queue_id = cpu_to_be32(idx), 2335 }; 2336 stats[stats_idx++] = (struct stats) { 2337 .stat_name = cpu_to_be32(TX_LAST_COMPLETION_PROCESSED), 2338 .value = cpu_to_be64(last_completion), 2339 .queue_id = cpu_to_be32(idx), 2340 }; 2341 stats[stats_idx++] = (struct stats) { 2342 .stat_name = cpu_to_be32(TX_TIMEOUT_CNT), 2343 .value = cpu_to_be64(priv->tx[idx].queue_timeout), 2344 .queue_id = cpu_to_be32(idx), 2345 }; 2346 } 2347 } 2348 /* rx stats */ 2349 if (priv->rx) { 2350 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { 2351 stats[stats_idx++] = (struct stats) { 2352 .stat_name = cpu_to_be32(RX_NEXT_EXPECTED_SEQUENCE), 2353 .value = cpu_to_be64(priv->rx[idx].desc.seqno), 2354 .queue_id = cpu_to_be32(idx), 2355 }; 2356 stats[stats_idx++] = (struct stats) { 2357 .stat_name = cpu_to_be32(RX_BUFFERS_POSTED), 2358 .value = cpu_to_be64(priv->rx[idx].fill_cnt), 2359 .queue_id = cpu_to_be32(idx), 2360 }; 2361 } 2362 } 2363 } 2364 2365 /* Handle NIC status register changes, reset requests and report stats */ 2366 static void gve_service_task(struct work_struct *work) 2367 { 2368 struct gve_priv *priv = container_of(work, struct gve_priv, 2369 service_task); 2370 u32 status = ioread32be(&priv->reg_bar0->device_status); 2371 2372 gve_handle_status(priv, status); 2373 2374 gve_handle_reset(priv); 2375 gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status); 2376 } 2377 2378 static void gve_set_netdev_xdp_features(struct gve_priv *priv) 2379 { 2380 xdp_features_t xdp_features; 2381 2382 if (priv->queue_format == GVE_GQI_QPL_FORMAT) { 2383 xdp_features = NETDEV_XDP_ACT_BASIC; 2384 xdp_features |= NETDEV_XDP_ACT_REDIRECT; 2385 xdp_features |= NETDEV_XDP_ACT_XSK_ZEROCOPY; 2386 } else if (priv->queue_format == GVE_DQO_RDA_FORMAT) { 2387 xdp_features = NETDEV_XDP_ACT_BASIC; 2388 xdp_features |= NETDEV_XDP_ACT_REDIRECT; 2389 xdp_features |= NETDEV_XDP_ACT_XSK_ZEROCOPY; 2390 } else { 2391 xdp_features = 0; 2392 } 2393 2394 xdp_set_features_flag_locked(priv->dev, xdp_features); 2395 } 2396 2397 static const struct xdp_metadata_ops gve_xdp_metadata_ops = { 2398 .xmo_rx_timestamp = gve_xdp_rx_timestamp, 2399 }; 2400 2401 static void gve_set_default_desc_cnt(struct gve_priv *priv, 2402 const struct gve_device_descriptor *descriptor) 2403 { 2404 priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries); 2405 priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries); 2406 2407 /* set default ranges */ 2408 priv->max_tx_desc_cnt = priv->tx_desc_cnt; 2409 priv->max_rx_desc_cnt = priv->rx_desc_cnt; 2410 priv->min_tx_desc_cnt = priv->tx_desc_cnt; 2411 priv->min_rx_desc_cnt = priv->rx_desc_cnt; 2412 } 2413 2414 void gve_set_queue_properties(struct gve_priv *priv, 2415 struct gve_device_descriptor *descriptor) 2416 { 2417 /* set default descriptor counts */ 2418 gve_set_default_desc_cnt(priv, descriptor); 2419 2420 priv->max_registered_pages = be64_to_cpu(descriptor->max_registered_pages); 2421 priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl); 2422 priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues); 2423 } 2424 2425 int gve_set_mtu(struct gve_priv *priv, 2426 struct gve_device_descriptor *descriptor) 2427 { 2428 u16 mtu; 2429 2430 mtu = be16_to_cpu(descriptor->mtu); 2431 if (mtu < ETH_MIN_MTU) { 2432 dev_err(&priv->pdev->dev, "MTU %d below minimum MTU\n", mtu); 2433 return -EINVAL; 2434 } 2435 priv->dev->max_mtu = mtu; 2436 2437 return 0; 2438 } 2439 2440 void gve_set_mac(struct gve_priv *priv, 2441 struct gve_device_descriptor *descriptor) 2442 { 2443 u8 *mac; 2444 2445 mac = descriptor->mac; 2446 eth_hw_addr_set(priv->dev, mac); 2447 dev_info(&priv->pdev->dev, "MAC addr: %pM\n", mac); 2448 } 2449 2450 static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device) 2451 { 2452 int err; 2453 2454 /* Set up the adminq */ 2455 err = gve_adminq_alloc(&priv->pdev->dev, priv); 2456 if (err) { 2457 dev_err(&priv->pdev->dev, 2458 "Failed to alloc admin queue: err=%d\n", err); 2459 return err; 2460 } 2461 2462 err = gve_verify_driver_compatibility(priv); 2463 if (err) { 2464 dev_err(&priv->pdev->dev, 2465 "Could not verify driver compatibility: err=%d\n", err); 2466 goto err; 2467 } 2468 2469 priv->num_registered_pages = 0; 2470 2471 if (skip_describe_device) 2472 goto setup_device; 2473 2474 priv->queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED; 2475 /* Get the initial information we need from the device */ 2476 err = gve_adminq_describe_device(priv); 2477 if (err) { 2478 dev_err(&priv->pdev->dev, 2479 "Could not get device information: err=%d\n", err); 2480 goto err; 2481 } 2482 2483 err = gve_set_num_ntfy_blks(priv); 2484 if (err) { 2485 dev_err(&priv->pdev->dev, 2486 "Could not setup notify blocks: err=%d\n", err); 2487 goto err; 2488 } 2489 2490 gve_set_num_queues(priv); 2491 dev_info(&priv->pdev->dev, "TX queues %d, RX queues %d\n", 2492 priv->tx_cfg.num_queues, priv->rx_cfg.num_queues); 2493 dev_info(&priv->pdev->dev, "Max TX queues %d, Max RX queues %d\n", 2494 priv->tx_cfg.max_queues, priv->rx_cfg.max_queues); 2495 2496 if (gve_is_dqo(priv)) { 2497 /* DQO supports HW-GRO and UDP_GSO */ 2498 u64 additional_features = NETIF_F_GRO_HW | NETIF_F_GSO_UDP_L4; 2499 2500 priv->dev->hw_features |= additional_features; 2501 priv->dev->features |= additional_features; 2502 2503 priv->tx_coalesce_usecs = GVE_TX_IRQ_RATELIMIT_US_DQO; 2504 priv->rx_coalesce_usecs = GVE_RX_IRQ_RATELIMIT_US_DQO; 2505 2506 /* Big TCP is only supported on DQO */ 2507 netif_set_tso_max_size(priv->dev, GVE_DQO_TX_MAX); 2508 } 2509 2510 priv->dev->mtu = priv->dev->max_mtu; 2511 priv->numa_node = dev_to_node(&priv->pdev->dev); 2512 priv->tx_cfg.num_xdp_queues = 0; 2513 priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK; 2514 priv->ts_config.tx_type = HWTSTAMP_TX_OFF; 2515 priv->ts_config.rx_filter = HWTSTAMP_FILTER_NONE; 2516 2517 setup_device: 2518 priv->xsk_pools = bitmap_zalloc(priv->rx_cfg.max_queues, GFP_KERNEL); 2519 if (!priv->xsk_pools) { 2520 err = -ENOMEM; 2521 goto err; 2522 } 2523 2524 gve_set_netdev_xdp_features(priv); 2525 if (!gve_is_gqi(priv)) 2526 priv->dev->xdp_metadata_ops = &gve_xdp_metadata_ops; 2527 2528 err = gve_setup_device_resources(priv); 2529 if (err) 2530 goto err_free_xsk_bitmap; 2531 2532 return 0; 2533 2534 err_free_xsk_bitmap: 2535 bitmap_free(priv->xsk_pools); 2536 priv->xsk_pools = NULL; 2537 err: 2538 gve_adminq_free(priv); 2539 return err; 2540 } 2541 2542 static void gve_teardown_priv_resources(struct gve_priv *priv) 2543 { 2544 gve_teardown_device_resources(priv); 2545 gve_adminq_free(priv); 2546 bitmap_free(priv->xsk_pools); 2547 priv->xsk_pools = NULL; 2548 } 2549 2550 static void gve_trigger_reset(struct gve_priv *priv) 2551 { 2552 /* Reset the device by releasing the AQ */ 2553 gve_adminq_release(priv); 2554 } 2555 2556 static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up) 2557 { 2558 gve_trigger_reset(priv); 2559 /* With the reset having already happened, close cannot fail */ 2560 if (was_up) 2561 gve_close(priv->dev); 2562 gve_teardown_priv_resources(priv); 2563 } 2564 2565 static int gve_reset_recovery(struct gve_priv *priv, bool was_up) 2566 { 2567 int err; 2568 2569 err = gve_init_priv(priv, true); 2570 if (err) 2571 goto err; 2572 if (was_up) { 2573 err = gve_open(priv->dev); 2574 if (err) 2575 goto err; 2576 } 2577 return 0; 2578 err: 2579 dev_err(&priv->pdev->dev, "Reset failed! !!! DISABLING ALL QUEUES !!!\n"); 2580 gve_turndown(priv); 2581 return err; 2582 } 2583 2584 int gve_reset(struct gve_priv *priv, bool attempt_teardown) 2585 { 2586 bool was_up = netif_running(priv->dev); 2587 int err; 2588 2589 dev_info(&priv->pdev->dev, "Performing reset\n"); 2590 gve_clear_do_reset(priv); 2591 gve_set_reset_in_progress(priv); 2592 /* If we aren't attempting to teardown normally, just go turndown and 2593 * reset right away. 2594 */ 2595 if (!attempt_teardown) { 2596 gve_turndown(priv); 2597 gve_reset_and_teardown(priv, was_up); 2598 } else { 2599 /* Otherwise attempt to close normally */ 2600 if (was_up) { 2601 err = gve_close(priv->dev); 2602 /* If that fails reset as we did above */ 2603 if (err) 2604 gve_reset_and_teardown(priv, was_up); 2605 } 2606 /* Clean up any remaining resources */ 2607 gve_teardown_priv_resources(priv); 2608 } 2609 2610 /* Set it all back up */ 2611 err = gve_reset_recovery(priv, was_up); 2612 gve_clear_reset_in_progress(priv); 2613 priv->reset_cnt++; 2614 priv->interface_up_cnt = 0; 2615 priv->interface_down_cnt = 0; 2616 priv->stats_report_trigger_cnt = 0; 2617 return err; 2618 } 2619 2620 static void gve_write_version(u8 __iomem *driver_version_register) 2621 { 2622 const char *c = gve_version_prefix; 2623 2624 while (*c) { 2625 writeb(*c, driver_version_register); 2626 c++; 2627 } 2628 2629 c = gve_version_str; 2630 while (*c) { 2631 writeb(*c, driver_version_register); 2632 c++; 2633 } 2634 writeb('\n', driver_version_register); 2635 } 2636 2637 static int gve_rx_queue_stop(struct net_device *dev, void *per_q_mem, int idx) 2638 { 2639 struct gve_priv *priv = netdev_priv(dev); 2640 struct gve_rx_ring *gve_per_q_mem; 2641 int err; 2642 2643 if (!priv->rx) 2644 return -EAGAIN; 2645 2646 /* Destroying queue 0 while other queues exist is not supported in DQO */ 2647 if (!gve_is_gqi(priv) && idx == 0) 2648 return -ERANGE; 2649 2650 /* Single-queue destruction requires quiescence on all queues */ 2651 gve_turndown(priv); 2652 2653 /* This failure will trigger a reset - no need to clean up */ 2654 err = gve_adminq_destroy_single_rx_queue(priv, idx); 2655 if (err) 2656 return err; 2657 2658 if (gve_is_qpl(priv)) { 2659 /* This failure will trigger a reset - no need to clean up */ 2660 err = gve_unregister_qpl(priv, gve_rx_get_qpl(priv, idx)); 2661 if (err) 2662 return err; 2663 } 2664 2665 gve_rx_stop_ring(priv, idx); 2666 2667 /* Turn the unstopped queues back up */ 2668 gve_turnup_and_check_status(priv); 2669 2670 gve_per_q_mem = (struct gve_rx_ring *)per_q_mem; 2671 *gve_per_q_mem = priv->rx[idx]; 2672 memset(&priv->rx[idx], 0, sizeof(priv->rx[idx])); 2673 return 0; 2674 } 2675 2676 static void gve_rx_queue_mem_free(struct net_device *dev, void *per_q_mem) 2677 { 2678 struct gve_priv *priv = netdev_priv(dev); 2679 struct gve_rx_alloc_rings_cfg cfg = {0}; 2680 struct gve_rx_ring *gve_per_q_mem; 2681 2682 gve_per_q_mem = (struct gve_rx_ring *)per_q_mem; 2683 gve_rx_get_curr_alloc_cfg(priv, &cfg); 2684 2685 if (gve_is_gqi(priv)) 2686 gve_rx_free_ring_gqi(priv, gve_per_q_mem, &cfg); 2687 else 2688 gve_rx_free_ring_dqo(priv, gve_per_q_mem, &cfg); 2689 } 2690 2691 static int gve_rx_queue_mem_alloc(struct net_device *dev, 2692 struct netdev_queue_config *qcfg, 2693 void *per_q_mem, int idx) 2694 { 2695 struct gve_priv *priv = netdev_priv(dev); 2696 struct gve_rx_alloc_rings_cfg cfg = {0}; 2697 struct gve_rx_ring *gve_per_q_mem; 2698 int err; 2699 2700 if (!priv->rx) 2701 return -EAGAIN; 2702 2703 gve_per_q_mem = (struct gve_rx_ring *)per_q_mem; 2704 gve_rx_get_curr_alloc_cfg(priv, &cfg); 2705 2706 if (gve_is_gqi(priv)) 2707 err = gve_rx_alloc_ring_gqi(priv, &cfg, gve_per_q_mem, idx); 2708 else 2709 err = gve_rx_alloc_ring_dqo(priv, &cfg, gve_per_q_mem, idx); 2710 2711 return err; 2712 } 2713 2714 static int gve_rx_queue_start(struct net_device *dev, 2715 struct netdev_queue_config *qcfg, 2716 void *per_q_mem, int idx) 2717 { 2718 struct gve_priv *priv = netdev_priv(dev); 2719 struct gve_rx_ring *gve_per_q_mem; 2720 int err; 2721 2722 if (!priv->rx) 2723 return -EAGAIN; 2724 2725 gve_per_q_mem = (struct gve_rx_ring *)per_q_mem; 2726 priv->rx[idx] = *gve_per_q_mem; 2727 2728 /* Single-queue creation requires quiescence on all queues */ 2729 gve_turndown(priv); 2730 2731 gve_rx_start_ring(priv, idx); 2732 2733 if (gve_is_qpl(priv)) { 2734 /* This failure will trigger a reset - no need to clean up */ 2735 err = gve_register_qpl(priv, gve_rx_get_qpl(priv, idx)); 2736 if (err) 2737 goto abort; 2738 } 2739 2740 /* This failure will trigger a reset - no need to clean up */ 2741 err = gve_adminq_create_single_rx_queue(priv, idx); 2742 if (err) 2743 goto abort; 2744 2745 if (gve_is_gqi(priv)) 2746 gve_rx_write_doorbell(priv, &priv->rx[idx]); 2747 else 2748 gve_rx_post_buffers_dqo(&priv->rx[idx]); 2749 2750 /* Turn the unstopped queues back up */ 2751 gve_turnup_and_check_status(priv); 2752 return 0; 2753 2754 abort: 2755 gve_rx_stop_ring(priv, idx); 2756 2757 /* All failures in this func result in a reset, by clearing the struct 2758 * at idx, we prevent a double free when that reset runs. The reset, 2759 * which needs the rtnl lock, will not run till this func returns and 2760 * its caller gives up the lock. 2761 */ 2762 memset(&priv->rx[idx], 0, sizeof(priv->rx[idx])); 2763 return err; 2764 } 2765 2766 static const struct netdev_queue_mgmt_ops gve_queue_mgmt_ops = { 2767 .ndo_queue_mem_size = sizeof(struct gve_rx_ring), 2768 .ndo_queue_mem_alloc = gve_rx_queue_mem_alloc, 2769 .ndo_queue_mem_free = gve_rx_queue_mem_free, 2770 .ndo_queue_start = gve_rx_queue_start, 2771 .ndo_queue_stop = gve_rx_queue_stop, 2772 }; 2773 2774 static void gve_get_rx_queue_stats(struct net_device *dev, int idx, 2775 struct netdev_queue_stats_rx *rx_stats) 2776 { 2777 struct gve_priv *priv = netdev_priv(dev); 2778 struct gve_rx_ring *rx = &priv->rx[idx]; 2779 unsigned int start; 2780 2781 do { 2782 start = u64_stats_fetch_begin(&rx->statss); 2783 rx_stats->packets = rx->rpackets; 2784 rx_stats->bytes = rx->rbytes; 2785 rx_stats->alloc_fail = rx->rx_skb_alloc_fail + 2786 rx->rx_buf_alloc_fail; 2787 } while (u64_stats_fetch_retry(&rx->statss, start)); 2788 } 2789 2790 static void gve_get_tx_queue_stats(struct net_device *dev, int idx, 2791 struct netdev_queue_stats_tx *tx_stats) 2792 { 2793 struct gve_priv *priv = netdev_priv(dev); 2794 struct gve_tx_ring *tx = &priv->tx[idx]; 2795 unsigned int start; 2796 2797 do { 2798 start = u64_stats_fetch_begin(&tx->statss); 2799 tx_stats->packets = tx->pkt_done; 2800 tx_stats->bytes = tx->bytes_done; 2801 } while (u64_stats_fetch_retry(&tx->statss, start)); 2802 } 2803 2804 static void gve_get_base_stats(struct net_device *dev, 2805 struct netdev_queue_stats_rx *rx, 2806 struct netdev_queue_stats_tx *tx) 2807 { 2808 rx->packets = 0; 2809 rx->bytes = 0; 2810 rx->alloc_fail = 0; 2811 2812 tx->packets = 0; 2813 tx->bytes = 0; 2814 } 2815 2816 static const struct netdev_stat_ops gve_stat_ops = { 2817 .get_queue_stats_rx = gve_get_rx_queue_stats, 2818 .get_queue_stats_tx = gve_get_tx_queue_stats, 2819 .get_base_stats = gve_get_base_stats, 2820 }; 2821 2822 static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 2823 { 2824 int max_tx_queues, max_rx_queues; 2825 struct net_device *dev; 2826 __be32 __iomem *db_bar; 2827 struct gve_registers __iomem *reg_bar; 2828 struct gve_priv *priv; 2829 int err; 2830 2831 err = pci_enable_device(pdev); 2832 if (err) 2833 return err; 2834 2835 err = pci_request_regions(pdev, gve_driver_name); 2836 if (err) 2837 goto abort_with_enabled; 2838 2839 pci_set_master(pdev); 2840 2841 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 2842 if (err) { 2843 dev_err(&pdev->dev, "Failed to set dma mask: err=%d\n", err); 2844 goto abort_with_pci_region; 2845 } 2846 2847 reg_bar = pci_iomap(pdev, GVE_REGISTER_BAR, 0); 2848 if (!reg_bar) { 2849 dev_err(&pdev->dev, "Failed to map pci bar!\n"); 2850 err = -ENOMEM; 2851 goto abort_with_pci_region; 2852 } 2853 2854 db_bar = pci_iomap(pdev, GVE_DOORBELL_BAR, 0); 2855 if (!db_bar) { 2856 dev_err(&pdev->dev, "Failed to map doorbell bar!\n"); 2857 err = -ENOMEM; 2858 goto abort_with_reg_bar; 2859 } 2860 2861 gve_write_version(®_bar->driver_version); 2862 /* Get max queues to alloc etherdev */ 2863 max_tx_queues = ioread32be(®_bar->max_tx_queues); 2864 max_rx_queues = ioread32be(®_bar->max_rx_queues); 2865 /* Alloc and setup the netdev and priv */ 2866 dev = alloc_etherdev_mqs(sizeof(*priv), max_tx_queues, max_rx_queues); 2867 if (!dev) { 2868 dev_err(&pdev->dev, "could not allocate netdev\n"); 2869 err = -ENOMEM; 2870 goto abort_with_db_bar; 2871 } 2872 SET_NETDEV_DEV(dev, &pdev->dev); 2873 pci_set_drvdata(pdev, dev); 2874 dev->ethtool_ops = &gve_ethtool_ops; 2875 dev->netdev_ops = &gve_netdev_ops; 2876 dev->queue_mgmt_ops = &gve_queue_mgmt_ops; 2877 dev->stat_ops = &gve_stat_ops; 2878 2879 /* Set default and supported features. 2880 * 2881 * Features might be set in other locations as well (such as 2882 * `gve_adminq_describe_device`). 2883 */ 2884 dev->hw_features = NETIF_F_HIGHDMA; 2885 dev->hw_features |= NETIF_F_SG; 2886 dev->hw_features |= NETIF_F_HW_CSUM; 2887 dev->hw_features |= NETIF_F_TSO; 2888 dev->hw_features |= NETIF_F_TSO6; 2889 dev->hw_features |= NETIF_F_TSO_ECN; 2890 dev->hw_features |= NETIF_F_RXCSUM; 2891 dev->hw_features |= NETIF_F_RXHASH; 2892 dev->features = dev->hw_features; 2893 dev->watchdog_timeo = 5 * HZ; 2894 dev->min_mtu = ETH_MIN_MTU; 2895 netif_carrier_off(dev); 2896 2897 priv = netdev_priv(dev); 2898 priv->dev = dev; 2899 priv->pdev = pdev; 2900 priv->msg_enable = DEFAULT_MSG_LEVEL; 2901 priv->reg_bar0 = reg_bar; 2902 priv->db_bar2 = db_bar; 2903 priv->service_task_flags = 0x0; 2904 priv->state_flags = 0x0; 2905 priv->ethtool_flags = 0x0; 2906 priv->rx_cfg.packet_buffer_size = GVE_DEFAULT_RX_BUFFER_SIZE; 2907 priv->max_rx_buffer_size = GVE_DEFAULT_RX_BUFFER_SIZE; 2908 2909 gve_set_probe_in_progress(priv); 2910 priv->gve_wq = alloc_ordered_workqueue("gve", 0); 2911 if (!priv->gve_wq) { 2912 dev_err(&pdev->dev, "Could not allocate workqueue"); 2913 err = -ENOMEM; 2914 goto abort_with_netdev; 2915 } 2916 INIT_WORK(&priv->service_task, gve_service_task); 2917 INIT_WORK(&priv->stats_report_task, gve_stats_report_task); 2918 priv->tx_cfg.max_queues = max_tx_queues; 2919 priv->rx_cfg.max_queues = max_rx_queues; 2920 2921 err = gve_init_priv(priv, false); 2922 if (err) 2923 goto abort_with_wq; 2924 2925 if (!gve_is_gqi(priv) && !gve_is_qpl(priv)) 2926 dev->netmem_tx = NETMEM_TX_DMA; 2927 2928 err = register_netdev(dev); 2929 if (err) 2930 goto abort_with_gve_init; 2931 2932 dev_info(&pdev->dev, "GVE version %s\n", gve_version_str); 2933 dev_info(&pdev->dev, "GVE queue format %d\n", (int)priv->queue_format); 2934 gve_clear_probe_in_progress(priv); 2935 queue_work(priv->gve_wq, &priv->service_task); 2936 return 0; 2937 2938 abort_with_gve_init: 2939 gve_teardown_priv_resources(priv); 2940 2941 abort_with_wq: 2942 destroy_workqueue(priv->gve_wq); 2943 2944 abort_with_netdev: 2945 free_netdev(dev); 2946 2947 abort_with_db_bar: 2948 pci_iounmap(pdev, db_bar); 2949 2950 abort_with_reg_bar: 2951 pci_iounmap(pdev, reg_bar); 2952 2953 abort_with_pci_region: 2954 pci_release_regions(pdev); 2955 2956 abort_with_enabled: 2957 pci_disable_device(pdev); 2958 return err; 2959 } 2960 2961 static void gve_remove(struct pci_dev *pdev) 2962 { 2963 struct net_device *netdev = pci_get_drvdata(pdev); 2964 struct gve_priv *priv = netdev_priv(netdev); 2965 __be32 __iomem *db_bar = priv->db_bar2; 2966 void __iomem *reg_bar = priv->reg_bar0; 2967 2968 unregister_netdev(netdev); 2969 gve_teardown_priv_resources(priv); 2970 destroy_workqueue(priv->gve_wq); 2971 free_netdev(netdev); 2972 pci_iounmap(pdev, db_bar); 2973 pci_iounmap(pdev, reg_bar); 2974 pci_release_regions(pdev); 2975 pci_disable_device(pdev); 2976 } 2977 2978 static void gve_shutdown(struct pci_dev *pdev) 2979 { 2980 struct net_device *netdev = pci_get_drvdata(pdev); 2981 struct gve_priv *priv = netdev_priv(netdev); 2982 bool was_up = netif_running(priv->dev); 2983 2984 netif_device_detach(netdev); 2985 2986 rtnl_lock(); 2987 netdev_lock(netdev); 2988 if (was_up && gve_close(priv->dev)) { 2989 /* If the dev was up, attempt to close, if close fails, reset */ 2990 gve_reset_and_teardown(priv, was_up); 2991 } else { 2992 /* If the dev wasn't up or close worked, finish tearing down */ 2993 gve_teardown_priv_resources(priv); 2994 } 2995 netdev_unlock(netdev); 2996 rtnl_unlock(); 2997 } 2998 2999 static int gve_suspend(struct device *dev) 3000 { 3001 struct pci_dev *pdev = to_pci_dev(dev); 3002 struct net_device *netdev = pci_get_drvdata(pdev); 3003 struct gve_priv *priv = netdev_priv(netdev); 3004 bool was_up = netif_running(priv->dev); 3005 3006 priv->suspend_cnt++; 3007 rtnl_lock(); 3008 netdev_lock(netdev); 3009 if (was_up && gve_close(priv->dev)) { 3010 /* If the dev was up, attempt to close, if close fails, reset */ 3011 gve_reset_and_teardown(priv, was_up); 3012 } else { 3013 /* If the dev wasn't up or close worked, finish tearing down */ 3014 gve_teardown_priv_resources(priv); 3015 } 3016 priv->up_before_suspend = was_up; 3017 netdev_unlock(netdev); 3018 rtnl_unlock(); 3019 return 0; 3020 } 3021 3022 static int gve_resume(struct device *dev) 3023 { 3024 struct pci_dev *pdev = to_pci_dev(dev); 3025 struct net_device *netdev = pci_get_drvdata(pdev); 3026 struct gve_priv *priv = netdev_priv(netdev); 3027 int err; 3028 3029 priv->resume_cnt++; 3030 rtnl_lock(); 3031 netdev_lock(netdev); 3032 err = gve_reset_recovery(priv, priv->up_before_suspend); 3033 netdev_unlock(netdev); 3034 rtnl_unlock(); 3035 return err; 3036 } 3037 3038 static DEFINE_SIMPLE_DEV_PM_OPS(gve_pm_ops, gve_suspend, gve_resume); 3039 3040 static const struct pci_device_id gve_id_table[] = { 3041 { PCI_DEVICE(PCI_VENDOR_ID_GOOGLE, PCI_DEV_ID_GVNIC) }, 3042 { } 3043 }; 3044 3045 static struct pci_driver gve_driver = { 3046 .name = gve_driver_name, 3047 .id_table = gve_id_table, 3048 .probe = gve_probe, 3049 .remove = gve_remove, 3050 .shutdown = gve_shutdown, 3051 .driver.pm = pm_sleep_ptr(&gve_pm_ops), 3052 }; 3053 3054 module_pci_driver(gve_driver); 3055 3056 MODULE_DEVICE_TABLE(pci, gve_id_table); 3057 MODULE_AUTHOR("Google, Inc."); 3058 MODULE_DESCRIPTION("Google Virtual NIC Driver"); 3059 MODULE_LICENSE("Dual MIT/GPL"); 3060 MODULE_VERSION(GVE_VERSION); 3061