1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 2023-2024 Google LLC 5 * 6 * Redistribution and use in source and binary forms, with or without modification, 7 * are permitted provided that the following conditions are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright notice, this 10 * list of conditions and the following disclaimer. 11 * 12 * 2. Redistributions in binary form must reproduce the above copyright notice, 13 * this list of conditions and the following disclaimer in the documentation 14 * and/or other materials provided with the distribution. 15 * 16 * 3. Neither the name of the copyright holder nor the names of its contributors 17 * may be used to endorse or promote products derived from this software without 18 * specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 22 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 24 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 25 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 27 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 29 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 #include "gve.h" 32 #include "gve_adminq.h" 33 #include "gve_dqo.h" 34 35 #define GVE_DRIVER_VERSION "GVE-FBSD-1.3.2\n" 36 #define GVE_VERSION_MAJOR 1 37 #define GVE_VERSION_MINOR 3 38 #define GVE_VERSION_SUB 2 39 40 #define GVE_DEFAULT_RX_COPYBREAK 256 41 42 /* Devices supported by this driver. */ 43 static struct gve_dev { 44 uint16_t vendor_id; 45 uint16_t device_id; 46 const char *name; 47 } gve_devs[] = { 48 { PCI_VENDOR_ID_GOOGLE, PCI_DEV_ID_GVNIC, "gVNIC" } 49 }; 50 51 struct sx gve_global_lock; 52 53 static int 54 gve_verify_driver_compatibility(struct gve_priv *priv) 55 { 56 int err; 57 struct gve_driver_info *driver_info; 58 struct gve_dma_handle driver_info_mem; 59 60 err = gve_dma_alloc_coherent(priv, sizeof(struct gve_driver_info), 61 PAGE_SIZE, &driver_info_mem); 62 63 if (err != 0) 64 return (ENOMEM); 65 66 driver_info = driver_info_mem.cpu_addr; 67 68 *driver_info = (struct gve_driver_info) { 69 .os_type = 3, /* Freebsd */ 70 .driver_major = GVE_VERSION_MAJOR, 71 .driver_minor = GVE_VERSION_MINOR, 72 .driver_sub = GVE_VERSION_SUB, 73 .os_version_major = htobe32(FBSD_VERSION_MAJOR), 74 .os_version_minor = htobe32(FBSD_VERSION_MINOR), 75 .os_version_sub = htobe32(FBSD_VERSION_PATCH), 76 .driver_capability_flags = { 77 htobe64(GVE_DRIVER_CAPABILITY_FLAGS1), 78 htobe64(GVE_DRIVER_CAPABILITY_FLAGS2), 79 htobe64(GVE_DRIVER_CAPABILITY_FLAGS3), 80 htobe64(GVE_DRIVER_CAPABILITY_FLAGS4), 81 }, 82 }; 83 84 snprintf(driver_info->os_version_str1, sizeof(driver_info->os_version_str1), 85 "FreeBSD %u", __FreeBSD_version); 86 87 bus_dmamap_sync(driver_info_mem.tag, driver_info_mem.map, 88 BUS_DMASYNC_PREREAD); 89 90 err = gve_adminq_verify_driver_compatibility(priv, 91 sizeof(struct gve_driver_info), driver_info_mem.bus_addr); 92 93 /* It's ok if the device doesn't support this */ 94 if (err == EOPNOTSUPP) 95 err = 0; 96 97 gve_dma_free_coherent(&driver_info_mem); 98 99 return (err); 100 } 101 102 static int 103 gve_up(struct gve_priv *priv) 104 { 105 if_t ifp = priv->ifp; 106 int err; 107 108 GVE_IFACE_LOCK_ASSERT(priv->gve_iface_lock); 109 110 if (device_is_attached(priv->dev) == 0) { 111 device_printf(priv->dev, "Cannot bring the iface up when detached\n"); 112 return (ENXIO); 113 } 114 115 if (gve_get_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP)) 116 return (0); 117 118 if_clearhwassist(ifp); 119 if (if_getcapenable(ifp) & IFCAP_TXCSUM) 120 if_sethwassistbits(ifp, CSUM_TCP | CSUM_UDP, 0); 121 if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6) 122 if_sethwassistbits(ifp, CSUM_IP6_TCP | CSUM_IP6_UDP, 0); 123 if (if_getcapenable(ifp) & IFCAP_TSO4) 124 if_sethwassistbits(ifp, CSUM_IP_TSO, 0); 125 if (if_getcapenable(ifp) & IFCAP_TSO6) 126 if_sethwassistbits(ifp, CSUM_IP6_TSO, 0); 127 128 if (gve_is_qpl(priv)) { 129 err = gve_register_qpls(priv); 130 if (err != 0) 131 goto reset; 132 } 133 134 err = gve_create_rx_rings(priv); 135 if (err != 0) 136 goto reset; 137 138 err = gve_create_tx_rings(priv); 139 if (err != 0) 140 goto reset; 141 142 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); 143 144 if (!gve_get_state_flag(priv, GVE_STATE_FLAG_LINK_UP)) { 145 if_link_state_change(ifp, LINK_STATE_UP); 146 gve_set_state_flag(priv, GVE_STATE_FLAG_LINK_UP); 147 } 148 149 gve_unmask_all_queue_irqs(priv); 150 gve_set_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP); 151 priv->interface_up_cnt++; 152 return (0); 153 154 reset: 155 gve_schedule_reset(priv); 156 return (err); 157 } 158 159 static void 160 gve_down(struct gve_priv *priv) 161 { 162 GVE_IFACE_LOCK_ASSERT(priv->gve_iface_lock); 163 164 if (!gve_get_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP)) 165 return; 166 167 if (gve_get_state_flag(priv, GVE_STATE_FLAG_LINK_UP)) { 168 if_link_state_change(priv->ifp, LINK_STATE_DOWN); 169 gve_clear_state_flag(priv, GVE_STATE_FLAG_LINK_UP); 170 } 171 172 if_setdrvflagbits(priv->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); 173 174 if (gve_destroy_rx_rings(priv) != 0) 175 goto reset; 176 177 if (gve_destroy_tx_rings(priv) != 0) 178 goto reset; 179 180 if (gve_is_qpl(priv)) { 181 if (gve_unregister_qpls(priv) != 0) 182 goto reset; 183 } 184 185 if (gve_is_gqi(priv)) 186 gve_mask_all_queue_irqs(priv); 187 gve_clear_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP); 188 priv->interface_down_cnt++; 189 return; 190 191 reset: 192 gve_schedule_reset(priv); 193 } 194 195 static int 196 gve_set_mtu(if_t ifp, uint32_t new_mtu) 197 { 198 struct gve_priv *priv = if_getsoftc(ifp); 199 const uint32_t max_problem_range = 8227; 200 const uint32_t min_problem_range = 7822; 201 int err; 202 203 if ((new_mtu > priv->max_mtu) || (new_mtu < ETHERMIN)) { 204 device_printf(priv->dev, "Invalid new MTU setting. new mtu: %d max mtu: %d min mtu: %d\n", 205 new_mtu, priv->max_mtu, ETHERMIN); 206 return (EINVAL); 207 } 208 209 /* 210 * When hardware LRO is enabled in DQ mode, MTUs within the range 211 * [7822, 8227] trigger hardware issues which cause a drastic drop 212 * in throughput. 213 */ 214 if (!gve_is_gqi(priv) && !gve_disable_hw_lro && 215 new_mtu >= min_problem_range && new_mtu <= max_problem_range) { 216 device_printf(priv->dev, 217 "Cannot set to MTU to %d within the range [%d, %d] while hardware LRO is enabled\n", 218 new_mtu, min_problem_range, max_problem_range); 219 return (EINVAL); 220 } 221 222 err = gve_adminq_set_mtu(priv, new_mtu); 223 if (err == 0) { 224 if (bootverbose) 225 device_printf(priv->dev, "MTU set to %d\n", new_mtu); 226 if_setmtu(ifp, new_mtu); 227 } else { 228 device_printf(priv->dev, "Failed to set MTU to %d\n", new_mtu); 229 } 230 231 return (err); 232 } 233 234 static void 235 gve_init(void *arg) 236 { 237 struct gve_priv *priv = (struct gve_priv *)arg; 238 239 if (!gve_get_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP)) { 240 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock); 241 gve_up(priv); 242 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock); 243 } 244 } 245 246 static int 247 gve_ioctl(if_t ifp, u_long command, caddr_t data) 248 { 249 struct gve_priv *priv; 250 struct ifreq *ifr; 251 int rc = 0; 252 253 priv = if_getsoftc(ifp); 254 ifr = (struct ifreq *)data; 255 256 switch (command) { 257 case SIOCSIFMTU: 258 if (if_getmtu(ifp) == ifr->ifr_mtu) 259 break; 260 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock); 261 gve_down(priv); 262 gve_set_mtu(ifp, ifr->ifr_mtu); 263 rc = gve_up(priv); 264 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock); 265 break; 266 267 case SIOCSIFFLAGS: 268 if ((if_getflags(ifp) & IFF_UP) != 0) { 269 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { 270 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock); 271 rc = gve_up(priv); 272 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock); 273 } 274 } else { 275 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) { 276 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock); 277 gve_down(priv); 278 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock); 279 } 280 } 281 break; 282 283 case SIOCSIFCAP: 284 if (ifr->ifr_reqcap == if_getcapenable(ifp)) 285 break; 286 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock); 287 gve_down(priv); 288 if_setcapenable(ifp, ifr->ifr_reqcap); 289 rc = gve_up(priv); 290 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock); 291 break; 292 293 case SIOCSIFMEDIA: 294 /* FALLTHROUGH */ 295 case SIOCGIFMEDIA: 296 rc = ifmedia_ioctl(ifp, ifr, &priv->media, command); 297 break; 298 299 default: 300 rc = ether_ioctl(ifp, command, data); 301 break; 302 } 303 304 return (rc); 305 } 306 307 static int 308 gve_media_change(if_t ifp) 309 { 310 struct gve_priv *priv = if_getsoftc(ifp); 311 312 device_printf(priv->dev, "Media change not supported\n"); 313 return (0); 314 } 315 316 static void 317 gve_media_status(if_t ifp, struct ifmediareq *ifmr) 318 { 319 struct gve_priv *priv = if_getsoftc(ifp); 320 321 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock); 322 323 ifmr->ifm_status = IFM_AVALID; 324 ifmr->ifm_active = IFM_ETHER; 325 326 if (gve_get_state_flag(priv, GVE_STATE_FLAG_LINK_UP)) { 327 ifmr->ifm_status |= IFM_ACTIVE; 328 ifmr->ifm_active |= IFM_AUTO; 329 } else { 330 ifmr->ifm_active |= IFM_NONE; 331 } 332 333 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock); 334 } 335 336 static uint64_t 337 gve_get_counter(if_t ifp, ift_counter cnt) 338 { 339 struct gve_priv *priv; 340 uint64_t rpackets = 0; 341 uint64_t tpackets = 0; 342 uint64_t rbytes = 0; 343 uint64_t tbytes = 0; 344 uint64_t rx_dropped_pkt = 0; 345 uint64_t tx_dropped_pkt = 0; 346 347 priv = if_getsoftc(ifp); 348 349 gve_accum_stats(priv, &rpackets, &rbytes, &rx_dropped_pkt, &tpackets, 350 &tbytes, &tx_dropped_pkt); 351 352 switch (cnt) { 353 case IFCOUNTER_IPACKETS: 354 return (rpackets); 355 356 case IFCOUNTER_OPACKETS: 357 return (tpackets); 358 359 case IFCOUNTER_IBYTES: 360 return (rbytes); 361 362 case IFCOUNTER_OBYTES: 363 return (tbytes); 364 365 case IFCOUNTER_IQDROPS: 366 return (rx_dropped_pkt); 367 368 case IFCOUNTER_OQDROPS: 369 return (tx_dropped_pkt); 370 371 default: 372 return (if_get_counter_default(ifp, cnt)); 373 } 374 } 375 376 static void 377 gve_setup_ifnet(device_t dev, struct gve_priv *priv) 378 { 379 int caps = 0; 380 if_t ifp; 381 382 ifp = priv->ifp = if_alloc(IFT_ETHER); 383 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 384 if_setsoftc(ifp, priv); 385 if_setdev(ifp, dev); 386 if_setinitfn(ifp, gve_init); 387 if_setioctlfn(ifp, gve_ioctl); 388 if_settransmitfn(ifp, gve_xmit_ifp); 389 if_setqflushfn(ifp, gve_qflush); 390 391 /* 392 * Set TSO limits, must match the arguments to bus_dma_tag_create 393 * when creating tx->dqo.buf_dmatag. Only applies to the RDA mode 394 * because in QPL we copy the entire packet into the bounce buffer 395 * and thus it does not matter how fragmented the mbuf is. 396 */ 397 if (!gve_is_gqi(priv) && !gve_is_qpl(priv)) { 398 if_sethwtsomaxsegcount(ifp, GVE_TX_MAX_DATA_DESCS_DQO); 399 if_sethwtsomaxsegsize(ifp, GVE_TX_MAX_BUF_SIZE_DQO); 400 } 401 if_sethwtsomax(ifp, GVE_TSO_MAXSIZE_DQO); 402 403 #if __FreeBSD_version >= 1400086 404 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); 405 #else 406 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST | IFF_KNOWSEPOCH); 407 #endif 408 409 ifmedia_init(&priv->media, IFM_IMASK, gve_media_change, gve_media_status); 410 if_setgetcounterfn(ifp, gve_get_counter); 411 412 caps = IFCAP_RXCSUM | 413 IFCAP_TXCSUM | 414 IFCAP_TXCSUM_IPV6 | 415 IFCAP_TSO | 416 IFCAP_LRO; 417 418 if ((priv->supported_features & GVE_SUP_JUMBO_FRAMES_MASK) != 0) 419 caps |= IFCAP_JUMBO_MTU; 420 421 if_setcapabilities(ifp, caps); 422 if_setcapenable(ifp, caps); 423 424 if (bootverbose) 425 device_printf(priv->dev, "Setting initial MTU to %d\n", priv->max_mtu); 426 if_setmtu(ifp, priv->max_mtu); 427 428 ether_ifattach(ifp, priv->mac); 429 430 ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO, 0, NULL); 431 ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO); 432 } 433 434 static int 435 gve_alloc_counter_array(struct gve_priv *priv) 436 { 437 int err; 438 439 err = gve_dma_alloc_coherent(priv, sizeof(uint32_t) * priv->num_event_counters, 440 PAGE_SIZE, &priv->counter_array_mem); 441 if (err != 0) 442 return (err); 443 444 priv->counters = priv->counter_array_mem.cpu_addr; 445 return (0); 446 } 447 448 static void 449 gve_free_counter_array(struct gve_priv *priv) 450 { 451 if (priv->counters != NULL) 452 gve_dma_free_coherent(&priv->counter_array_mem); 453 priv->counter_array_mem = (struct gve_dma_handle){}; 454 } 455 456 static int 457 gve_alloc_irq_db_array(struct gve_priv *priv) 458 { 459 int err; 460 461 err = gve_dma_alloc_coherent(priv, 462 sizeof(struct gve_irq_db) * (priv->num_queues), PAGE_SIZE, 463 &priv->irqs_db_mem); 464 if (err != 0) 465 return (err); 466 467 priv->irq_db_indices = priv->irqs_db_mem.cpu_addr; 468 return (0); 469 } 470 471 static void 472 gve_free_irq_db_array(struct gve_priv *priv) 473 { 474 if (priv->irq_db_indices != NULL) 475 gve_dma_free_coherent(&priv->irqs_db_mem); 476 priv->irqs_db_mem = (struct gve_dma_handle){}; 477 } 478 479 static void 480 gve_free_rings(struct gve_priv *priv) 481 { 482 gve_free_irqs(priv); 483 gve_free_tx_rings(priv); 484 gve_free_rx_rings(priv); 485 if (gve_is_qpl(priv)) 486 gve_free_qpls(priv); 487 } 488 489 static int 490 gve_alloc_rings(struct gve_priv *priv) 491 { 492 int err; 493 494 if (gve_is_qpl(priv)) { 495 err = gve_alloc_qpls(priv); 496 if (err != 0) 497 goto abort; 498 } 499 500 err = gve_alloc_rx_rings(priv); 501 if (err != 0) 502 goto abort; 503 504 err = gve_alloc_tx_rings(priv); 505 if (err != 0) 506 goto abort; 507 508 err = gve_alloc_irqs(priv); 509 if (err != 0) 510 goto abort; 511 512 return (0); 513 514 abort: 515 gve_free_rings(priv); 516 return (err); 517 } 518 519 static void 520 gve_deconfigure_and_free_device_resources(struct gve_priv *priv) 521 { 522 int err; 523 524 if (gve_get_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK)) { 525 err = gve_adminq_deconfigure_device_resources(priv); 526 if (err != 0) { 527 device_printf(priv->dev, "Failed to deconfigure device resources: err=%d\n", 528 err); 529 return; 530 } 531 if (bootverbose) 532 device_printf(priv->dev, "Deconfigured device resources\n"); 533 gve_clear_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK); 534 } 535 536 gve_free_irq_db_array(priv); 537 gve_free_counter_array(priv); 538 539 if (priv->ptype_lut_dqo) { 540 free(priv->ptype_lut_dqo, M_GVE); 541 priv->ptype_lut_dqo = NULL; 542 } 543 } 544 545 static int 546 gve_alloc_and_configure_device_resources(struct gve_priv *priv) 547 { 548 int err; 549 550 if (gve_get_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK)) 551 return (0); 552 553 err = gve_alloc_counter_array(priv); 554 if (err != 0) 555 return (err); 556 557 err = gve_alloc_irq_db_array(priv); 558 if (err != 0) 559 goto abort; 560 561 err = gve_adminq_configure_device_resources(priv); 562 if (err != 0) { 563 device_printf(priv->dev, "Failed to configure device resources: err=%d\n", 564 err); 565 err = (ENXIO); 566 goto abort; 567 } 568 569 if (!gve_is_gqi(priv)) { 570 priv->ptype_lut_dqo = malloc(sizeof(*priv->ptype_lut_dqo), M_GVE, 571 M_WAITOK | M_ZERO); 572 573 err = gve_adminq_get_ptype_map_dqo(priv, priv->ptype_lut_dqo); 574 if (err != 0) { 575 device_printf(priv->dev, "Failed to configure ptype lut: err=%d\n", 576 err); 577 goto abort; 578 } 579 } 580 581 gve_set_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK); 582 if (bootverbose) 583 device_printf(priv->dev, "Configured device resources\n"); 584 return (0); 585 586 abort: 587 gve_deconfigure_and_free_device_resources(priv); 588 return (err); 589 } 590 591 static void 592 gve_set_queue_cnts(struct gve_priv *priv) 593 { 594 priv->tx_cfg.max_queues = gve_reg_bar_read_4(priv, MAX_TX_QUEUES); 595 priv->rx_cfg.max_queues = gve_reg_bar_read_4(priv, MAX_RX_QUEUES); 596 priv->tx_cfg.num_queues = priv->tx_cfg.max_queues; 597 priv->rx_cfg.num_queues = priv->rx_cfg.max_queues; 598 599 if (priv->default_num_queues > 0) { 600 priv->tx_cfg.num_queues = MIN(priv->default_num_queues, 601 priv->tx_cfg.num_queues); 602 priv->rx_cfg.num_queues = MIN(priv->default_num_queues, 603 priv->rx_cfg.num_queues); 604 } 605 606 priv->num_queues = priv->tx_cfg.num_queues + priv->rx_cfg.num_queues; 607 priv->mgmt_msix_idx = priv->num_queues; 608 } 609 610 static int 611 gve_alloc_adminq_and_describe_device(struct gve_priv *priv) 612 { 613 int err; 614 615 if ((err = gve_adminq_alloc(priv)) != 0) 616 return (err); 617 618 if ((err = gve_verify_driver_compatibility(priv)) != 0) { 619 device_printf(priv->dev, 620 "Failed to verify driver compatibility: err=%d\n", err); 621 goto abort; 622 } 623 624 if ((err = gve_adminq_describe_device(priv)) != 0) 625 goto abort; 626 627 gve_set_queue_cnts(priv); 628 629 priv->num_registered_pages = 0; 630 return (0); 631 632 abort: 633 gve_release_adminq(priv); 634 return (err); 635 } 636 637 void 638 gve_schedule_reset(struct gve_priv *priv) 639 { 640 if (gve_get_state_flag(priv, GVE_STATE_FLAG_IN_RESET)) 641 return; 642 643 device_printf(priv->dev, "Scheduling reset task!\n"); 644 gve_set_state_flag(priv, GVE_STATE_FLAG_DO_RESET); 645 taskqueue_enqueue(priv->service_tq, &priv->service_task); 646 } 647 648 static void 649 gve_destroy(struct gve_priv *priv) 650 { 651 gve_down(priv); 652 gve_deconfigure_and_free_device_resources(priv); 653 gve_release_adminq(priv); 654 } 655 656 static void 657 gve_restore(struct gve_priv *priv) 658 { 659 int err; 660 661 err = gve_adminq_alloc(priv); 662 if (err != 0) 663 goto abort; 664 665 err = gve_adminq_configure_device_resources(priv); 666 if (err != 0) { 667 device_printf(priv->dev, "Failed to configure device resources: err=%d\n", 668 err); 669 err = (ENXIO); 670 goto abort; 671 } 672 if (!gve_is_gqi(priv)) { 673 err = gve_adminq_get_ptype_map_dqo(priv, priv->ptype_lut_dqo); 674 if (err != 0) { 675 device_printf(priv->dev, "Failed to configure ptype lut: err=%d\n", 676 err); 677 goto abort; 678 } 679 } 680 681 err = gve_up(priv); 682 if (err != 0) 683 goto abort; 684 685 return; 686 687 abort: 688 device_printf(priv->dev, "Restore failed!\n"); 689 return; 690 } 691 692 static void 693 gve_clear_device_resources(struct gve_priv *priv) 694 { 695 int i; 696 697 for (i = 0; i < priv->num_event_counters; i++) 698 priv->counters[i] = 0; 699 bus_dmamap_sync(priv->counter_array_mem.tag, priv->counter_array_mem.map, 700 BUS_DMASYNC_PREWRITE); 701 702 for (i = 0; i < priv->num_queues; i++) 703 priv->irq_db_indices[i] = (struct gve_irq_db){}; 704 bus_dmamap_sync(priv->irqs_db_mem.tag, priv->irqs_db_mem.map, 705 BUS_DMASYNC_PREWRITE); 706 707 if (priv->ptype_lut_dqo) 708 *priv->ptype_lut_dqo = (struct gve_ptype_lut){0}; 709 } 710 711 static void 712 gve_handle_reset(struct gve_priv *priv) 713 { 714 if (!gve_get_state_flag(priv, GVE_STATE_FLAG_DO_RESET)) 715 return; 716 717 gve_clear_state_flag(priv, GVE_STATE_FLAG_DO_RESET); 718 gve_set_state_flag(priv, GVE_STATE_FLAG_IN_RESET); 719 720 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock); 721 722 if_setdrvflagbits(priv->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); 723 if_link_state_change(priv->ifp, LINK_STATE_DOWN); 724 gve_clear_state_flag(priv, GVE_STATE_FLAG_LINK_UP); 725 726 /* 727 * Releasing the adminq causes the NIC to destroy all resources 728 * registered with it, so by clearing the flags beneath we cause 729 * the subsequent gve_down call below to not attempt to tell the 730 * NIC to destroy these resources again. 731 * 732 * The call to gve_down is needed in the first place to refresh 733 * the state and the DMA-able memory within each driver ring. 734 */ 735 gve_release_adminq(priv); 736 gve_clear_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK); 737 gve_clear_state_flag(priv, GVE_STATE_FLAG_QPLREG_OK); 738 gve_clear_state_flag(priv, GVE_STATE_FLAG_RX_RINGS_OK); 739 gve_clear_state_flag(priv, GVE_STATE_FLAG_TX_RINGS_OK); 740 741 gve_down(priv); 742 gve_clear_device_resources(priv); 743 744 gve_restore(priv); 745 746 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock); 747 748 priv->reset_cnt++; 749 gve_clear_state_flag(priv, GVE_STATE_FLAG_IN_RESET); 750 } 751 752 static void 753 gve_handle_link_status(struct gve_priv *priv) 754 { 755 uint32_t status = gve_reg_bar_read_4(priv, DEVICE_STATUS); 756 bool link_up = status & GVE_DEVICE_STATUS_LINK_STATUS; 757 758 if (link_up == gve_get_state_flag(priv, GVE_STATE_FLAG_LINK_UP)) 759 return; 760 761 if (link_up) { 762 if (bootverbose) 763 device_printf(priv->dev, "Device link is up.\n"); 764 if_link_state_change(priv->ifp, LINK_STATE_UP); 765 gve_set_state_flag(priv, GVE_STATE_FLAG_LINK_UP); 766 } else { 767 device_printf(priv->dev, "Device link is down.\n"); 768 if_link_state_change(priv->ifp, LINK_STATE_DOWN); 769 gve_clear_state_flag(priv, GVE_STATE_FLAG_LINK_UP); 770 } 771 } 772 773 static void 774 gve_service_task(void *arg, int pending) 775 { 776 struct gve_priv *priv = (struct gve_priv *)arg; 777 uint32_t status = gve_reg_bar_read_4(priv, DEVICE_STATUS); 778 779 if (((GVE_DEVICE_STATUS_RESET_MASK & status) != 0) && 780 !gve_get_state_flag(priv, GVE_STATE_FLAG_IN_RESET)) { 781 device_printf(priv->dev, "Device requested reset\n"); 782 gve_set_state_flag(priv, GVE_STATE_FLAG_DO_RESET); 783 } 784 785 gve_handle_reset(priv); 786 gve_handle_link_status(priv); 787 } 788 789 static int 790 gve_probe(device_t dev) 791 { 792 uint16_t deviceid, vendorid; 793 int i; 794 795 vendorid = pci_get_vendor(dev); 796 deviceid = pci_get_device(dev); 797 798 for (i = 0; i < nitems(gve_devs); i++) { 799 if (vendorid == gve_devs[i].vendor_id && 800 deviceid == gve_devs[i].device_id) { 801 device_set_desc(dev, gve_devs[i].name); 802 return (BUS_PROBE_DEFAULT); 803 } 804 } 805 return (ENXIO); 806 } 807 808 static void 809 gve_free_sys_res_mem(struct gve_priv *priv) 810 { 811 if (priv->msix_table != NULL) 812 bus_release_resource(priv->dev, SYS_RES_MEMORY, 813 rman_get_rid(priv->msix_table), priv->msix_table); 814 815 if (priv->db_bar != NULL) 816 bus_release_resource(priv->dev, SYS_RES_MEMORY, 817 rman_get_rid(priv->db_bar), priv->db_bar); 818 819 if (priv->reg_bar != NULL) 820 bus_release_resource(priv->dev, SYS_RES_MEMORY, 821 rman_get_rid(priv->reg_bar), priv->reg_bar); 822 } 823 824 static int 825 gve_attach(device_t dev) 826 { 827 struct gve_priv *priv; 828 int rid; 829 int err; 830 831 snprintf(gve_version, sizeof(gve_version), "%d.%d.%d", 832 GVE_VERSION_MAJOR, GVE_VERSION_MINOR, GVE_VERSION_SUB); 833 834 priv = device_get_softc(dev); 835 priv->dev = dev; 836 GVE_IFACE_LOCK_INIT(priv->gve_iface_lock); 837 838 pci_enable_busmaster(dev); 839 840 rid = PCIR_BAR(GVE_REGISTER_BAR); 841 priv->reg_bar = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 842 &rid, RF_ACTIVE); 843 if (priv->reg_bar == NULL) { 844 device_printf(dev, "Failed to allocate BAR0\n"); 845 err = ENXIO; 846 goto abort; 847 } 848 849 rid = PCIR_BAR(GVE_DOORBELL_BAR); 850 priv->db_bar = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 851 &rid, RF_ACTIVE); 852 if (priv->db_bar == NULL) { 853 device_printf(dev, "Failed to allocate BAR2\n"); 854 err = ENXIO; 855 goto abort; 856 } 857 858 rid = pci_msix_table_bar(priv->dev); 859 priv->msix_table = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 860 &rid, RF_ACTIVE); 861 if (priv->msix_table == NULL) { 862 device_printf(dev, "Failed to allocate msix table\n"); 863 err = ENXIO; 864 goto abort; 865 } 866 867 err = gve_alloc_adminq_and_describe_device(priv); 868 if (err != 0) 869 goto abort; 870 871 err = gve_alloc_and_configure_device_resources(priv); 872 if (err != 0) 873 goto abort; 874 875 err = gve_alloc_rings(priv); 876 if (err != 0) 877 goto abort; 878 879 gve_setup_ifnet(dev, priv); 880 881 priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK; 882 883 bus_write_multi_1(priv->reg_bar, DRIVER_VERSION, GVE_DRIVER_VERSION, 884 sizeof(GVE_DRIVER_VERSION) - 1); 885 886 TASK_INIT(&priv->service_task, 0, gve_service_task, priv); 887 priv->service_tq = taskqueue_create("gve service", M_WAITOK | M_ZERO, 888 taskqueue_thread_enqueue, &priv->service_tq); 889 taskqueue_start_threads(&priv->service_tq, 1, PI_NET, "%s service tq", 890 device_get_nameunit(priv->dev)); 891 892 gve_setup_sysctl(priv); 893 894 if (bootverbose) 895 device_printf(priv->dev, "Successfully attached %s", GVE_DRIVER_VERSION); 896 return (0); 897 898 abort: 899 gve_free_rings(priv); 900 gve_deconfigure_and_free_device_resources(priv); 901 gve_release_adminq(priv); 902 gve_free_sys_res_mem(priv); 903 GVE_IFACE_LOCK_DESTROY(priv->gve_iface_lock); 904 return (err); 905 } 906 907 static int 908 gve_detach(device_t dev) 909 { 910 struct gve_priv *priv = device_get_softc(dev); 911 if_t ifp = priv->ifp; 912 int error; 913 914 error = bus_generic_detach(dev); 915 if (error != 0) 916 return (error); 917 918 ether_ifdetach(ifp); 919 920 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock); 921 gve_destroy(priv); 922 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock); 923 924 gve_free_rings(priv); 925 gve_free_sys_res_mem(priv); 926 GVE_IFACE_LOCK_DESTROY(priv->gve_iface_lock); 927 928 while (taskqueue_cancel(priv->service_tq, &priv->service_task, NULL)) 929 taskqueue_drain(priv->service_tq, &priv->service_task); 930 taskqueue_free(priv->service_tq); 931 932 if_free(ifp); 933 return (0); 934 } 935 936 static device_method_t gve_methods[] = { 937 DEVMETHOD(device_probe, gve_probe), 938 DEVMETHOD(device_attach, gve_attach), 939 DEVMETHOD(device_detach, gve_detach), 940 DEVMETHOD_END 941 }; 942 943 static driver_t gve_driver = { 944 "gve", 945 gve_methods, 946 sizeof(struct gve_priv) 947 }; 948 949 #if __FreeBSD_version < 1301503 950 static devclass_t gve_devclass; 951 952 DRIVER_MODULE(gve, pci, gve_driver, gve_devclass, 0, 0); 953 #else 954 DRIVER_MODULE(gve, pci, gve_driver, 0, 0); 955 #endif 956 MODULE_PNP_INFO("U16:vendor;U16:device;D:#", pci, gve, gve_devs, 957 nitems(gve_devs)); 958