1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2001-2024, Intel Corporation 5 * Copyright (c) 2026 Kevin Bowling <kbowling@FreeBSD.org> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include "if_em.h" 30 31 #include <sys/sbuf.h> 32 33 #define IGBV_82576_QUEUES 2 34 #define IGBV_I350_QUEUES 1 35 #define IGBV_MAX_MAC_FILTERS 3 36 #define IGBV_QUEUE_DISABLE_BUSY_RETRIES 10 37 #define IGBV_QUEUE_DISABLE_DELAY_US 10 38 #define IGBV_QUEUE_DISABLE_PAUSE (100 * SBT_1US) 39 #define IGBV_QUEUE_DISABLE_RETRIES 20 40 #define IGBV_QUEUE_SANITIZE_ATTEMPTS 3 41 #define IGBV_VLAN_RETRY_BATCH 4 42 #define IGBV_VLAN_RETRY_WINDOW (8 * SBT_1S) 43 44 /* Reading these six configuration bytes performs the Hyper-V VF handshake. */ 45 #define IGBV_HV_RESET_OFFSET 0x201 46 47 static const struct timeval igbv_queue_log_interval = { 2, 0 }; 48 static const struct timeval igbv_mbx_log_interval = { 60, 0 }; 49 static const sbintime_t igbv_queue_retry_delay[] = { 50 100 * SBT_1MS, 51 500 * SBT_1MS, 52 }; 53 static const sbintime_t igbv_mbx_retry_delay[] = { 54 250 * SBT_1MS, 55 1 * SBT_1S, 56 4 * SBT_1S, 57 8 * SBT_1S, 58 }; 59 _Static_assert(nitems(igbv_queue_retry_delay) + 1 == 60 IGBV_QUEUE_SANITIZE_ATTEMPTS, "missing queue retry delay"); 61 62 struct igb_vf_uc_addr_list { 63 struct e1000_softc *sc; 64 u8 addrs[IGBV_MAX_MAC_FILTERS][ETHER_ADDR_LEN]; 65 }; 66 67 static bool igbv_tx_pending(struct e1000_softc *); 68 static bool igbv_vlan_retry_pending(const struct e1000_softc *); 69 static void igbv_vlan_retry_tick(struct e1000_softc *); 70 71 static s32 72 igbv_hv_read_mac_addr(struct e1000_hw *hw) 73 { 74 75 if (!em_is_valid_ether_addr(hw->mac.perm_addr)) 76 return (-E1000_ERR_MAC_INIT); 77 memcpy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN); 78 return (E1000_SUCCESS); 79 } 80 81 static s32 82 igbv_hv_reset_hw(struct e1000_hw *hw) 83 { 84 struct e1000_osdep *osdep; 85 u8 addr[ETHER_ADDR_LEN]; 86 u32 ctrl; 87 int i; 88 89 osdep = hw->back; 90 hw->mac.get_link_status = true; 91 memset(hw->mac.perm_addr, 0, ETHER_ADDR_LEN); 92 /* Hyper-V does not service the native VF posted-message protocol. */ 93 hw->mbx.timeout = 0; 94 ctrl = E1000_READ_REG(hw, E1000_CTRL); 95 if (ctrl == 0xffffffff) 96 return (-E1000_ERR_RESET); 97 /* Reset VF-local state before the Hyper-V host reset/MAC exchange. */ 98 E1000_WRITE_REG(hw, E1000_CTRL, ctrl | E1000_CTRL_RST); 99 E1000_WRITE_FLUSH(hw); 100 for (i = 0; i < E1000_VF_INIT_TIMEOUT; i++) { 101 if (hw->mbx.ops.check_for_rst(hw, 0) != E1000_SUCCESS) 102 break; 103 DELAY(5); 104 } 105 106 /* 107 * The PF handles 0x201..0x206 as a reset/MAC exchange, including 108 * partial reads. Do not use these bytes for routine status polling. 109 * The host-assigned address also identifies the matching hn interface. 110 */ 111 for (i = 0; i < ETHER_ADDR_LEN; i++) 112 addr[i] = pci_read_config(osdep->dev, IGBV_HV_RESET_OFFSET + i, 1); 113 if (!em_is_valid_ether_addr(addr)) 114 return (-E1000_ERR_MAC_INIT); 115 memcpy(hw->mac.perm_addr, addr, ETHER_ADDR_LEN); 116 return (E1000_SUCCESS); 117 } 118 119 static s32 120 igbv_hv_check_for_link(struct e1000_hw *hw) 121 { 122 u32 status; 123 124 /* Leave reset indications for the statistics counter-epoch check. */ 125 status = E1000_READ_REG(hw, E1000_STATUS); 126 hw->mac.get_link_status = true; 127 if (status == 0xffffffff) 128 return (-E1000_ERR_MAC_INIT); 129 /* Poll even after link-up so a host port link-down cannot stay cached. */ 130 hw->mac.get_link_status = (status & E1000_STATUS_LU) == 0; 131 return (E1000_SUCCESS); 132 } 133 134 static int 135 igbv_hv_rar_set(struct e1000_hw *hw, u8 *addr, u32 index) 136 { 137 int error; 138 139 error = index == 0 && 140 memcmp(addr, hw->mac.perm_addr, ETHER_ADDR_LEN) == 0 ? 141 E1000_SUCCESS : -E1000_ERR_MAC_INIT; 142 if (igbv_hv_read_mac_addr(hw) != E1000_SUCCESS) 143 return (-E1000_ERR_MAC_INIT); 144 return (error); 145 } 146 147 static void 148 igbv_hv_update_mc_addr_list(struct e1000_hw *hw __unused, 149 u8 *addrs __unused, u32 count __unused) 150 { 151 152 /* The host owns receive-filter policy for the synthetic/VF pair. */ 153 } 154 155 void 156 igbv_init_hv_ops(struct e1000_hw *hw) 157 { 158 159 /* Install after the ordinary VF MAC and mailbox parameter setup. */ 160 hw->mac.ops.reset_hw = igbv_hv_reset_hw; 161 /* Native init_hw calls rar_set_vf directly, bypassing the RAR op. */ 162 hw->mac.ops.init_hw = igbv_hv_read_mac_addr; 163 hw->mac.ops.read_mac_addr = igbv_hv_read_mac_addr; 164 hw->mac.ops.rar_set = igbv_hv_rar_set; 165 hw->mac.ops.check_for_link = igbv_hv_check_for_link; 166 hw->mac.ops.update_mc_addr_list = igbv_hv_update_mc_addr_list; 167 } 168 169 static void 170 igbv_queue_retry_callout(void *arg) 171 { 172 struct e1000_softc *sc; 173 if_t ifp; 174 175 sc = arg; 176 if (atomic_readandclear_32(&sc->vf_queue_retry_pending) == 0) 177 return; 178 ifp = iflib_get_ifp(sc->ctx); 179 if ((if_getflags(ifp) & IFF_UP) == 0) { 180 atomic_set_32(&sc->vf_queue_retry_new_epoch, 1); 181 return; 182 } 183 iflib_request_reset_if_up(sc->ctx); 184 iflib_admin_intr_deferred(sc->ctx); 185 } 186 187 /* 188 * A missing PF can make the posted reset handshake wait for a full mailbox 189 * timeout. Keep that work out of stopped status paths. An administratively 190 * up VF retries complete initialization with an exponential delay capped at 191 * eight seconds, so it recovers without creating a tight mailbox poller. 192 */ 193 static void 194 igbv_mbx_retry_callout(void *arg) 195 { 196 struct e1000_softc *sc; 197 if_t ifp; 198 199 sc = arg; 200 if (atomic_readandclear_32(&sc->vf_mbx_retry_pending) == 0 || 201 atomic_load_acq_32(&sc->vf_mbx_ready) != 0 || 202 iflib_in_detach(sc->ctx)) 203 return; 204 ifp = iflib_get_ifp(sc->ctx); 205 if ((if_getflags(ifp) & IFF_UP) == 0) 206 return; 207 208 iflib_request_reset_if_up(sc->ctx); 209 iflib_admin_intr_deferred(sc->ctx); 210 } 211 212 static const char * 213 igbv_reset_error_desc(s32 error) 214 { 215 216 switch (error) { 217 case -E1000_ERR_RESET: 218 return ("PF reset acknowledgement timed out"); 219 case -E1000_ERR_MAC_INIT: 220 return ("PF returned an invalid VF reset response"); 221 case -E1000_ERR_MBX: 222 return ("PF mailbox reset exchange failed"); 223 default: 224 return ("VF reset handshake failed"); 225 } 226 } 227 228 void 229 igbv_log_reset_failure(struct e1000_softc *sc, s32 error, bool attaching) 230 { 231 232 /* Report each backoff stage, then limit the steady eight-second retry. */ 233 if (sc->vf_mbx_retry_stage == nitems(igbv_mbx_retry_delay) - 1 && 234 !ratecheck(&sc->vf_last_mbx_log, &igbv_mbx_log_interval)) 235 return; 236 device_printf(sc->dev, "%s (%d)%s\n", igbv_reset_error_desc(error), 237 error, attaching ? "; continuing attach" : ""); 238 } 239 240 void 241 igbv_mbx_retry_detach(struct e1000_softc *sc) 242 { 243 244 if (!sc->vf_mbx_retry_initialized) 245 return; 246 atomic_readandclear_32(&sc->vf_mbx_retry_pending); 247 callout_drain(&sc->vf_mbx_retry); 248 sc->vf_mbx_retry_initialized = false; 249 } 250 251 void 252 igbv_mbx_retry_prepare(struct e1000_softc *sc) 253 { 254 255 if (!sc->vf_mbx_retry_initialized) 256 return; 257 atomic_readandclear_32(&sc->vf_mbx_retry_pending); 258 callout_drain(&sc->vf_mbx_retry); 259 } 260 261 void 262 igbv_mbx_retry_stop(struct e1000_softc *sc) 263 { 264 if_t ifp; 265 266 if (!sc->vf_mbx_retry_initialized) 267 return; 268 atomic_readandclear_32(&sc->vf_mbx_retry_pending); 269 callout_drain(&sc->vf_mbx_retry); 270 ifp = iflib_get_ifp(sc->ctx); 271 if ((if_getflags(ifp) & IFF_UP) == 0) 272 sc->vf_mbx_retry_stage = 0; 273 } 274 275 void 276 igbv_mbx_retry_failed(if_ctx_t ctx) 277 { 278 struct e1000_softc *sc; 279 if_t ifp; 280 sbintime_t delay; 281 u_int stage; 282 283 sc = iflib_get_softc(ctx); 284 atomic_store_rel_32(&sc->vf_mbx_ready, 0); 285 sc->link_speed = 0; 286 sc->link_duplex = 0; 287 if (sc->link_state != EM_LINK_STATE_DOWN) { 288 sc->link_state = EM_LINK_STATE_DOWN; 289 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 290 } 291 iflib_init_failed(ctx); 292 293 ifp = iflib_get_ifp(ctx); 294 if (!sc->vf_mbx_retry_initialized || 295 (if_getflags(ifp) & IFF_UP) == 0) 296 return; 297 stage = sc->vf_mbx_retry_stage; 298 if (stage >= nitems(igbv_mbx_retry_delay)) 299 stage = nitems(igbv_mbx_retry_delay) - 1; 300 delay = igbv_mbx_retry_delay[stage]; 301 if (sc->vf_mbx_retry_stage + 1 < nitems(igbv_mbx_retry_delay)) 302 sc->vf_mbx_retry_stage++; 303 atomic_set_32(&sc->vf_mbx_retry_pending, 1); 304 callout_reset_sbt(&sc->vf_mbx_retry, delay, 0, 305 igbv_mbx_retry_callout, sc, C_PREL(1)); 306 } 307 308 static void 309 igbv_mbx_retry_succeeded(struct e1000_softc *sc) 310 { 311 312 atomic_store_rel_32(&sc->vf_mbx_ready, 1); 313 atomic_readandclear_32(&sc->vf_mbx_retry_pending); 314 if (sc->vf_mbx_retry_initialized) 315 callout_stop(&sc->vf_mbx_retry); 316 sc->vf_mbx_retry_stage = 0; 317 sc->vf_last_mbx_log.tv_sec = 0; 318 sc->vf_last_mbx_log.tv_usec = 0; 319 } 320 321 void 322 igbv_queue_retry_detach(struct e1000_softc *sc) 323 { 324 325 if (!sc->vf_queue_retry_initialized) 326 return; 327 atomic_readandclear_32(&sc->vf_queue_retry_pending); 328 callout_drain(&sc->vf_queue_retry); 329 sc->vf_queue_retry_initialized = false; 330 } 331 332 void 333 igbv_queue_retry_stop(struct e1000_softc *sc) 334 { 335 336 if (!sc->vf_queue_retry_initialized) 337 return; 338 if (atomic_readandclear_32(&sc->vf_queue_retry_pending) != 0) 339 atomic_set_32(&sc->vf_queue_retry_new_epoch, 1); 340 callout_stop(&sc->vf_queue_retry); 341 } 342 343 void 344 igbv_queue_retry_prepare(struct e1000_softc *sc) 345 { 346 bool new_epoch; 347 348 new_epoch = 349 atomic_readandclear_32(&sc->vf_queue_retry_new_epoch) != 0; 350 if (!sc->vf_queue_gave_up && !new_epoch) 351 return; 352 sc->vf_queue_failures = 0; 353 sc->vf_queue_gave_up = false; 354 } 355 356 static void 357 igbv_queue_retry_succeeded(struct e1000_softc *sc) 358 { 359 360 atomic_readandclear_32(&sc->vf_queue_retry_pending); 361 atomic_readandclear_32(&sc->vf_queue_retry_new_epoch); 362 if (sc->vf_queue_retry_initialized) 363 callout_stop(&sc->vf_queue_retry); 364 sc->vf_queue_failures = 0; 365 sc->vf_queue_gave_up = false; 366 } 367 368 void 369 igbv_queue_retry_failed(if_ctx_t ctx) 370 { 371 struct e1000_softc *sc; 372 sbintime_t delay; 373 374 sc = iflib_get_softc(ctx); 375 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); 376 377 if (sc->vf_queue_failures < IGBV_QUEUE_SANITIZE_ATTEMPTS) 378 sc->vf_queue_failures++; 379 if (sc->vf_queue_failures < IGBV_QUEUE_SANITIZE_ATTEMPTS) { 380 delay = igbv_queue_retry_delay[sc->vf_queue_failures - 1]; 381 atomic_set_32(&sc->vf_queue_retry_pending, 1); 382 callout_reset_sbt(&sc->vf_queue_retry, delay, 0, 383 igbv_queue_retry_callout, sc, C_PREL(1)); 384 } else if (!sc->vf_queue_gave_up) { 385 atomic_readandclear_32(&sc->vf_queue_retry_pending); 386 callout_stop(&sc->vf_queue_retry); 387 sc->vf_queue_gave_up = true; 388 device_printf(sc->dev, 389 "retained VF queues remained active after %u attempts; " 390 "interface left down; toggle it down/up to retry\n", 391 sc->vf_queue_failures); 392 } 393 394 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 395 iflib_init_failed(ctx); 396 } 397 398 void 399 igbv_vlan_retry_add(struct e1000_softc *sc, u16 vid) 400 { 401 bool pending; 402 403 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); 404 pending = igbv_vlan_retry_pending(sc); 405 sc->vf_vfta_retry[vid >> 5] |= 1U << (vid & 0x1f); 406 /* Bound the whole batch from its first failure, not each new VID. */ 407 if (!pending) 408 sc->vf_vlan_retry_deadline = 409 getsbinuptime() + IGBV_VLAN_RETRY_WINDOW; 410 } 411 412 void 413 igbv_vlan_retry_clear(struct e1000_softc *sc, u16 vid) 414 { 415 416 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); 417 sc->vf_vfta_retry[vid >> 5] &= ~(1U << (vid & 0x1f)); 418 } 419 420 static bool 421 igbv_vlan_retry_pending(const struct e1000_softc *sc) 422 { 423 int i; 424 425 for (i = 0; i < EM_VFTA_SIZE; i++) 426 if (sc->vf_vfta_retry[i] != 0) 427 return (true); 428 return (false); 429 } 430 431 static void 432 igbv_vlan_retry_tick(struct e1000_softc *sc) 433 { 434 u32 bit; 435 u16 vid; 436 int attempts, i, remaining; 437 438 if (!igbv_vlan_retry_pending(sc)) { 439 sc->vf_vlan_retry_deadline = 0; 440 return; 441 } 442 if (getsbinuptime() >= sc->vf_vlan_retry_deadline) { 443 remaining = 0; 444 for (i = 0; i < EM_VFTA_SIZE; i++) 445 remaining += bitcount32(sc->vf_vfta_retry[i]); 446 memset(sc->vf_vfta_retry, 0, sizeof(sc->vf_vfta_retry)); 447 sc->vf_vlan_retry_deadline = 0; 448 device_printf(sc->dev, 449 "VF VLAN restore retries exhausted for %d VIDs\n", 450 remaining); 451 return; 452 } 453 454 /* 455 * The mailbox NACK does not distinguish a transient PF rate limit 456 * from permanent VLVF exhaustion. Retry at the PF's sustained 457 * allowance, but bound the entire recovery window so ENOSPC cannot 458 * create a permanent mailbox poller. 459 */ 460 for (attempts = 0, i = 0; 461 attempts < IGBV_VLAN_RETRY_BATCH && i < 4096; i++) { 462 vid = sc->vf_vlan_retry_cursor; 463 sc->vf_vlan_retry_cursor = (vid + 1) & 0xfff; 464 bit = 1U << (vid & 0x1f); 465 if ((sc->vf_vfta_retry[vid >> 5] & bit) == 0) 466 continue; 467 attempts++; 468 if ((sc->shadow_vfta[vid >> 5] & bit) == 0 || 469 e1000_vfta_set_vf(&sc->hw, vid, true) == 470 E1000_SUCCESS) 471 sc->vf_vfta_retry[vid >> 5] &= ~bit; 472 } 473 if (!igbv_vlan_retry_pending(sc)) 474 sc->vf_vlan_retry_deadline = 0; 475 } 476 477 int 478 igbv_if_attach_pre(if_ctx_t ctx) 479 { 480 struct e1000_softc *sc; 481 device_t dev; 482 int error; 483 484 dev = iflib_get_dev(ctx); 485 if (pci_msix_count(dev) < 2) { 486 device_printf(dev, "VF operation requires two MSI-X vectors\n"); 487 return (ENXIO); 488 } 489 error = em_if_attach_pre(ctx); 490 if (error != 0) 491 return (error); 492 493 sc = iflib_get_softc(ctx); 494 callout_init(&sc->vf_queue_retry, 1); 495 sc->vf_queue_retry_initialized = true; 496 callout_init(&sc->vf_mbx_retry, 1); 497 sc->vf_mbx_retry_initialized = true; 498 499 KASSERT(sc->vf_ifp && 500 (iflib_get_sctx(ctx)->isc_flags & IFLIB_IS_VF) != 0, 501 ("%s: igbv attached without VF policy", __func__)); 502 return (0); 503 } 504 505 int 506 igbv_if_attach_post(if_ctx_t ctx) 507 { 508 struct e1000_softc *sc; 509 int error; 510 511 sc = iflib_get_softc(ctx); 512 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); 513 if (sc->intr_type != IFLIB_INTR_MSIX) { 514 device_printf(sc->dev, "VF operation requires MSI-X\n"); 515 return (ENXIO); 516 } 517 error = em_if_attach_post(ctx); 518 if (error != 0) 519 return (error); 520 521 /* 522 * Attach failures can leave the device sysctl tree registered when 523 * hw.bus.disable_failed_devices is set. Do not publish handlers with 524 * softc arguments until iflib has successfully allocated MSI-X. 525 */ 526 em_add_device_sysctls(sc); 527 return (0); 528 } 529 530 int 531 igbv_if_media_change(if_ctx_t ctx __unused) 532 { 533 534 return (EOPNOTSUPP); 535 } 536 537 void 538 igbv_if_update_admin_status(if_ctx_t ctx) 539 { 540 struct e1000_softc *sc; 541 struct e1000_hw *hw; 542 device_t dev; 543 bool link_check, timer_tick; 544 545 sc = iflib_get_softc(ctx); 546 hw = &sc->hw; 547 dev = iflib_get_dev(ctx); 548 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); 549 550 if (!iflib_is_running(ctx) || 551 !sc->vf_queues_sanitized || 552 atomic_load_acq_32(&sc->vf_mbx_ready) == 0) { 553 if (sc->link_state != EM_LINK_STATE_DOWN) { 554 sc->link_speed = 0; 555 sc->link_duplex = 0; 556 sc->link_state = EM_LINK_STATE_DOWN; 557 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 558 } 559 return; 560 } 561 562 if (igbv_is_hyperv(sc)) { 563 u32 status, txdctl; 564 565 if (sc->vf_reset_pending) 566 return; 567 txdctl = E1000_READ_REG(hw, E1000_TXDCTL(0)); 568 if (txdctl != 0xffffffff && 569 (txdctl & E1000_TXDCTL_QUEUE_ENABLE) == 0) { 570 /* Carrier can remain up after the PF disables queue DMA. */ 571 sc->vf_stats_valid = false; 572 sc->link_speed = sc->link_duplex = 0; 573 if (sc->link_state != EM_LINK_STATE_DOWN) { 574 sc->link_state = EM_LINK_STATE_DOWN; 575 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 576 } 577 /* 578 * Do not reset while carrier is down. Limit requests 579 * if the host keeps the queue disabled with carrier up. 580 */ 581 status = E1000_READ_REG(hw, E1000_STATUS); 582 if (status != 0xffffffff && 583 (status & E1000_STATUS_LU) != 0 && 584 ratecheck(&sc->vf_last_queue_log, 585 &igbv_queue_log_interval)) { 586 device_printf(dev, 587 "Hyper-V VF queue disabled; requesting recovery\n"); 588 sc->vf_reset_pending = true; 589 iflib_request_reset_if_up(ctx); 590 iflib_admin_intr_deferred(ctx); 591 } 592 return; 593 } 594 } 595 596 if (!sc->vf_reset_pending && 597 atomic_readandclear_32(&sc->promisc_pending) != 0) 598 (void)em_if_set_promisc_impl(ctx, 599 if_getflags(iflib_get_ifp(ctx))); 600 601 if (e1000_check_for_link(hw) != E1000_SUCCESS && 602 !sc->vf_reset_pending) { 603 sc->vf_reset_pending = true; 604 iflib_request_reset(ctx); 605 iflib_admin_intr_deferred(ctx); 606 } 607 link_check = !hw->mac.get_link_status; 608 609 if (link_check && 610 (sc->link_state == EM_LINK_STATE_DOWN || 611 sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING)) { 612 e1000_get_speed_and_duplex(hw, &sc->link_speed, 613 &sc->link_duplex); 614 if (bootverbose) 615 device_printf(dev, "Link is up %d Mbps %s\n", 616 sc->link_speed, 617 sc->link_duplex == FULL_DUPLEX ? 618 "Full Duplex" : "Half Duplex"); 619 sc->link_state = EM_LINK_STATE_UP; 620 iflib_link_state_change(ctx, LINK_STATE_UP, 621 IF_Mbps(sc->link_speed)); 622 } else if (!link_check && 623 (sc->link_state == EM_LINK_STATE_UP || 624 sc->link_state == EM_LINK_STATE_UP_RESET_PENDING)) { 625 sc->link_speed = 0; 626 sc->link_duplex = 0; 627 sc->link_state = EM_LINK_STATE_DOWN; 628 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 629 } 630 631 /* 632 * A VF stops transmit DMA when its PF reports link down. Reset if 633 * descriptors remain queued so they cannot be sent stale when carrier 634 * returns, matching the periodic check in Linux igbvf. 635 */ 636 if (!link_check && !sc->vf_reset_pending && igbv_tx_pending(sc)) { 637 sc->vf_reset_pending = true; 638 iflib_request_reset(ctx); 639 iflib_admin_intr_deferred(ctx); 640 } 641 /* em_if_init() establishes a new counter baseline after the reset. */ 642 timer_tick = !sc->vf_reset_pending && 643 atomic_readandclear_32(&sc->stats_pending) != 0; 644 if (timer_tick) { 645 em_update_stats_counters(sc); 646 /* iflib closes admission before stop; do not replay after reset. */ 647 if (iflib_is_running(ctx)) 648 igbv_vlan_retry_tick(sc); 649 } 650 } 651 652 static bool 653 igbv_tx_pending(struct e1000_softc *sc) 654 { 655 struct tx_ring *txr; 656 u32 head, tail; 657 658 for (int i = 0; i < sc->tx_num_queues; i++) { 659 txr = &sc->tx_queues[i].txr; 660 head = E1000_READ_REG(&sc->hw, E1000_TDH(txr->me)); 661 tail = E1000_READ_REG(&sc->hw, E1000_TDT(txr->me)); 662 if (head != tail) 663 return (true); 664 } 665 return (false); 666 } 667 668 bool 669 igbv_sanitize_queues(struct e1000_softc *sc) 670 { 671 struct e1000_hw *hw; 672 u32 rxdctl, txdctl; 673 int i, nqueues, retry; 674 675 hw = &sc->hw; 676 switch (hw->mac.type) { 677 case e1000_vfadapt: 678 nqueues = IGBV_82576_QUEUES; 679 break; 680 case e1000_vfadapt_i350: 681 nqueues = IGBV_I350_QUEUES; 682 break; 683 default: 684 return (true); 685 } 686 687 /* 688 * The 82576 and I350 specification updates, Software Clarification 3, 689 * note that VFLR leaves this queue configuration intact. Clear it 690 * before programming the new rings so igbv does not depend on its PF 691 * to sanitize state left by a previous VF owner. igbv uses only queue 692 * zero, but must also clear the unused second 82576 queue. 693 * 694 * Disable every queue first and wait for outstanding DMA activity to 695 * stop before programming TDWBAL/H. Spin only for the normal fast 696 * transition, then sleep until the bounded deadline. 697 */ 698 for (i = 0; i < nqueues; i++) { 699 E1000_WRITE_REG(hw, E1000_RXDCTL(i), 0); 700 E1000_WRITE_REG(hw, E1000_TXDCTL(i), 0); 701 } 702 E1000_WRITE_FLUSH(hw); 703 for (retry = 0; retry < IGBV_QUEUE_DISABLE_RETRIES; retry++) { 704 for (i = 0; i < nqueues; i++) { 705 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); 706 txdctl = E1000_READ_REG(hw, E1000_TXDCTL(i)); 707 if ((rxdctl & E1000_RXDCTL_QUEUE_ENABLE) != 0 || 708 (txdctl & E1000_TXDCTL_QUEUE_ENABLE) != 0) 709 break; 710 } 711 if (i == nqueues) 712 break; 713 if (retry + 1 < IGBV_QUEUE_DISABLE_RETRIES) { 714 if (retry < IGBV_QUEUE_DISABLE_BUSY_RETRIES) 715 DELAY(IGBV_QUEUE_DISABLE_DELAY_US); 716 else 717 pause_sbt("igbvqds", 718 IGBV_QUEUE_DISABLE_PAUSE, 0, 719 C_PREL(1)); 720 } 721 } 722 if (retry == IGBV_QUEUE_DISABLE_RETRIES) { 723 if (ratecheck(&sc->vf_last_queue_log, 724 &igbv_queue_log_interval)) 725 device_printf(sc->dev, 726 "could not disable retained VF queues; " 727 "reset deferred\n"); 728 return (false); 729 } 730 731 for (i = 0; i < nqueues; i++) { 732 E1000_WRITE_REG(hw, E1000_SRRCTL(i), 0); 733 E1000_WRITE_REG(hw, E1000_DCA_RXCTRL(i), 0); 734 E1000_WRITE_REG(hw, E1000_TDWBAL(i), 0); 735 E1000_WRITE_REG(hw, E1000_TDWBAH(i), 0); 736 E1000_WRITE_REG(hw, E1000_DCA_TXCTRL(i), 0); 737 } 738 E1000_WRITE_REG(hw, E1000_VFPSRTYPE, 0); 739 E1000_WRITE_FLUSH(hw); 740 return (true); 741 } 742 743 bool 744 igbv_reset(if_ctx_t ctx) 745 { 746 struct e1000_softc *sc; 747 struct e1000_hw *hw; 748 s32 error; 749 750 sc = iflib_get_softc(ctx); 751 hw = &sc->hw; 752 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); 753 754 /* 755 * Receive-buffer allocation and flow control are port resources owned 756 * by the PF. Zero is an unavailable PBA sentinel, not a per-VF size. 757 */ 758 sc->pba = 0; 759 hw->fc = (struct e1000_fc_info){ 760 .current_mode = e1000_fc_none, 761 .requested_mode = e1000_fc_none, 762 }; 763 764 error = e1000_reset_hw(hw); 765 atomic_store_rel_32(&sc->vf_mbx_ready, 0); 766 sc->vf_queues_sanitized = igbv_sanitize_queues(sc); 767 if (!sc->vf_queues_sanitized) { 768 return (false); 769 } 770 igbv_queue_retry_succeeded(sc); 771 if (error != E1000_SUCCESS) { 772 igbv_log_reset_failure(sc, error, false); 773 return (false); 774 } 775 memset(sc->vf_vfta_stale, 0, sizeof(sc->vf_vfta_stale)); 776 memset(sc->vf_vfta_retry, 0, sizeof(sc->vf_vfta_retry)); 777 sc->vf_vlan_retry_deadline = 0; 778 sc->vf_vlan_retry_cursor = 0; 779 if (e1000_init_hw(hw) < 0) { 780 device_printf(sc->dev, "Hardware Initialization Failed\n"); 781 return (false); 782 } 783 e1000_check_for_link(hw); 784 igbv_mbx_retry_succeeded(sc); 785 return (true); 786 } 787 788 void 789 igbv_initialize_transmit_unit(if_ctx_t ctx) 790 { 791 792 KASSERT(((struct e1000_softc *)iflib_get_softc(ctx))->vf_ifp, 793 ("%s called for a PF", __func__)); 794 em_initialize_transmit_rings(ctx); 795 } 796 797 void 798 igbv_initialize_receive_unit(if_ctx_t ctx) 799 { 800 801 KASSERT(((struct e1000_softc *)iflib_get_softc(ctx))->vf_ifp, 802 ("%s called for a PF", __func__)); 803 igb_initialize_receive_rings(ctx, true); 804 } 805 806 void 807 igbv_if_intr_enable(if_ctx_t ctx) 808 { 809 struct e1000_softc *sc; 810 struct e1000_hw *hw; 811 u32 mask; 812 813 sc = iflib_get_softc(ctx); 814 hw = &sc->hw; 815 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); 816 if (!sc->vf_queues_sanitized || 817 atomic_load_acq_32(&sc->vf_mbx_ready) == 0) 818 return; 819 mask = sc->que_mask | sc->link_mask; 820 821 E1000_WRITE_REG(hw, E1000_EIAC, mask); 822 E1000_WRITE_REG(hw, E1000_EIAM, mask); 823 E1000_WRITE_REG(hw, E1000_EIMS, mask); 824 E1000_WRITE_FLUSH(hw); 825 } 826 827 void 828 igbv_if_intr_disable(if_ctx_t ctx) 829 { 830 struct e1000_softc *sc; 831 struct e1000_hw *hw; 832 833 sc = iflib_get_softc(ctx); 834 hw = &sc->hw; 835 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); 836 837 E1000_WRITE_REG(hw, E1000_EIMC, 0xffffffff); 838 E1000_WRITE_REG(hw, E1000_EIAC, 0); 839 E1000_WRITE_FLUSH(hw); 840 } 841 842 int 843 igbv_get_regs(SYSCTL_HANDLER_ARGS) 844 { 845 struct e1000_softc *sc; 846 struct e1000_hw *hw; 847 struct sbuf *sb; 848 int error; 849 850 sc = (struct e1000_softc *)arg1; 851 hw = &sc->hw; 852 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); 853 854 sb = sbuf_new_for_sysctl(NULL, NULL, 512, req); 855 if (sb == NULL) 856 return (ENOMEM); 857 858 /* 859 * Limited VF register set: 860 * Don't read EICR here because it is clear-on-read. The VF register 861 * file exposes its queue pair at index zero, so this diagnostic does 862 * not depend on the narrower lifetime of iflib's queue arrays. 863 */ 864 sbuf_printf(sb, "VF Registers\n"); 865 sbuf_printf(sb, "\tVTCTRL\t %08x\n", 866 E1000_READ_REG(hw, E1000_CTRL)); 867 sbuf_printf(sb, "\tSTATUS\t %08x\n", 868 E1000_READ_REG(hw, E1000_STATUS)); 869 sbuf_printf(sb, "\tRDLEN\t %08x\n", 870 E1000_READ_REG(hw, E1000_RDLEN(0))); 871 sbuf_printf(sb, "\tRDH\t %08x\n", 872 E1000_READ_REG(hw, E1000_RDH(0))); 873 sbuf_printf(sb, "\tRDT\t %08x\n", 874 E1000_READ_REG(hw, E1000_RDT(0))); 875 sbuf_printf(sb, "\tTDLEN\t %08x\n", 876 E1000_READ_REG(hw, E1000_TDLEN(0))); 877 sbuf_printf(sb, "\tTDH\t %08x\n", 878 E1000_READ_REG(hw, E1000_TDH(0))); 879 sbuf_printf(sb, "\tTDT\t %08x\n", 880 E1000_READ_REG(hw, E1000_TDT(0))); 881 882 error = sbuf_finish(sb); 883 sbuf_delete(sb); 884 return (error); 885 } 886 887 static u_int 888 igbv_copy_uc_addr(void *arg, struct sockaddr_dl *sdl, u_int idx) 889 { 890 struct igb_vf_uc_addr_list *list; 891 const u8 *addr; 892 893 list = arg; 894 addr = (const u8 *)LLADDR(sdl); 895 if (memcmp(addr, list->sc->hw.mac.addr, ETHER_ADDR_LEN) == 0) 896 return (0); 897 if (idx < IGBV_MAX_MAC_FILTERS) 898 memcpy(list->addrs[idx], addr, ETHER_ADDR_LEN); 899 return (1); 900 } 901 902 void 903 igbv_update_uc_addr_list(struct e1000_softc *sc, if_t ifp) 904 { 905 struct igb_vf_uc_addr_list list = { 906 .sc = sc, 907 }; 908 u_int count; 909 910 if (igbv_is_hyperv(sc)) 911 return; 912 913 count = if_foreach_lladdr(ifp, igbv_copy_uc_addr, &list); 914 if (count > IGBV_MAX_MAC_FILTERS) { 915 device_printf(sc->dev, 916 "too many secondary unicast addresses; maximum is %u\n", 917 IGBV_MAX_MAC_FILTERS); 918 } 919 if (count == 0 && !sc->vf_uc_filters_set) 920 return; 921 /* 922 * Linux igb PFs validate the address field before dispatching the CLR 923 * subcommand. Supply the primary address rather than the zero payload 924 * used by igbvf so those PFs actually remove the old filters. FreeBSD 925 * PFs dispatch CLR before inspecting the otherwise-ignored address. 926 */ 927 if (e1000_set_uc_addr_vf(&sc->hw, E1000_VF_MAC_FILTER_CLR, 928 sc->hw.mac.addr) != E1000_SUCCESS) { 929 device_printf(sc->dev, 930 "VF secondary unicast filter clear request failed\n"); 931 return; 932 } 933 sc->vf_uc_filters_set = false; 934 if (count > IGBV_MAX_MAC_FILTERS) 935 return; 936 937 for (u_int i = 0; i < count; i++) { 938 if (e1000_set_uc_addr_vf(&sc->hw, E1000_VF_MAC_FILTER_ADD, 939 list.addrs[i]) != E1000_SUCCESS) { 940 device_printf(sc->dev, 941 "VF secondary unicast filter add request failed " 942 "for %6D\n", list.addrs[i], ":"); 943 } else 944 sc->vf_uc_filters_set = true; 945 usec_delay(200); 946 } 947 } 948 949 void 950 igbv_reconcile_mac(struct e1000_softc *sc, if_t ifp) 951 { 952 u8 *lladdr; 953 954 if (!em_is_valid_ether_addr(sc->hw.mac.addr)) 955 return; 956 lladdr = (u8 *)if_getlladdr(ifp); 957 if (memcmp(lladdr, sc->hw.mac.addr, ETHER_ADDR_LEN) == 0) 958 return; 959 960 device_printf(sc->dev, 961 "PF rejected or replaced the requested MAC; using %6D\n", 962 sc->hw.mac.addr, ":"); 963 /* 964 * if_setlladdr() would re-enter the driver's address-change path. 965 * Initialization already holds the context lock, so update the 966 * storage directly and issue the notification it would have sent. 967 */ 968 memcpy(lladdr, sc->hw.mac.addr, ETHER_ADDR_LEN); 969 970 CURVNET_SET_QUIET(if_getvnet(ifp)); 971 EVENTHANDLER_INVOKE(iflladdr_event, ifp); 972 CURVNET_RESTORE(); 973 } 974