1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2001-2024, Intel Corporation
5 * Copyright (c) 2026 Kevin Bowling <kbowling@FreeBSD.org>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include "if_em.h"
30
31 #include <sys/sbuf.h>
32
33 #define IGBV_82576_QUEUES 2
34 #define IGBV_I350_QUEUES 1
35 #define IGBV_MAX_MAC_FILTERS 3
36 #define IGBV_QUEUE_DISABLE_BUSY_RETRIES 10
37 #define IGBV_QUEUE_DISABLE_DELAY_US 10
38 #define IGBV_QUEUE_DISABLE_PAUSE (100 * SBT_1US)
39 #define IGBV_QUEUE_DISABLE_RETRIES 20
40 #define IGBV_QUEUE_SANITIZE_ATTEMPTS 3
41 #define IGBV_VLAN_RETRY_BATCH 4
42 #define IGBV_VLAN_RETRY_WINDOW (8 * SBT_1S)
43
44 /* Reading these six configuration bytes performs the Hyper-V VF handshake. */
45 #define IGBV_HV_RESET_OFFSET 0x201
46
47 static const struct timeval igbv_queue_log_interval = { 2, 0 };
48 static const struct timeval igbv_mbx_log_interval = { 60, 0 };
49 static const sbintime_t igbv_queue_retry_delay[] = {
50 100 * SBT_1MS,
51 500 * SBT_1MS,
52 };
53 static const sbintime_t igbv_mbx_retry_delay[] = {
54 250 * SBT_1MS,
55 1 * SBT_1S,
56 4 * SBT_1S,
57 8 * SBT_1S,
58 };
59 _Static_assert(nitems(igbv_queue_retry_delay) + 1 ==
60 IGBV_QUEUE_SANITIZE_ATTEMPTS, "missing queue retry delay");
61
62 struct igb_vf_uc_addr_list {
63 struct e1000_softc *sc;
64 u8 addrs[IGBV_MAX_MAC_FILTERS][ETHER_ADDR_LEN];
65 };
66
67 static bool igbv_tx_pending(struct e1000_softc *);
68 static bool igbv_vlan_retry_pending(const struct e1000_softc *);
69 static void igbv_vlan_retry_tick(struct e1000_softc *);
70
71 static s32
igbv_hv_read_mac_addr(struct e1000_hw * hw)72 igbv_hv_read_mac_addr(struct e1000_hw *hw)
73 {
74
75 if (!em_is_valid_ether_addr(hw->mac.perm_addr))
76 return (-E1000_ERR_MAC_INIT);
77 memcpy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN);
78 return (E1000_SUCCESS);
79 }
80
81 static s32
igbv_hv_reset_hw(struct e1000_hw * hw)82 igbv_hv_reset_hw(struct e1000_hw *hw)
83 {
84 struct e1000_osdep *osdep;
85 u8 addr[ETHER_ADDR_LEN];
86 u32 ctrl;
87 int i;
88
89 osdep = hw->back;
90 hw->mac.get_link_status = true;
91 memset(hw->mac.perm_addr, 0, ETHER_ADDR_LEN);
92 /* Hyper-V does not service the native VF posted-message protocol. */
93 hw->mbx.timeout = 0;
94 ctrl = E1000_READ_REG(hw, E1000_CTRL);
95 if (ctrl == 0xffffffff)
96 return (-E1000_ERR_RESET);
97 /* Reset VF-local state before the Hyper-V host reset/MAC exchange. */
98 E1000_WRITE_REG(hw, E1000_CTRL, ctrl | E1000_CTRL_RST);
99 E1000_WRITE_FLUSH(hw);
100 for (i = 0; i < E1000_VF_INIT_TIMEOUT; i++) {
101 if (hw->mbx.ops.check_for_rst(hw, 0) != E1000_SUCCESS)
102 break;
103 DELAY(5);
104 }
105
106 /*
107 * The PF handles 0x201..0x206 as a reset/MAC exchange, including
108 * partial reads. Do not use these bytes for routine status polling.
109 * The host-assigned address also identifies the matching hn interface.
110 */
111 for (i = 0; i < ETHER_ADDR_LEN; i++)
112 addr[i] = pci_read_config(osdep->dev, IGBV_HV_RESET_OFFSET + i, 1);
113 if (!em_is_valid_ether_addr(addr))
114 return (-E1000_ERR_MAC_INIT);
115 memcpy(hw->mac.perm_addr, addr, ETHER_ADDR_LEN);
116 return (E1000_SUCCESS);
117 }
118
119 static s32
igbv_hv_check_for_link(struct e1000_hw * hw)120 igbv_hv_check_for_link(struct e1000_hw *hw)
121 {
122 u32 status;
123
124 /* Leave reset indications for the statistics counter-epoch check. */
125 status = E1000_READ_REG(hw, E1000_STATUS);
126 hw->mac.get_link_status = true;
127 if (status == 0xffffffff)
128 return (-E1000_ERR_MAC_INIT);
129 /* Poll even after link-up so a host port link-down cannot stay cached. */
130 hw->mac.get_link_status = (status & E1000_STATUS_LU) == 0;
131 return (E1000_SUCCESS);
132 }
133
134 static int
igbv_hv_rar_set(struct e1000_hw * hw,u8 * addr,u32 index)135 igbv_hv_rar_set(struct e1000_hw *hw, u8 *addr, u32 index)
136 {
137 int error;
138
139 error = index == 0 &&
140 memcmp(addr, hw->mac.perm_addr, ETHER_ADDR_LEN) == 0 ?
141 E1000_SUCCESS : -E1000_ERR_MAC_INIT;
142 if (igbv_hv_read_mac_addr(hw) != E1000_SUCCESS)
143 return (-E1000_ERR_MAC_INIT);
144 return (error);
145 }
146
147 static void
igbv_hv_update_mc_addr_list(struct e1000_hw * hw __unused,u8 * addrs __unused,u32 count __unused)148 igbv_hv_update_mc_addr_list(struct e1000_hw *hw __unused,
149 u8 *addrs __unused, u32 count __unused)
150 {
151
152 /* The host owns receive-filter policy for the synthetic/VF pair. */
153 }
154
155 void
igbv_init_hv_ops(struct e1000_hw * hw)156 igbv_init_hv_ops(struct e1000_hw *hw)
157 {
158
159 /* Install after the ordinary VF MAC and mailbox parameter setup. */
160 hw->mac.ops.reset_hw = igbv_hv_reset_hw;
161 /* Native init_hw calls rar_set_vf directly, bypassing the RAR op. */
162 hw->mac.ops.init_hw = igbv_hv_read_mac_addr;
163 hw->mac.ops.read_mac_addr = igbv_hv_read_mac_addr;
164 hw->mac.ops.rar_set = igbv_hv_rar_set;
165 hw->mac.ops.check_for_link = igbv_hv_check_for_link;
166 hw->mac.ops.update_mc_addr_list = igbv_hv_update_mc_addr_list;
167 }
168
169 static void
igbv_queue_retry_callout(void * arg)170 igbv_queue_retry_callout(void *arg)
171 {
172 struct e1000_softc *sc;
173 if_t ifp;
174
175 sc = arg;
176 if (atomic_readandclear_32(&sc->vf_queue_retry_pending) == 0)
177 return;
178 ifp = iflib_get_ifp(sc->ctx);
179 if ((if_getflags(ifp) & IFF_UP) == 0) {
180 atomic_set_32(&sc->vf_queue_retry_new_epoch, 1);
181 return;
182 }
183 iflib_request_reset_if_up(sc->ctx);
184 iflib_admin_intr_deferred(sc->ctx);
185 }
186
187 /*
188 * A missing PF can make the posted reset handshake wait for a full mailbox
189 * timeout. Keep that work out of stopped status paths. An administratively
190 * up VF retries complete initialization with an exponential delay capped at
191 * eight seconds, so it recovers without creating a tight mailbox poller.
192 */
193 static void
igbv_mbx_retry_callout(void * arg)194 igbv_mbx_retry_callout(void *arg)
195 {
196 struct e1000_softc *sc;
197 if_t ifp;
198
199 sc = arg;
200 if (atomic_readandclear_32(&sc->vf_mbx_retry_pending) == 0 ||
201 atomic_load_acq_32(&sc->vf_mbx_ready) != 0 ||
202 iflib_in_detach(sc->ctx))
203 return;
204 ifp = iflib_get_ifp(sc->ctx);
205 if ((if_getflags(ifp) & IFF_UP) == 0)
206 return;
207
208 iflib_request_reset_if_up(sc->ctx);
209 iflib_admin_intr_deferred(sc->ctx);
210 }
211
212 static const char *
igbv_reset_error_desc(s32 error)213 igbv_reset_error_desc(s32 error)
214 {
215
216 switch (error) {
217 case -E1000_ERR_RESET:
218 return ("PF reset acknowledgement timed out");
219 case -E1000_ERR_MAC_INIT:
220 return ("PF returned an invalid VF reset response");
221 case -E1000_ERR_MBX:
222 return ("PF mailbox reset exchange failed");
223 default:
224 return ("VF reset handshake failed");
225 }
226 }
227
228 void
igbv_log_reset_failure(struct e1000_softc * sc,s32 error,bool attaching)229 igbv_log_reset_failure(struct e1000_softc *sc, s32 error, bool attaching)
230 {
231
232 /* Report each backoff stage, then limit the steady eight-second retry. */
233 if (sc->vf_mbx_retry_stage == nitems(igbv_mbx_retry_delay) - 1 &&
234 !ratecheck(&sc->vf_last_mbx_log, &igbv_mbx_log_interval))
235 return;
236 device_printf(sc->dev, "%s (%d)%s\n", igbv_reset_error_desc(error),
237 error, attaching ? "; continuing attach" : "");
238 }
239
240 void
igbv_mbx_retry_detach(struct e1000_softc * sc)241 igbv_mbx_retry_detach(struct e1000_softc *sc)
242 {
243
244 if (!sc->vf_mbx_retry_initialized)
245 return;
246 atomic_readandclear_32(&sc->vf_mbx_retry_pending);
247 callout_drain(&sc->vf_mbx_retry);
248 sc->vf_mbx_retry_initialized = false;
249 }
250
251 void
igbv_mbx_retry_prepare(struct e1000_softc * sc)252 igbv_mbx_retry_prepare(struct e1000_softc *sc)
253 {
254
255 if (!sc->vf_mbx_retry_initialized)
256 return;
257 atomic_readandclear_32(&sc->vf_mbx_retry_pending);
258 callout_drain(&sc->vf_mbx_retry);
259 }
260
261 void
igbv_mbx_retry_stop(struct e1000_softc * sc)262 igbv_mbx_retry_stop(struct e1000_softc *sc)
263 {
264 if_t ifp;
265
266 if (!sc->vf_mbx_retry_initialized)
267 return;
268 atomic_readandclear_32(&sc->vf_mbx_retry_pending);
269 callout_drain(&sc->vf_mbx_retry);
270 ifp = iflib_get_ifp(sc->ctx);
271 if ((if_getflags(ifp) & IFF_UP) == 0)
272 sc->vf_mbx_retry_stage = 0;
273 }
274
275 void
igbv_mbx_retry_failed(if_ctx_t ctx)276 igbv_mbx_retry_failed(if_ctx_t ctx)
277 {
278 struct e1000_softc *sc;
279 if_t ifp;
280 sbintime_t delay;
281 u_int stage;
282
283 sc = iflib_get_softc(ctx);
284 atomic_store_rel_32(&sc->vf_mbx_ready, 0);
285 sc->link_speed = 0;
286 sc->link_duplex = 0;
287 if (sc->link_state != EM_LINK_STATE_DOWN) {
288 sc->link_state = EM_LINK_STATE_DOWN;
289 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
290 }
291 iflib_init_failed(ctx);
292
293 ifp = iflib_get_ifp(ctx);
294 if (!sc->vf_mbx_retry_initialized ||
295 (if_getflags(ifp) & IFF_UP) == 0)
296 return;
297 stage = sc->vf_mbx_retry_stage;
298 if (stage >= nitems(igbv_mbx_retry_delay))
299 stage = nitems(igbv_mbx_retry_delay) - 1;
300 delay = igbv_mbx_retry_delay[stage];
301 if (sc->vf_mbx_retry_stage + 1 < nitems(igbv_mbx_retry_delay))
302 sc->vf_mbx_retry_stage++;
303 atomic_set_32(&sc->vf_mbx_retry_pending, 1);
304 callout_reset_sbt(&sc->vf_mbx_retry, delay, 0,
305 igbv_mbx_retry_callout, sc, C_PREL(1));
306 }
307
308 static void
igbv_mbx_retry_succeeded(struct e1000_softc * sc)309 igbv_mbx_retry_succeeded(struct e1000_softc *sc)
310 {
311
312 atomic_store_rel_32(&sc->vf_mbx_ready, 1);
313 atomic_readandclear_32(&sc->vf_mbx_retry_pending);
314 if (sc->vf_mbx_retry_initialized)
315 callout_stop(&sc->vf_mbx_retry);
316 sc->vf_mbx_retry_stage = 0;
317 sc->vf_last_mbx_log.tv_sec = 0;
318 sc->vf_last_mbx_log.tv_usec = 0;
319 }
320
321 void
igbv_queue_retry_detach(struct e1000_softc * sc)322 igbv_queue_retry_detach(struct e1000_softc *sc)
323 {
324
325 if (!sc->vf_queue_retry_initialized)
326 return;
327 atomic_readandclear_32(&sc->vf_queue_retry_pending);
328 callout_drain(&sc->vf_queue_retry);
329 sc->vf_queue_retry_initialized = false;
330 }
331
332 void
igbv_queue_retry_stop(struct e1000_softc * sc)333 igbv_queue_retry_stop(struct e1000_softc *sc)
334 {
335
336 if (!sc->vf_queue_retry_initialized)
337 return;
338 if (atomic_readandclear_32(&sc->vf_queue_retry_pending) != 0)
339 atomic_set_32(&sc->vf_queue_retry_new_epoch, 1);
340 callout_stop(&sc->vf_queue_retry);
341 }
342
343 void
igbv_queue_retry_prepare(struct e1000_softc * sc)344 igbv_queue_retry_prepare(struct e1000_softc *sc)
345 {
346 bool new_epoch;
347
348 new_epoch =
349 atomic_readandclear_32(&sc->vf_queue_retry_new_epoch) != 0;
350 if (!sc->vf_queue_gave_up && !new_epoch)
351 return;
352 sc->vf_queue_failures = 0;
353 sc->vf_queue_gave_up = false;
354 }
355
356 static void
igbv_queue_retry_succeeded(struct e1000_softc * sc)357 igbv_queue_retry_succeeded(struct e1000_softc *sc)
358 {
359
360 atomic_readandclear_32(&sc->vf_queue_retry_pending);
361 atomic_readandclear_32(&sc->vf_queue_retry_new_epoch);
362 if (sc->vf_queue_retry_initialized)
363 callout_stop(&sc->vf_queue_retry);
364 sc->vf_queue_failures = 0;
365 sc->vf_queue_gave_up = false;
366 }
367
368 void
igbv_queue_retry_failed(if_ctx_t ctx)369 igbv_queue_retry_failed(if_ctx_t ctx)
370 {
371 struct e1000_softc *sc;
372 sbintime_t delay;
373
374 sc = iflib_get_softc(ctx);
375 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
376
377 if (sc->vf_queue_failures < IGBV_QUEUE_SANITIZE_ATTEMPTS)
378 sc->vf_queue_failures++;
379 if (sc->vf_queue_failures < IGBV_QUEUE_SANITIZE_ATTEMPTS) {
380 delay = igbv_queue_retry_delay[sc->vf_queue_failures - 1];
381 atomic_set_32(&sc->vf_queue_retry_pending, 1);
382 callout_reset_sbt(&sc->vf_queue_retry, delay, 0,
383 igbv_queue_retry_callout, sc, C_PREL(1));
384 } else if (!sc->vf_queue_gave_up) {
385 atomic_readandclear_32(&sc->vf_queue_retry_pending);
386 callout_stop(&sc->vf_queue_retry);
387 sc->vf_queue_gave_up = true;
388 device_printf(sc->dev,
389 "retained VF queues remained active after %u attempts; "
390 "interface left down; toggle it down/up to retry\n",
391 sc->vf_queue_failures);
392 }
393
394 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
395 iflib_init_failed(ctx);
396 }
397
398 void
igbv_vlan_retry_add(struct e1000_softc * sc,u16 vid)399 igbv_vlan_retry_add(struct e1000_softc *sc, u16 vid)
400 {
401 bool pending;
402
403 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
404 pending = igbv_vlan_retry_pending(sc);
405 sc->vf_vfta_retry[vid >> 5] |= 1U << (vid & 0x1f);
406 /* Bound the whole batch from its first failure, not each new VID. */
407 if (!pending)
408 sc->vf_vlan_retry_deadline =
409 getsbinuptime() + IGBV_VLAN_RETRY_WINDOW;
410 }
411
412 void
igbv_vlan_retry_clear(struct e1000_softc * sc,u16 vid)413 igbv_vlan_retry_clear(struct e1000_softc *sc, u16 vid)
414 {
415
416 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
417 sc->vf_vfta_retry[vid >> 5] &= ~(1U << (vid & 0x1f));
418 }
419
420 static bool
igbv_vlan_retry_pending(const struct e1000_softc * sc)421 igbv_vlan_retry_pending(const struct e1000_softc *sc)
422 {
423 int i;
424
425 for (i = 0; i < EM_VFTA_SIZE; i++)
426 if (sc->vf_vfta_retry[i] != 0)
427 return (true);
428 return (false);
429 }
430
431 static void
igbv_vlan_retry_tick(struct e1000_softc * sc)432 igbv_vlan_retry_tick(struct e1000_softc *sc)
433 {
434 u32 bit;
435 u16 vid;
436 int attempts, i, remaining;
437
438 if (!igbv_vlan_retry_pending(sc)) {
439 sc->vf_vlan_retry_deadline = 0;
440 return;
441 }
442 if (getsbinuptime() >= sc->vf_vlan_retry_deadline) {
443 remaining = 0;
444 for (i = 0; i < EM_VFTA_SIZE; i++)
445 remaining += bitcount32(sc->vf_vfta_retry[i]);
446 memset(sc->vf_vfta_retry, 0, sizeof(sc->vf_vfta_retry));
447 sc->vf_vlan_retry_deadline = 0;
448 device_printf(sc->dev,
449 "VF VLAN restore retries exhausted for %d VIDs\n",
450 remaining);
451 return;
452 }
453
454 /*
455 * The mailbox NACK does not distinguish a transient PF rate limit
456 * from permanent VLVF exhaustion. Retry at the PF's sustained
457 * allowance, but bound the entire recovery window so ENOSPC cannot
458 * create a permanent mailbox poller.
459 */
460 for (attempts = 0, i = 0;
461 attempts < IGBV_VLAN_RETRY_BATCH && i < 4096; i++) {
462 vid = sc->vf_vlan_retry_cursor;
463 sc->vf_vlan_retry_cursor = (vid + 1) & 0xfff;
464 bit = 1U << (vid & 0x1f);
465 if ((sc->vf_vfta_retry[vid >> 5] & bit) == 0)
466 continue;
467 attempts++;
468 if ((sc->shadow_vfta[vid >> 5] & bit) == 0 ||
469 e1000_vfta_set_vf(&sc->hw, vid, true) ==
470 E1000_SUCCESS)
471 sc->vf_vfta_retry[vid >> 5] &= ~bit;
472 }
473 if (!igbv_vlan_retry_pending(sc))
474 sc->vf_vlan_retry_deadline = 0;
475 }
476
477 int
igbv_if_attach_pre(if_ctx_t ctx)478 igbv_if_attach_pre(if_ctx_t ctx)
479 {
480 struct e1000_softc *sc;
481 device_t dev;
482 int error;
483
484 dev = iflib_get_dev(ctx);
485 if (pci_msix_count(dev) < 2) {
486 device_printf(dev, "VF operation requires two MSI-X vectors\n");
487 return (ENXIO);
488 }
489 error = em_if_attach_pre(ctx);
490 if (error != 0)
491 return (error);
492
493 sc = iflib_get_softc(ctx);
494 callout_init(&sc->vf_queue_retry, 1);
495 sc->vf_queue_retry_initialized = true;
496 callout_init(&sc->vf_mbx_retry, 1);
497 sc->vf_mbx_retry_initialized = true;
498
499 KASSERT(sc->vf_ifp &&
500 (iflib_get_sctx(ctx)->isc_flags & IFLIB_IS_VF) != 0,
501 ("%s: igbv attached without VF policy", __func__));
502 return (0);
503 }
504
505 int
igbv_if_attach_post(if_ctx_t ctx)506 igbv_if_attach_post(if_ctx_t ctx)
507 {
508 struct e1000_softc *sc;
509 int error;
510
511 sc = iflib_get_softc(ctx);
512 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
513 if (sc->intr_type != IFLIB_INTR_MSIX) {
514 device_printf(sc->dev, "VF operation requires MSI-X\n");
515 return (ENXIO);
516 }
517 error = em_if_attach_post(ctx);
518 if (error != 0)
519 return (error);
520
521 /*
522 * Attach failures can leave the device sysctl tree registered when
523 * hw.bus.disable_failed_devices is set. Do not publish handlers with
524 * softc arguments until iflib has successfully allocated MSI-X.
525 */
526 em_add_device_sysctls(sc);
527 return (0);
528 }
529
530 int
igbv_if_media_change(if_ctx_t ctx __unused)531 igbv_if_media_change(if_ctx_t ctx __unused)
532 {
533
534 return (EOPNOTSUPP);
535 }
536
537 void
igbv_if_update_admin_status(if_ctx_t ctx)538 igbv_if_update_admin_status(if_ctx_t ctx)
539 {
540 struct e1000_softc *sc;
541 struct e1000_hw *hw;
542 device_t dev;
543 bool link_check, timer_tick;
544
545 sc = iflib_get_softc(ctx);
546 hw = &sc->hw;
547 dev = iflib_get_dev(ctx);
548 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
549
550 if (!iflib_is_running(ctx) ||
551 !sc->vf_queues_sanitized ||
552 atomic_load_acq_32(&sc->vf_mbx_ready) == 0) {
553 if (sc->link_state != EM_LINK_STATE_DOWN) {
554 sc->link_speed = 0;
555 sc->link_duplex = 0;
556 sc->link_state = EM_LINK_STATE_DOWN;
557 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
558 }
559 return;
560 }
561
562 if (igbv_is_hyperv(sc)) {
563 u32 status, txdctl;
564
565 if (sc->vf_reset_pending)
566 return;
567 txdctl = E1000_READ_REG(hw, E1000_TXDCTL(0));
568 if (txdctl != 0xffffffff &&
569 (txdctl & E1000_TXDCTL_QUEUE_ENABLE) == 0) {
570 /* Carrier can remain up after the PF disables queue DMA. */
571 sc->vf_stats_valid = false;
572 sc->link_speed = sc->link_duplex = 0;
573 if (sc->link_state != EM_LINK_STATE_DOWN) {
574 sc->link_state = EM_LINK_STATE_DOWN;
575 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
576 }
577 /*
578 * Do not reset while carrier is down. Limit requests
579 * if the host keeps the queue disabled with carrier up.
580 */
581 status = E1000_READ_REG(hw, E1000_STATUS);
582 if (status != 0xffffffff &&
583 (status & E1000_STATUS_LU) != 0 &&
584 ratecheck(&sc->vf_last_queue_log,
585 &igbv_queue_log_interval)) {
586 device_printf(dev,
587 "Hyper-V VF queue disabled; requesting recovery\n");
588 sc->vf_reset_pending = true;
589 iflib_request_reset_if_up(ctx);
590 iflib_admin_intr_deferred(ctx);
591 }
592 return;
593 }
594 }
595
596 if (!sc->vf_reset_pending &&
597 atomic_readandclear_32(&sc->promisc_pending) != 0)
598 (void)em_if_set_promisc_impl(ctx,
599 if_getflags(iflib_get_ifp(ctx)));
600
601 if (e1000_check_for_link(hw) != E1000_SUCCESS &&
602 !sc->vf_reset_pending) {
603 sc->vf_reset_pending = true;
604 iflib_request_reset(ctx);
605 iflib_admin_intr_deferred(ctx);
606 }
607 link_check = !hw->mac.get_link_status;
608
609 if (link_check &&
610 (sc->link_state == EM_LINK_STATE_DOWN ||
611 sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING)) {
612 e1000_get_speed_and_duplex(hw, &sc->link_speed,
613 &sc->link_duplex);
614 if (bootverbose)
615 device_printf(dev, "Link is up %d Mbps %s\n",
616 sc->link_speed,
617 sc->link_duplex == FULL_DUPLEX ?
618 "Full Duplex" : "Half Duplex");
619 sc->link_state = EM_LINK_STATE_UP;
620 iflib_link_state_change(ctx, LINK_STATE_UP,
621 IF_Mbps(sc->link_speed));
622 } else if (!link_check &&
623 (sc->link_state == EM_LINK_STATE_UP ||
624 sc->link_state == EM_LINK_STATE_UP_RESET_PENDING)) {
625 sc->link_speed = 0;
626 sc->link_duplex = 0;
627 sc->link_state = EM_LINK_STATE_DOWN;
628 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
629 }
630
631 /*
632 * A VF stops transmit DMA when its PF reports link down. Reset if
633 * descriptors remain queued so they cannot be sent stale when carrier
634 * returns, matching the periodic check in Linux igbvf.
635 */
636 if (!link_check && !sc->vf_reset_pending && igbv_tx_pending(sc)) {
637 sc->vf_reset_pending = true;
638 iflib_request_reset(ctx);
639 iflib_admin_intr_deferred(ctx);
640 }
641 /* em_if_init() establishes a new counter baseline after the reset. */
642 timer_tick = !sc->vf_reset_pending &&
643 atomic_readandclear_32(&sc->stats_pending) != 0;
644 if (timer_tick) {
645 em_update_stats_counters(sc);
646 /* iflib closes admission before stop; do not replay after reset. */
647 if (iflib_is_running(ctx))
648 igbv_vlan_retry_tick(sc);
649 }
650 }
651
652 static bool
igbv_tx_pending(struct e1000_softc * sc)653 igbv_tx_pending(struct e1000_softc *sc)
654 {
655 struct tx_ring *txr;
656 u32 head, tail;
657
658 for (int i = 0; i < sc->tx_num_queues; i++) {
659 txr = &sc->tx_queues[i].txr;
660 head = E1000_READ_REG(&sc->hw, E1000_TDH(txr->me));
661 tail = E1000_READ_REG(&sc->hw, E1000_TDT(txr->me));
662 if (head != tail)
663 return (true);
664 }
665 return (false);
666 }
667
668 bool
igbv_sanitize_queues(struct e1000_softc * sc)669 igbv_sanitize_queues(struct e1000_softc *sc)
670 {
671 struct e1000_hw *hw;
672 u32 rxdctl, txdctl;
673 int i, nqueues, retry;
674
675 hw = &sc->hw;
676 switch (hw->mac.type) {
677 case e1000_vfadapt:
678 nqueues = IGBV_82576_QUEUES;
679 break;
680 case e1000_vfadapt_i350:
681 nqueues = IGBV_I350_QUEUES;
682 break;
683 default:
684 return (true);
685 }
686
687 /*
688 * The 82576 and I350 specification updates, Software Clarification 3,
689 * note that VFLR leaves this queue configuration intact. Clear it
690 * before programming the new rings so igbv does not depend on its PF
691 * to sanitize state left by a previous VF owner. igbv uses only queue
692 * zero, but must also clear the unused second 82576 queue.
693 *
694 * Disable every queue first and wait for outstanding DMA activity to
695 * stop before programming TDWBAL/H. Spin only for the normal fast
696 * transition, then sleep until the bounded deadline.
697 */
698 for (i = 0; i < nqueues; i++) {
699 E1000_WRITE_REG(hw, E1000_RXDCTL(i), 0);
700 E1000_WRITE_REG(hw, E1000_TXDCTL(i), 0);
701 }
702 E1000_WRITE_FLUSH(hw);
703 for (retry = 0; retry < IGBV_QUEUE_DISABLE_RETRIES; retry++) {
704 for (i = 0; i < nqueues; i++) {
705 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
706 txdctl = E1000_READ_REG(hw, E1000_TXDCTL(i));
707 if ((rxdctl & E1000_RXDCTL_QUEUE_ENABLE) != 0 ||
708 (txdctl & E1000_TXDCTL_QUEUE_ENABLE) != 0)
709 break;
710 }
711 if (i == nqueues)
712 break;
713 if (retry + 1 < IGBV_QUEUE_DISABLE_RETRIES) {
714 if (retry < IGBV_QUEUE_DISABLE_BUSY_RETRIES)
715 DELAY(IGBV_QUEUE_DISABLE_DELAY_US);
716 else
717 pause_sbt("igbvqds",
718 IGBV_QUEUE_DISABLE_PAUSE, 0,
719 C_PREL(1));
720 }
721 }
722 if (retry == IGBV_QUEUE_DISABLE_RETRIES) {
723 if (ratecheck(&sc->vf_last_queue_log,
724 &igbv_queue_log_interval))
725 device_printf(sc->dev,
726 "could not disable retained VF queues; "
727 "reset deferred\n");
728 return (false);
729 }
730
731 for (i = 0; i < nqueues; i++) {
732 E1000_WRITE_REG(hw, E1000_SRRCTL(i), 0);
733 E1000_WRITE_REG(hw, E1000_DCA_RXCTRL(i), 0);
734 E1000_WRITE_REG(hw, E1000_TDWBAL(i), 0);
735 E1000_WRITE_REG(hw, E1000_TDWBAH(i), 0);
736 E1000_WRITE_REG(hw, E1000_DCA_TXCTRL(i), 0);
737 }
738 E1000_WRITE_REG(hw, E1000_VFPSRTYPE, 0);
739 E1000_WRITE_FLUSH(hw);
740 return (true);
741 }
742
743 bool
igbv_reset(if_ctx_t ctx)744 igbv_reset(if_ctx_t ctx)
745 {
746 struct e1000_softc *sc;
747 struct e1000_hw *hw;
748 s32 error;
749
750 sc = iflib_get_softc(ctx);
751 hw = &sc->hw;
752 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
753
754 /*
755 * Receive-buffer allocation and flow control are port resources owned
756 * by the PF. Zero is an unavailable PBA sentinel, not a per-VF size.
757 */
758 sc->pba = 0;
759 hw->fc = (struct e1000_fc_info){
760 .current_mode = e1000_fc_none,
761 .requested_mode = e1000_fc_none,
762 };
763
764 error = e1000_reset_hw(hw);
765 atomic_store_rel_32(&sc->vf_mbx_ready, 0);
766 sc->vf_queues_sanitized = igbv_sanitize_queues(sc);
767 if (!sc->vf_queues_sanitized) {
768 return (false);
769 }
770 igbv_queue_retry_succeeded(sc);
771 if (error != E1000_SUCCESS) {
772 igbv_log_reset_failure(sc, error, false);
773 return (false);
774 }
775 memset(sc->vf_vfta_stale, 0, sizeof(sc->vf_vfta_stale));
776 memset(sc->vf_vfta_retry, 0, sizeof(sc->vf_vfta_retry));
777 sc->vf_vlan_retry_deadline = 0;
778 sc->vf_vlan_retry_cursor = 0;
779 if (e1000_init_hw(hw) < 0) {
780 device_printf(sc->dev, "Hardware Initialization Failed\n");
781 return (false);
782 }
783 e1000_check_for_link(hw);
784 igbv_mbx_retry_succeeded(sc);
785 return (true);
786 }
787
788 void
igbv_initialize_transmit_unit(if_ctx_t ctx)789 igbv_initialize_transmit_unit(if_ctx_t ctx)
790 {
791
792 KASSERT(((struct e1000_softc *)iflib_get_softc(ctx))->vf_ifp,
793 ("%s called for a PF", __func__));
794 em_initialize_transmit_rings(ctx);
795 }
796
797 void
igbv_initialize_receive_unit(if_ctx_t ctx)798 igbv_initialize_receive_unit(if_ctx_t ctx)
799 {
800
801 KASSERT(((struct e1000_softc *)iflib_get_softc(ctx))->vf_ifp,
802 ("%s called for a PF", __func__));
803 igb_initialize_receive_rings(ctx, true);
804 }
805
806 void
igbv_if_intr_enable(if_ctx_t ctx)807 igbv_if_intr_enable(if_ctx_t ctx)
808 {
809 struct e1000_softc *sc;
810 struct e1000_hw *hw;
811 u32 mask;
812
813 sc = iflib_get_softc(ctx);
814 hw = &sc->hw;
815 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
816 if (!sc->vf_queues_sanitized ||
817 atomic_load_acq_32(&sc->vf_mbx_ready) == 0)
818 return;
819 mask = sc->que_mask | sc->link_mask;
820
821 E1000_WRITE_REG(hw, E1000_EIAC, mask);
822 E1000_WRITE_REG(hw, E1000_EIAM, mask);
823 E1000_WRITE_REG(hw, E1000_EIMS, mask);
824 E1000_WRITE_FLUSH(hw);
825 }
826
827 void
igbv_if_intr_disable(if_ctx_t ctx)828 igbv_if_intr_disable(if_ctx_t ctx)
829 {
830 struct e1000_softc *sc;
831 struct e1000_hw *hw;
832
833 sc = iflib_get_softc(ctx);
834 hw = &sc->hw;
835 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
836
837 E1000_WRITE_REG(hw, E1000_EIMC, 0xffffffff);
838 E1000_WRITE_REG(hw, E1000_EIAC, 0);
839 E1000_WRITE_FLUSH(hw);
840 }
841
842 int
igbv_get_regs(SYSCTL_HANDLER_ARGS)843 igbv_get_regs(SYSCTL_HANDLER_ARGS)
844 {
845 struct e1000_softc *sc;
846 struct e1000_hw *hw;
847 struct sbuf *sb;
848 int error;
849
850 sc = (struct e1000_softc *)arg1;
851 hw = &sc->hw;
852 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
853
854 sb = sbuf_new_for_sysctl(NULL, NULL, 512, req);
855 if (sb == NULL)
856 return (ENOMEM);
857
858 /*
859 * Limited VF register set:
860 * Don't read EICR here because it is clear-on-read. The VF register
861 * file exposes its queue pair at index zero, so this diagnostic does
862 * not depend on the narrower lifetime of iflib's queue arrays.
863 */
864 sbuf_printf(sb, "VF Registers\n");
865 sbuf_printf(sb, "\tVTCTRL\t %08x\n",
866 E1000_READ_REG(hw, E1000_CTRL));
867 sbuf_printf(sb, "\tSTATUS\t %08x\n",
868 E1000_READ_REG(hw, E1000_STATUS));
869 sbuf_printf(sb, "\tRDLEN\t %08x\n",
870 E1000_READ_REG(hw, E1000_RDLEN(0)));
871 sbuf_printf(sb, "\tRDH\t %08x\n",
872 E1000_READ_REG(hw, E1000_RDH(0)));
873 sbuf_printf(sb, "\tRDT\t %08x\n",
874 E1000_READ_REG(hw, E1000_RDT(0)));
875 sbuf_printf(sb, "\tTDLEN\t %08x\n",
876 E1000_READ_REG(hw, E1000_TDLEN(0)));
877 sbuf_printf(sb, "\tTDH\t %08x\n",
878 E1000_READ_REG(hw, E1000_TDH(0)));
879 sbuf_printf(sb, "\tTDT\t %08x\n",
880 E1000_READ_REG(hw, E1000_TDT(0)));
881
882 error = sbuf_finish(sb);
883 sbuf_delete(sb);
884 return (error);
885 }
886
887 static u_int
igbv_copy_uc_addr(void * arg,struct sockaddr_dl * sdl,u_int idx)888 igbv_copy_uc_addr(void *arg, struct sockaddr_dl *sdl, u_int idx)
889 {
890 struct igb_vf_uc_addr_list *list;
891 const u8 *addr;
892
893 list = arg;
894 addr = (const u8 *)LLADDR(sdl);
895 if (memcmp(addr, list->sc->hw.mac.addr, ETHER_ADDR_LEN) == 0)
896 return (0);
897 if (idx < IGBV_MAX_MAC_FILTERS)
898 memcpy(list->addrs[idx], addr, ETHER_ADDR_LEN);
899 return (1);
900 }
901
902 void
igbv_update_uc_addr_list(struct e1000_softc * sc,if_t ifp)903 igbv_update_uc_addr_list(struct e1000_softc *sc, if_t ifp)
904 {
905 struct igb_vf_uc_addr_list list = {
906 .sc = sc,
907 };
908 u_int count;
909
910 if (igbv_is_hyperv(sc))
911 return;
912
913 count = if_foreach_lladdr(ifp, igbv_copy_uc_addr, &list);
914 if (count > IGBV_MAX_MAC_FILTERS) {
915 device_printf(sc->dev,
916 "too many secondary unicast addresses; maximum is %u\n",
917 IGBV_MAX_MAC_FILTERS);
918 }
919 if (count == 0 && !sc->vf_uc_filters_set)
920 return;
921 /*
922 * Linux igb PFs validate the address field before dispatching the CLR
923 * subcommand. Supply the primary address rather than the zero payload
924 * used by igbvf so those PFs actually remove the old filters. FreeBSD
925 * PFs dispatch CLR before inspecting the otherwise-ignored address.
926 */
927 if (e1000_set_uc_addr_vf(&sc->hw, E1000_VF_MAC_FILTER_CLR,
928 sc->hw.mac.addr) != E1000_SUCCESS) {
929 device_printf(sc->dev,
930 "VF secondary unicast filter clear request failed\n");
931 return;
932 }
933 sc->vf_uc_filters_set = false;
934 if (count > IGBV_MAX_MAC_FILTERS)
935 return;
936
937 for (u_int i = 0; i < count; i++) {
938 if (e1000_set_uc_addr_vf(&sc->hw, E1000_VF_MAC_FILTER_ADD,
939 list.addrs[i]) != E1000_SUCCESS) {
940 device_printf(sc->dev,
941 "VF secondary unicast filter add request failed "
942 "for %6D\n", list.addrs[i], ":");
943 } else
944 sc->vf_uc_filters_set = true;
945 usec_delay(200);
946 }
947 }
948
949 void
igbv_reconcile_mac(struct e1000_softc * sc,if_t ifp)950 igbv_reconcile_mac(struct e1000_softc *sc, if_t ifp)
951 {
952 u8 *lladdr;
953
954 if (!em_is_valid_ether_addr(sc->hw.mac.addr))
955 return;
956 lladdr = (u8 *)if_getlladdr(ifp);
957 if (memcmp(lladdr, sc->hw.mac.addr, ETHER_ADDR_LEN) == 0)
958 return;
959
960 device_printf(sc->dev,
961 "PF rejected or replaced the requested MAC; using %6D\n",
962 sc->hw.mac.addr, ":");
963 /*
964 * if_setlladdr() would re-enter the driver's address-change path.
965 * Initialization already holds the context lock, so update the
966 * storage directly and issue the notification it would have sent.
967 */
968 memcpy(lladdr, sc->hw.mac.addr, ETHER_ADDR_LEN);
969
970 CURVNET_SET_QUIET(if_getvnet(ifp));
971 EVENTHANDLER_INVOKE(iflladdr_event, ifp);
972 CURVNET_RESTORE();
973 }
974