1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2001-2024, Intel Corporation
5 * Copyright (c) 2026 Kevin Bowling <kbowling@FreeBSD.org>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include "if_em.h"
30
31 #include <sys/sbuf.h>
32
33 #define IGBV_82576_QUEUES 2
34 #define IGBV_I350_QUEUES 1
35 #define IGBV_MAX_MAC_FILTERS 3
36 #define IGBV_QUEUE_DISABLE_BUSY_RETRIES 10
37 #define IGBV_QUEUE_DISABLE_DELAY_US 10
38 #define IGBV_QUEUE_DISABLE_PAUSE (100 * SBT_1US)
39 #define IGBV_QUEUE_DISABLE_RETRIES 20
40 #define IGBV_QUEUE_SANITIZE_ATTEMPTS 3
41 #define IGBV_VLAN_RETRY_BATCH 4
42 #define IGBV_VLAN_RETRY_WINDOW (8 * SBT_1S)
43
44 static const struct timeval igbv_queue_log_interval = { 2, 0 };
45 static const struct timeval igbv_mbx_log_interval = { 60, 0 };
46 static const sbintime_t igbv_queue_retry_delay[] = {
47 100 * SBT_1MS,
48 500 * SBT_1MS,
49 };
50 static const sbintime_t igbv_mbx_retry_delay[] = {
51 250 * SBT_1MS,
52 1 * SBT_1S,
53 4 * SBT_1S,
54 8 * SBT_1S,
55 };
56 _Static_assert(nitems(igbv_queue_retry_delay) + 1 ==
57 IGBV_QUEUE_SANITIZE_ATTEMPTS, "missing queue retry delay");
58
59 struct igb_vf_uc_addr_list {
60 struct e1000_softc *sc;
61 u8 addrs[IGBV_MAX_MAC_FILTERS][ETHER_ADDR_LEN];
62 };
63
64 static bool igbv_tx_pending(struct e1000_softc *);
65 static bool igbv_vlan_retry_pending(const struct e1000_softc *);
66 static void igbv_vlan_retry_tick(struct e1000_softc *);
67
68 static void
igbv_queue_retry_callout(void * arg)69 igbv_queue_retry_callout(void *arg)
70 {
71 struct e1000_softc *sc;
72 if_t ifp;
73
74 sc = arg;
75 if (atomic_readandclear_32(&sc->vf_queue_retry_pending) == 0)
76 return;
77 ifp = iflib_get_ifp(sc->ctx);
78 if ((if_getflags(ifp) & IFF_UP) == 0) {
79 atomic_set_32(&sc->vf_queue_retry_new_epoch, 1);
80 return;
81 }
82 iflib_request_reset_if_up(sc->ctx);
83 iflib_admin_intr_deferred(sc->ctx);
84 }
85
86 /*
87 * A missing PF can make the posted reset handshake wait for a full mailbox
88 * timeout. Keep that work out of stopped status paths. An administratively
89 * up VF retries complete initialization with an exponential delay capped at
90 * eight seconds, so it recovers without creating a tight mailbox poller.
91 */
92 static void
igbv_mbx_retry_callout(void * arg)93 igbv_mbx_retry_callout(void *arg)
94 {
95 struct e1000_softc *sc;
96 if_t ifp;
97
98 sc = arg;
99 if (atomic_readandclear_32(&sc->vf_mbx_retry_pending) == 0 ||
100 atomic_load_acq_32(&sc->vf_mbx_ready) != 0 ||
101 iflib_in_detach(sc->ctx))
102 return;
103 ifp = iflib_get_ifp(sc->ctx);
104 if ((if_getflags(ifp) & IFF_UP) == 0)
105 return;
106
107 iflib_request_reset_if_up(sc->ctx);
108 iflib_admin_intr_deferred(sc->ctx);
109 }
110
111 static const char *
igbv_reset_error_desc(s32 error)112 igbv_reset_error_desc(s32 error)
113 {
114
115 switch (error) {
116 case -E1000_ERR_RESET:
117 return ("PF reset acknowledgement timed out");
118 case -E1000_ERR_MAC_INIT:
119 return ("PF returned an invalid VF reset response");
120 case -E1000_ERR_MBX:
121 return ("PF mailbox reset exchange failed");
122 default:
123 return ("VF reset handshake failed");
124 }
125 }
126
127 void
igbv_log_reset_failure(struct e1000_softc * sc,s32 error,bool attaching)128 igbv_log_reset_failure(struct e1000_softc *sc, s32 error, bool attaching)
129 {
130
131 /* Report each backoff stage, then limit the steady eight-second retry. */
132 if (sc->vf_mbx_retry_stage == nitems(igbv_mbx_retry_delay) - 1 &&
133 !ratecheck(&sc->vf_last_mbx_log, &igbv_mbx_log_interval))
134 return;
135 device_printf(sc->dev, "%s (%d)%s\n", igbv_reset_error_desc(error),
136 error, attaching ? "; continuing attach" : "");
137 }
138
139 void
igbv_mbx_retry_detach(struct e1000_softc * sc)140 igbv_mbx_retry_detach(struct e1000_softc *sc)
141 {
142
143 if (!sc->vf_mbx_retry_initialized)
144 return;
145 atomic_readandclear_32(&sc->vf_mbx_retry_pending);
146 callout_drain(&sc->vf_mbx_retry);
147 sc->vf_mbx_retry_initialized = false;
148 }
149
150 void
igbv_mbx_retry_prepare(struct e1000_softc * sc)151 igbv_mbx_retry_prepare(struct e1000_softc *sc)
152 {
153
154 if (!sc->vf_mbx_retry_initialized)
155 return;
156 atomic_readandclear_32(&sc->vf_mbx_retry_pending);
157 callout_drain(&sc->vf_mbx_retry);
158 }
159
160 void
igbv_mbx_retry_stop(struct e1000_softc * sc)161 igbv_mbx_retry_stop(struct e1000_softc *sc)
162 {
163 if_t ifp;
164
165 if (!sc->vf_mbx_retry_initialized)
166 return;
167 atomic_readandclear_32(&sc->vf_mbx_retry_pending);
168 callout_drain(&sc->vf_mbx_retry);
169 ifp = iflib_get_ifp(sc->ctx);
170 if ((if_getflags(ifp) & IFF_UP) == 0)
171 sc->vf_mbx_retry_stage = 0;
172 }
173
174 void
igbv_mbx_retry_failed(if_ctx_t ctx)175 igbv_mbx_retry_failed(if_ctx_t ctx)
176 {
177 struct e1000_softc *sc;
178 if_t ifp;
179 sbintime_t delay;
180 u_int stage;
181
182 sc = iflib_get_softc(ctx);
183 atomic_store_rel_32(&sc->vf_mbx_ready, 0);
184 sc->link_speed = 0;
185 sc->link_duplex = 0;
186 if (sc->link_state != EM_LINK_STATE_DOWN) {
187 sc->link_state = EM_LINK_STATE_DOWN;
188 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
189 }
190 iflib_init_failed(ctx);
191
192 ifp = iflib_get_ifp(ctx);
193 if (!sc->vf_mbx_retry_initialized ||
194 (if_getflags(ifp) & IFF_UP) == 0)
195 return;
196 stage = sc->vf_mbx_retry_stage;
197 if (stage >= nitems(igbv_mbx_retry_delay))
198 stage = nitems(igbv_mbx_retry_delay) - 1;
199 delay = igbv_mbx_retry_delay[stage];
200 if (sc->vf_mbx_retry_stage + 1 < nitems(igbv_mbx_retry_delay))
201 sc->vf_mbx_retry_stage++;
202 atomic_set_32(&sc->vf_mbx_retry_pending, 1);
203 callout_reset_sbt(&sc->vf_mbx_retry, delay, 0,
204 igbv_mbx_retry_callout, sc, C_PREL(1));
205 }
206
207 static void
igbv_mbx_retry_succeeded(struct e1000_softc * sc)208 igbv_mbx_retry_succeeded(struct e1000_softc *sc)
209 {
210
211 atomic_store_rel_32(&sc->vf_mbx_ready, 1);
212 atomic_readandclear_32(&sc->vf_mbx_retry_pending);
213 if (sc->vf_mbx_retry_initialized)
214 callout_stop(&sc->vf_mbx_retry);
215 sc->vf_mbx_retry_stage = 0;
216 sc->vf_last_mbx_log.tv_sec = 0;
217 sc->vf_last_mbx_log.tv_usec = 0;
218 }
219
220 void
igbv_queue_retry_detach(struct e1000_softc * sc)221 igbv_queue_retry_detach(struct e1000_softc *sc)
222 {
223
224 if (!sc->vf_queue_retry_initialized)
225 return;
226 atomic_readandclear_32(&sc->vf_queue_retry_pending);
227 callout_drain(&sc->vf_queue_retry);
228 sc->vf_queue_retry_initialized = false;
229 }
230
231 void
igbv_queue_retry_stop(struct e1000_softc * sc)232 igbv_queue_retry_stop(struct e1000_softc *sc)
233 {
234
235 if (!sc->vf_queue_retry_initialized)
236 return;
237 if (atomic_readandclear_32(&sc->vf_queue_retry_pending) != 0)
238 atomic_set_32(&sc->vf_queue_retry_new_epoch, 1);
239 callout_stop(&sc->vf_queue_retry);
240 }
241
242 void
igbv_queue_retry_prepare(struct e1000_softc * sc)243 igbv_queue_retry_prepare(struct e1000_softc *sc)
244 {
245 bool new_epoch;
246
247 new_epoch =
248 atomic_readandclear_32(&sc->vf_queue_retry_new_epoch) != 0;
249 if (!sc->vf_queue_gave_up && !new_epoch)
250 return;
251 sc->vf_queue_failures = 0;
252 sc->vf_queue_gave_up = false;
253 }
254
255 static void
igbv_queue_retry_succeeded(struct e1000_softc * sc)256 igbv_queue_retry_succeeded(struct e1000_softc *sc)
257 {
258
259 atomic_readandclear_32(&sc->vf_queue_retry_pending);
260 atomic_readandclear_32(&sc->vf_queue_retry_new_epoch);
261 if (sc->vf_queue_retry_initialized)
262 callout_stop(&sc->vf_queue_retry);
263 sc->vf_queue_failures = 0;
264 sc->vf_queue_gave_up = false;
265 }
266
267 void
igbv_queue_retry_failed(if_ctx_t ctx)268 igbv_queue_retry_failed(if_ctx_t ctx)
269 {
270 struct e1000_softc *sc;
271 sbintime_t delay;
272
273 sc = iflib_get_softc(ctx);
274 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
275
276 if (sc->vf_queue_failures < IGBV_QUEUE_SANITIZE_ATTEMPTS)
277 sc->vf_queue_failures++;
278 if (sc->vf_queue_failures < IGBV_QUEUE_SANITIZE_ATTEMPTS) {
279 delay = igbv_queue_retry_delay[sc->vf_queue_failures - 1];
280 atomic_set_32(&sc->vf_queue_retry_pending, 1);
281 callout_reset_sbt(&sc->vf_queue_retry, delay, 0,
282 igbv_queue_retry_callout, sc, C_PREL(1));
283 } else if (!sc->vf_queue_gave_up) {
284 atomic_readandclear_32(&sc->vf_queue_retry_pending);
285 callout_stop(&sc->vf_queue_retry);
286 sc->vf_queue_gave_up = true;
287 device_printf(sc->dev,
288 "retained VF queues remained active after %u attempts; "
289 "interface left down; toggle it down/up to retry\n",
290 sc->vf_queue_failures);
291 }
292
293 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
294 iflib_init_failed(ctx);
295 }
296
297 void
igbv_vlan_retry_add(struct e1000_softc * sc,u16 vid)298 igbv_vlan_retry_add(struct e1000_softc *sc, u16 vid)
299 {
300 bool pending;
301
302 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
303 pending = igbv_vlan_retry_pending(sc);
304 sc->vf_vfta_retry[vid >> 5] |= 1U << (vid & 0x1f);
305 /* Bound the whole batch from its first failure, not each new VID. */
306 if (!pending)
307 sc->vf_vlan_retry_deadline =
308 getsbinuptime() + IGBV_VLAN_RETRY_WINDOW;
309 }
310
311 void
igbv_vlan_retry_clear(struct e1000_softc * sc,u16 vid)312 igbv_vlan_retry_clear(struct e1000_softc *sc, u16 vid)
313 {
314
315 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
316 sc->vf_vfta_retry[vid >> 5] &= ~(1U << (vid & 0x1f));
317 }
318
319 static bool
igbv_vlan_retry_pending(const struct e1000_softc * sc)320 igbv_vlan_retry_pending(const struct e1000_softc *sc)
321 {
322 int i;
323
324 for (i = 0; i < EM_VFTA_SIZE; i++)
325 if (sc->vf_vfta_retry[i] != 0)
326 return (true);
327 return (false);
328 }
329
330 static void
igbv_vlan_retry_tick(struct e1000_softc * sc)331 igbv_vlan_retry_tick(struct e1000_softc *sc)
332 {
333 u32 bit;
334 u16 vid;
335 int attempts, i, remaining;
336
337 if (!igbv_vlan_retry_pending(sc)) {
338 sc->vf_vlan_retry_deadline = 0;
339 return;
340 }
341 if (getsbinuptime() >= sc->vf_vlan_retry_deadline) {
342 remaining = 0;
343 for (i = 0; i < EM_VFTA_SIZE; i++)
344 remaining += bitcount32(sc->vf_vfta_retry[i]);
345 memset(sc->vf_vfta_retry, 0, sizeof(sc->vf_vfta_retry));
346 sc->vf_vlan_retry_deadline = 0;
347 device_printf(sc->dev,
348 "VF VLAN restore retries exhausted for %d VIDs\n",
349 remaining);
350 return;
351 }
352
353 /*
354 * The mailbox NACK does not distinguish a transient PF rate limit
355 * from permanent VLVF exhaustion. Retry at the PF's sustained
356 * allowance, but bound the entire recovery window so ENOSPC cannot
357 * create a permanent mailbox poller.
358 */
359 for (attempts = 0, i = 0;
360 attempts < IGBV_VLAN_RETRY_BATCH && i < 4096; i++) {
361 vid = sc->vf_vlan_retry_cursor;
362 sc->vf_vlan_retry_cursor = (vid + 1) & 0xfff;
363 bit = 1U << (vid & 0x1f);
364 if ((sc->vf_vfta_retry[vid >> 5] & bit) == 0)
365 continue;
366 attempts++;
367 if ((sc->shadow_vfta[vid >> 5] & bit) == 0 ||
368 e1000_vfta_set_vf(&sc->hw, vid, true) ==
369 E1000_SUCCESS)
370 sc->vf_vfta_retry[vid >> 5] &= ~bit;
371 }
372 if (!igbv_vlan_retry_pending(sc))
373 sc->vf_vlan_retry_deadline = 0;
374 }
375
376 int
igbv_if_attach_pre(if_ctx_t ctx)377 igbv_if_attach_pre(if_ctx_t ctx)
378 {
379 struct e1000_softc *sc;
380 device_t dev;
381 int error;
382
383 dev = iflib_get_dev(ctx);
384 if (pci_msix_count(dev) < 2) {
385 device_printf(dev, "VF operation requires two MSI-X vectors\n");
386 return (ENXIO);
387 }
388 error = em_if_attach_pre(ctx);
389 if (error != 0)
390 return (error);
391
392 sc = iflib_get_softc(ctx);
393 callout_init(&sc->vf_queue_retry, 1);
394 sc->vf_queue_retry_initialized = true;
395 callout_init(&sc->vf_mbx_retry, 1);
396 sc->vf_mbx_retry_initialized = true;
397
398 KASSERT(sc->vf_ifp &&
399 (iflib_get_sctx(ctx)->isc_flags & IFLIB_IS_VF) != 0,
400 ("%s: igbv attached without VF policy", __func__));
401 return (0);
402 }
403
404 int
igbv_if_attach_post(if_ctx_t ctx)405 igbv_if_attach_post(if_ctx_t ctx)
406 {
407 struct e1000_softc *sc;
408 int error;
409
410 sc = iflib_get_softc(ctx);
411 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
412 if (sc->intr_type != IFLIB_INTR_MSIX) {
413 device_printf(sc->dev, "VF operation requires MSI-X\n");
414 return (ENXIO);
415 }
416 error = em_if_attach_post(ctx);
417 if (error != 0)
418 return (error);
419
420 /*
421 * Attach failures can leave the device sysctl tree registered when
422 * hw.bus.disable_failed_devices is set. Do not publish handlers with
423 * softc arguments until iflib has successfully allocated MSI-X.
424 */
425 em_add_device_sysctls(sc);
426 return (0);
427 }
428
429 int
igbv_if_media_change(if_ctx_t ctx __unused)430 igbv_if_media_change(if_ctx_t ctx __unused)
431 {
432
433 return (EOPNOTSUPP);
434 }
435
436 void
igbv_if_update_admin_status(if_ctx_t ctx)437 igbv_if_update_admin_status(if_ctx_t ctx)
438 {
439 struct e1000_softc *sc;
440 struct e1000_hw *hw;
441 device_t dev;
442 bool link_check, timer_tick;
443
444 sc = iflib_get_softc(ctx);
445 hw = &sc->hw;
446 dev = iflib_get_dev(ctx);
447 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
448
449 if ((if_getdrvflags(iflib_get_ifp(ctx)) & IFF_DRV_RUNNING) == 0 ||
450 !sc->vf_queues_sanitized ||
451 atomic_load_acq_32(&sc->vf_mbx_ready) == 0) {
452 if (sc->link_state != EM_LINK_STATE_DOWN) {
453 sc->link_speed = 0;
454 sc->link_duplex = 0;
455 sc->link_state = EM_LINK_STATE_DOWN;
456 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
457 }
458 return;
459 }
460
461 if (!sc->vf_reset_pending &&
462 atomic_readandclear_32(&sc->promisc_pending) != 0)
463 (void)em_if_set_promisc_impl(ctx,
464 if_getflags(iflib_get_ifp(ctx)));
465
466 if (e1000_check_for_link(hw) != E1000_SUCCESS &&
467 !sc->vf_reset_pending) {
468 sc->vf_reset_pending = true;
469 iflib_request_reset(ctx);
470 iflib_admin_intr_deferred(ctx);
471 }
472 link_check = !hw->mac.get_link_status;
473
474 if (link_check &&
475 (sc->link_state == EM_LINK_STATE_DOWN ||
476 sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING)) {
477 e1000_get_speed_and_duplex(hw, &sc->link_speed,
478 &sc->link_duplex);
479 if (bootverbose)
480 device_printf(dev, "Link is up %d Mbps %s\n",
481 sc->link_speed,
482 sc->link_duplex == FULL_DUPLEX ?
483 "Full Duplex" : "Half Duplex");
484 sc->link_state = EM_LINK_STATE_UP;
485 iflib_link_state_change(ctx, LINK_STATE_UP,
486 IF_Mbps(sc->link_speed));
487 } else if (!link_check &&
488 (sc->link_state == EM_LINK_STATE_UP ||
489 sc->link_state == EM_LINK_STATE_UP_RESET_PENDING)) {
490 sc->link_speed = 0;
491 sc->link_duplex = 0;
492 sc->link_state = EM_LINK_STATE_DOWN;
493 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
494 }
495
496 /*
497 * A VF stops transmit DMA when its PF reports link down. Reset if
498 * descriptors remain queued so they cannot be sent stale when carrier
499 * returns, matching the periodic check in Linux igbvf.
500 */
501 if (!link_check && !sc->vf_reset_pending && igbv_tx_pending(sc)) {
502 sc->vf_reset_pending = true;
503 iflib_request_reset(ctx);
504 iflib_admin_intr_deferred(ctx);
505 }
506 /* em_if_init() establishes a new counter baseline after the reset. */
507 timer_tick = !sc->vf_reset_pending &&
508 atomic_readandclear_32(&sc->stats_pending) != 0;
509 if (timer_tick) {
510 em_update_stats_counters(sc);
511 /* iflib clears RUNNING before stop; do not replay after reset. */
512 if ((if_getdrvflags(iflib_get_ifp(ctx)) &
513 IFF_DRV_RUNNING) != 0)
514 igbv_vlan_retry_tick(sc);
515 }
516 }
517
518 static bool
igbv_tx_pending(struct e1000_softc * sc)519 igbv_tx_pending(struct e1000_softc *sc)
520 {
521 struct tx_ring *txr;
522 u32 head, tail;
523
524 for (int i = 0; i < sc->tx_num_queues; i++) {
525 txr = &sc->tx_queues[i].txr;
526 head = E1000_READ_REG(&sc->hw, E1000_TDH(txr->me));
527 tail = E1000_READ_REG(&sc->hw, E1000_TDT(txr->me));
528 if (head != tail)
529 return (true);
530 }
531 return (false);
532 }
533
534 static bool
igbv_sanitize_queues(struct e1000_softc * sc)535 igbv_sanitize_queues(struct e1000_softc *sc)
536 {
537 struct e1000_hw *hw;
538 u32 rxdctl, txdctl;
539 int i, nqueues, retry;
540
541 hw = &sc->hw;
542 switch (hw->mac.type) {
543 case e1000_vfadapt:
544 nqueues = IGBV_82576_QUEUES;
545 break;
546 case e1000_vfadapt_i350:
547 nqueues = IGBV_I350_QUEUES;
548 break;
549 default:
550 return (true);
551 }
552
553 /*
554 * The 82576 and I350 specification updates, Software Clarification 3,
555 * note that VFLR leaves this queue configuration intact. Clear it
556 * before programming the new rings so igbv does not depend on its PF
557 * to sanitize state left by a previous VF owner. igbv uses only queue
558 * zero, but must also clear the unused second 82576 queue.
559 *
560 * Disable every queue first and wait for outstanding DMA activity to
561 * stop before programming TDWBAL/H. Spin only for the normal fast
562 * transition, then sleep until the bounded deadline.
563 */
564 for (i = 0; i < nqueues; i++) {
565 E1000_WRITE_REG(hw, E1000_RXDCTL(i), 0);
566 E1000_WRITE_REG(hw, E1000_TXDCTL(i), 0);
567 }
568 E1000_WRITE_FLUSH(hw);
569 for (retry = 0; retry < IGBV_QUEUE_DISABLE_RETRIES; retry++) {
570 for (i = 0; i < nqueues; i++) {
571 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
572 txdctl = E1000_READ_REG(hw, E1000_TXDCTL(i));
573 if ((rxdctl & E1000_RXDCTL_QUEUE_ENABLE) != 0 ||
574 (txdctl & E1000_TXDCTL_QUEUE_ENABLE) != 0)
575 break;
576 }
577 if (i == nqueues)
578 break;
579 if (retry + 1 < IGBV_QUEUE_DISABLE_RETRIES) {
580 if (retry < IGBV_QUEUE_DISABLE_BUSY_RETRIES)
581 DELAY(IGBV_QUEUE_DISABLE_DELAY_US);
582 else
583 pause_sbt("igbvqds",
584 IGBV_QUEUE_DISABLE_PAUSE, 0,
585 C_PREL(1));
586 }
587 }
588 if (retry == IGBV_QUEUE_DISABLE_RETRIES) {
589 if (ratecheck(&sc->vf_last_queue_log,
590 &igbv_queue_log_interval))
591 device_printf(sc->dev,
592 "could not disable retained VF queues; "
593 "reset deferred\n");
594 return (false);
595 }
596
597 for (i = 0; i < nqueues; i++) {
598 E1000_WRITE_REG(hw, E1000_SRRCTL(i), 0);
599 E1000_WRITE_REG(hw, E1000_DCA_RXCTRL(i), 0);
600 E1000_WRITE_REG(hw, E1000_TDWBAL(i), 0);
601 E1000_WRITE_REG(hw, E1000_TDWBAH(i), 0);
602 E1000_WRITE_REG(hw, E1000_DCA_TXCTRL(i), 0);
603 }
604 E1000_WRITE_REG(hw, E1000_VFPSRTYPE, 0);
605 E1000_WRITE_FLUSH(hw);
606 return (true);
607 }
608
609 bool
igbv_reset(if_ctx_t ctx)610 igbv_reset(if_ctx_t ctx)
611 {
612 struct e1000_softc *sc;
613 struct e1000_hw *hw;
614 s32 error;
615
616 sc = iflib_get_softc(ctx);
617 hw = &sc->hw;
618 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
619
620 /*
621 * Receive-buffer allocation and flow control are port resources owned
622 * by the PF. Zero is an unavailable PBA sentinel, not a per-VF size.
623 */
624 sc->pba = 0;
625 hw->fc = (struct e1000_fc_info){
626 .current_mode = e1000_fc_none,
627 .requested_mode = e1000_fc_none,
628 };
629
630 error = e1000_reset_hw(hw);
631 atomic_store_rel_32(&sc->vf_mbx_ready, 0);
632 sc->vf_queues_sanitized = igbv_sanitize_queues(sc);
633 if (!sc->vf_queues_sanitized) {
634 return (false);
635 }
636 igbv_queue_retry_succeeded(sc);
637 if (error != E1000_SUCCESS) {
638 igbv_log_reset_failure(sc, error, false);
639 return (false);
640 }
641 memset(sc->vf_vfta_stale, 0, sizeof(sc->vf_vfta_stale));
642 memset(sc->vf_vfta_retry, 0, sizeof(sc->vf_vfta_retry));
643 sc->vf_vlan_retry_deadline = 0;
644 sc->vf_vlan_retry_cursor = 0;
645 if (e1000_init_hw(hw) < 0) {
646 device_printf(sc->dev, "Hardware Initialization Failed\n");
647 return (false);
648 }
649 e1000_check_for_link(hw);
650 igbv_mbx_retry_succeeded(sc);
651 return (true);
652 }
653
654 void
igbv_initialize_transmit_unit(if_ctx_t ctx)655 igbv_initialize_transmit_unit(if_ctx_t ctx)
656 {
657
658 KASSERT(((struct e1000_softc *)iflib_get_softc(ctx))->vf_ifp,
659 ("%s called for a PF", __func__));
660 em_initialize_transmit_rings(ctx);
661 }
662
663 void
igbv_initialize_receive_unit(if_ctx_t ctx)664 igbv_initialize_receive_unit(if_ctx_t ctx)
665 {
666
667 KASSERT(((struct e1000_softc *)iflib_get_softc(ctx))->vf_ifp,
668 ("%s called for a PF", __func__));
669 igb_initialize_receive_rings(ctx, true);
670 }
671
672 void
igbv_if_intr_enable(if_ctx_t ctx)673 igbv_if_intr_enable(if_ctx_t ctx)
674 {
675 struct e1000_softc *sc;
676 struct e1000_hw *hw;
677 u32 mask;
678
679 sc = iflib_get_softc(ctx);
680 hw = &sc->hw;
681 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
682 if (!sc->vf_queues_sanitized ||
683 atomic_load_acq_32(&sc->vf_mbx_ready) == 0)
684 return;
685 mask = sc->que_mask | sc->link_mask;
686
687 E1000_WRITE_REG(hw, E1000_EIAC, mask);
688 E1000_WRITE_REG(hw, E1000_EIAM, mask);
689 E1000_WRITE_REG(hw, E1000_EIMS, mask);
690 E1000_WRITE_FLUSH(hw);
691 }
692
693 void
igbv_if_intr_disable(if_ctx_t ctx)694 igbv_if_intr_disable(if_ctx_t ctx)
695 {
696 struct e1000_softc *sc;
697 struct e1000_hw *hw;
698
699 sc = iflib_get_softc(ctx);
700 hw = &sc->hw;
701 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
702
703 E1000_WRITE_REG(hw, E1000_EIMC, 0xffffffff);
704 E1000_WRITE_REG(hw, E1000_EIAC, 0);
705 E1000_WRITE_FLUSH(hw);
706 }
707
708 int
igbv_get_regs(SYSCTL_HANDLER_ARGS)709 igbv_get_regs(SYSCTL_HANDLER_ARGS)
710 {
711 struct e1000_softc *sc;
712 struct e1000_hw *hw;
713 struct sbuf *sb;
714 int error;
715
716 sc = (struct e1000_softc *)arg1;
717 hw = &sc->hw;
718 KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
719
720 sb = sbuf_new_for_sysctl(NULL, NULL, 512, req);
721 if (sb == NULL)
722 return (ENOMEM);
723
724 /*
725 * Limited VF register set:
726 * Don't read EICR here because it is clear-on-read. The VF register
727 * file exposes its queue pair at index zero, so this diagnostic does
728 * not depend on the narrower lifetime of iflib's queue arrays.
729 */
730 sbuf_printf(sb, "VF Registers\n");
731 sbuf_printf(sb, "\tVTCTRL\t %08x\n",
732 E1000_READ_REG(hw, E1000_CTRL));
733 sbuf_printf(sb, "\tSTATUS\t %08x\n",
734 E1000_READ_REG(hw, E1000_STATUS));
735 sbuf_printf(sb, "\tRDLEN\t %08x\n",
736 E1000_READ_REG(hw, E1000_RDLEN(0)));
737 sbuf_printf(sb, "\tRDH\t %08x\n",
738 E1000_READ_REG(hw, E1000_RDH(0)));
739 sbuf_printf(sb, "\tRDT\t %08x\n",
740 E1000_READ_REG(hw, E1000_RDT(0)));
741 sbuf_printf(sb, "\tTDLEN\t %08x\n",
742 E1000_READ_REG(hw, E1000_TDLEN(0)));
743 sbuf_printf(sb, "\tTDH\t %08x\n",
744 E1000_READ_REG(hw, E1000_TDH(0)));
745 sbuf_printf(sb, "\tTDT\t %08x\n",
746 E1000_READ_REG(hw, E1000_TDT(0)));
747
748 error = sbuf_finish(sb);
749 sbuf_delete(sb);
750 return (error);
751 }
752
753 static u_int
igbv_copy_uc_addr(void * arg,struct sockaddr_dl * sdl,u_int idx)754 igbv_copy_uc_addr(void *arg, struct sockaddr_dl *sdl, u_int idx)
755 {
756 struct igb_vf_uc_addr_list *list;
757 const u8 *addr;
758
759 list = arg;
760 addr = (const u8 *)LLADDR(sdl);
761 if (memcmp(addr, list->sc->hw.mac.addr, ETHER_ADDR_LEN) == 0)
762 return (0);
763 if (idx < IGBV_MAX_MAC_FILTERS)
764 memcpy(list->addrs[idx], addr, ETHER_ADDR_LEN);
765 return (1);
766 }
767
768 void
igbv_update_uc_addr_list(struct e1000_softc * sc,if_t ifp)769 igbv_update_uc_addr_list(struct e1000_softc *sc, if_t ifp)
770 {
771 struct igb_vf_uc_addr_list list = {
772 .sc = sc,
773 };
774 u_int count;
775
776 count = if_foreach_lladdr(ifp, igbv_copy_uc_addr, &list);
777 if (count > IGBV_MAX_MAC_FILTERS) {
778 device_printf(sc->dev,
779 "too many secondary unicast addresses; maximum is %u\n",
780 IGBV_MAX_MAC_FILTERS);
781 }
782 if (count == 0 && !sc->vf_uc_filters_set)
783 return;
784 /*
785 * Linux igb PFs validate the address field before dispatching the CLR
786 * subcommand. Supply the primary address rather than the zero payload
787 * used by igbvf so those PFs actually remove the old filters. FreeBSD
788 * PFs dispatch CLR before inspecting the otherwise-ignored address.
789 */
790 if (e1000_set_uc_addr_vf(&sc->hw, E1000_VF_MAC_FILTER_CLR,
791 sc->hw.mac.addr) != E1000_SUCCESS) {
792 device_printf(sc->dev,
793 "VF secondary unicast filter clear request failed\n");
794 return;
795 }
796 sc->vf_uc_filters_set = false;
797 if (count > IGBV_MAX_MAC_FILTERS)
798 return;
799
800 for (u_int i = 0; i < count; i++) {
801 if (e1000_set_uc_addr_vf(&sc->hw, E1000_VF_MAC_FILTER_ADD,
802 list.addrs[i]) != E1000_SUCCESS) {
803 device_printf(sc->dev,
804 "VF secondary unicast filter add request failed "
805 "for %6D\n", list.addrs[i], ":");
806 } else
807 sc->vf_uc_filters_set = true;
808 usec_delay(200);
809 }
810 }
811
812 void
igbv_reconcile_mac(struct e1000_softc * sc,if_t ifp)813 igbv_reconcile_mac(struct e1000_softc *sc, if_t ifp)
814 {
815 u8 *lladdr;
816
817 if (!em_is_valid_ether_addr(sc->hw.mac.addr))
818 return;
819 lladdr = (u8 *)if_getlladdr(ifp);
820 if (memcmp(lladdr, sc->hw.mac.addr, ETHER_ADDR_LEN) == 0)
821 return;
822
823 device_printf(sc->dev,
824 "PF rejected or replaced the requested MAC; using %6D\n",
825 sc->hw.mac.addr, ":");
826 /*
827 * if_setlladdr() would re-enter the driver's address-change path.
828 * Initialization already holds the context lock, so update the
829 * storage directly and issue the notification it would have sent.
830 */
831 memcpy(lladdr, sc->hw.mac.addr, ETHER_ADDR_LEN);
832
833 CURVNET_SET_QUIET(if_getvnet(ifp));
834 EVENTHANDLER_INVOKE(iflladdr_event, ifp);
835 CURVNET_RESTORE();
836 }
837