1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 2010-2016, Intel Corporation
5 * Copyright (c) 2026 Kevin Bowling <kbowling@FreeBSD.org>
6 */
7
8 #include "if_em.h"
9 #include "if_igb_iov.h"
10
11 #ifdef PCI_IOV
12
13 #include <sys/iov.h>
14 #include <sys/sdt.h>
15 #include <sys/time.h>
16
17 #include <net/if_vf_status.h>
18
19 #define IGB_IOV_RAH_POOLSEL_SHIFT 18
20 #define IGB_IOV_RAH_POOLSEL_MASK (0xffU << IGB_IOV_RAH_POOLSEL_SHIFT)
21 #define IGB_IOV_MAX_MAC_FILTERS 3
22 #define IGB_IOV_MAX_MC_HASHES 30
23 #define IGB_IOV_MBX_RETRY_COUNT 6
24 /* Allow two complete 31-VID replays, then sustain eight additions/second. */
25 #define IGB_IOV_VLAN_CHANGE_BURST 64
26 #define IGB_IOV_VLAN_CHANGE_INTERVAL (SBT_1S / 8)
27 /* 82576 Datasheet rev. 2.0, Section 8.14.16: VMOLR[31] must be one. */
28 #define IGB_82576_VMOLR_RSV (1U << 31)
29 #define IGB_82576_LVMMC_BLOCK_MASK 0x1c
30 #define IGB_82576_NUM_QUEUES 16
31 #define IGB_82576_QUEUE_MASK 0xffff
32 #define IGB_82576_STAGGERED_QUEUE_SHIFT 8
33 #define IGB_82576_VF_QUEUE_STRIDE 8
34 #define IGB_82576_VF_QUEUES 2
35 #define IGB_I350_DTXCTL_ENABLE_SPOOF_QUEUE (1U << 2)
36 #define IGB_I350_LVMMC_MAC_VLAN_SPOOF (1U << 25)
37 #define IGB_I350_LVMMC_LAST_Q_SHIFT 29
38 #define IGB_I350_LVMMC_LAST_Q_MASK 0x7
39 #define IGB_I350_NUM_QUEUES 8
40 #define IGB_I350_QUEUE_MASK 0xff
41 #define IGB_I350_RESET_ACK_TIMEOUT (100 * SBT_1MS)
42 #define IGB_I350_VF_QUEUES 1
43 #define IGB_IOV_QUEUE_DISABLE_BUSY_RETRIES 10
44 #define IGB_IOV_QUEUE_DISABLE_DELAY_US 10
45 #define IGB_IOV_QUEUE_DISABLE_PAUSE (100 * SBT_1US)
46 #define IGB_IOV_QUEUE_DISABLE_RETRIES 20
47 #define IGB_IOV_VF_QUEUES_MAX 2
48
49 #define IGB_VF_CTS (1U << 0)
50 #define IGB_VF_CAP_MAC (1U << 1)
51 #define IGB_VF_ACTIVE (1U << 2)
52 #define IGB_VF_MAC_ANTI_SPOOF (1U << 3)
53 #define IGB_VF_ALLOW_PROMISC (1U << 4)
54 #define IGB_VF_UCAST_PROMISC (1U << 5)
55 #define IGB_VF_MCAST_PROMISC (1U << 6)
56 #define IGB_VF_MCAST_OVERFLOW (1U << 7)
57 #define IGB_VF_MCAST_OVERFLOW_WARNED (1U << 8)
58 #define IGB_VF_MDD_BLOCKED (1U << 9)
59 #define IGB_VF_MBX_PENDING (1U << 10)
60 /*
61 * After bounded PFU retries, suppress future or overlapping VF requests until
62 * RST/VFLR starts a new mailbox epoch. Intel VF drivers assert CTRL.RST
63 * before sending their mailbox reset request.
64 */
65 #define IGB_VF_MBX_GAVE_UP (1U << 11)
66 #define IGB_VF_MDD_NOTIFY_PENDING (1U << 12)
67
68 struct igb_vf {
69 u32 flags;
70 struct timeval last_nack;
71 struct timeval last_mbx_log;
72 struct timeval last_spoof_log;
73 struct timeval last_mdd_log;
74 struct timeval last_queue_log;
75 sbintime_t mbx_retry_at;
76 sbintime_t mdd_notify_at;
77 sbintime_t vlan_token_time;
78 u16 pool;
79 u16 rar_index;
80 u16 max_frame_size;
81 u16 mc_count;
82 u16 vlan_count;
83 u16 vlan_tokens;
84 u16 default_vlan;
85 u8 mbx_retry_count;
86 u8 mac[ETHER_ADDR_LEN];
87 u16 mc_hashes[IGB_IOV_MAX_MC_HASHES];
88 u32 vlans[EM_VFTA_SIZE];
89 };
90
91 struct igb_vf_mac_filter {
92 bool active;
93 u16 pool;
94 u16 rar_index;
95 u8 mac[ETHER_ADDR_LEN];
96 };
97
98 MALLOC_DEFINE(M_IGB_IOV, "igb_iov", "igb SR-IOV allocations");
99
100 /*
101 * These logical-write probes let hardware tests verify the elision policy.
102 * e1000_write_vfta_i350() expands one VFTA call into ten physical writes, so
103 * the probes intentionally count calls made by the rebuild rather than MMIO
104 * transactions. The state probe exposes the final software images while the
105 * stack arrays are still live.
106 */
107 SDT_PROVIDER_DEFINE(igb_iov);
108 SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, vfta_clear,
109 "struct e1000_softc *", "u_int", "uint32_t");
110 SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, vlvf_write,
111 "struct e1000_softc *", "u_int", "uint32_t");
112 SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, vfta_set,
113 "struct e1000_softc *", "u_int", "uint32_t");
114 SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, state,
115 "struct e1000_softc *", "uint32_t *", "uint32_t *");
116 SDT_PROBE_DEFINE4(igb_iov, mdd, sample, wvbr,
117 "struct e1000_softc *", "uint32_t", "uint32_t", "uint32_t");
118
119 static const struct timeval igb_iov_nack_interval = { 2, 0 };
120 static const struct timeval igb_iov_mbx_log_interval = { 2, 0 };
121 static const struct timeval igb_iov_spoof_log_interval = { 2, 0 };
122 static const struct timeval igb_iov_mdd_log_interval = { 2, 0 };
123 static const sbintime_t igb_iov_mdd_notify_retry = SBT_1S / 2;
124 static const sbintime_t igb_iov_mbx_retry_delay[IGB_IOV_MBX_RETRY_COUNT] = {
125 SBT_1MS,
126 2 * SBT_1MS,
127 4 * SBT_1MS,
128 8 * SBT_1MS,
129 16 * SBT_1MS,
130 32 * SBT_1MS,
131 };
132
133 static void igb_iov_clear_mac_filters(struct e1000_softc *,
134 const struct igb_vf *);
135 static bool igb_iov_mac_in_use(struct e1000_softc *, const u8 *,
136 const struct igb_vf *);
137 static bool igb_iov_vlan_present(struct e1000_softc *, u16, bool);
138 static int igb_iov_vlan_unique_count(struct e1000_softc *, bool);
139
140 static void
igb_iov_mbx_retry_callout(void * arg)141 igb_iov_mbx_retry_callout(void *arg)
142 {
143 struct e1000_softc *sc;
144
145 sc = arg;
146 /*
147 * Mailbox service is serialized by iflib's context lock. The
148 * callout only re-enters through the ordinary admin task.
149 */
150 iflib_admin_intr_deferred(sc->ctx);
151 }
152
153 static u_int
igb_iov_copy_maddr(void * arg,struct sockaddr_dl * sdl,u_int idx)154 igb_iov_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int idx)
155 {
156 u8 *mta;
157
158 if (idx == MAX_NUM_MULTICAST_ADDRESSES)
159 return (0);
160 mta = arg;
161 memcpy(&mta[idx * ETHER_ADDR_LEN], LLADDR(sdl), ETHER_ADDR_LEN);
162 return (1);
163 }
164
165 static bool
igb_iov_pf_vlan_promisc(struct e1000_softc * sc)166 igb_iov_pf_vlan_promisc(struct e1000_softc *sc)
167 {
168 if_t ifp;
169
170 ifp = iflib_get_ifp(sc->ctx);
171 return (sc->iov_pf_vlan_promisc ||
172 (if_getflags(ifp) & IFF_PROMISC) != 0);
173 }
174
175 static bool
igb_iov_mac_valid(const u8 * mac)176 igb_iov_mac_valid(const u8 *mac)
177 {
178 static const u8 zero[ETHER_ADDR_LEN];
179
180 return (!ETHER_IS_MULTICAST(mac) &&
181 memcmp(mac, zero, ETHER_ADDR_LEN) != 0);
182 }
183
184 static bool
igb_iov_nack_allowed(struct igb_vf * vf)185 igb_iov_nack_allowed(struct igb_vf *vf)
186 {
187 return (ratecheck(&vf->last_nack, &igb_iov_nack_interval) != 0);
188 }
189
190 static void
igb_iov_reset_vlan_rate(struct igb_vf * vf)191 igb_iov_reset_vlan_rate(struct igb_vf *vf)
192 {
193
194 vf->vlan_token_time = getsbinuptime();
195 vf->vlan_tokens = IGB_IOV_VLAN_CHANGE_BURST;
196 }
197
198 static bool
igb_iov_vlan_add_allowed(struct igb_vf * vf)199 igb_iov_vlan_add_allowed(struct igb_vf *vf)
200 {
201 sbintime_t elapsed, now;
202 uint64_t refill;
203
204 now = getsbinuptime();
205 elapsed = now - vf->vlan_token_time;
206 if (elapsed >= IGB_IOV_VLAN_CHANGE_INTERVAL) {
207 refill = elapsed / IGB_IOV_VLAN_CHANGE_INTERVAL;
208 vf->vlan_tokens = min((uint64_t)IGB_IOV_VLAN_CHANGE_BURST,
209 vf->vlan_tokens + refill);
210 vf->vlan_token_time = now;
211 }
212 if (vf->vlan_tokens == 0)
213 return (false);
214 vf->vlan_tokens--;
215 return (true);
216 }
217
218 static u32
igb_iov_reply_header(u32 request,bool cts,bool ack)219 igb_iov_reply_header(u32 request, bool cts, bool ack)
220 {
221 u32 reply, type;
222
223 type = request & 0xffff;
224 if (type == E1000_VF_SET_MAC_ADDR &&
225 (request & E1000_VT_MSGINFO_MASK) != 0)
226 reply = request;
227 else
228 reply = type;
229 reply &= ~(E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK |
230 E1000_VT_MSGTYPE_CTS);
231 if (cts)
232 reply |= E1000_VT_MSGTYPE_CTS;
233 reply |= ack ? E1000_VT_MSGTYPE_ACK : E1000_VT_MSGTYPE_NACK;
234 return (reply);
235 }
236
237 bool
igb_iov_supported(const struct e1000_softc * sc)238 igb_iov_supported(const struct e1000_softc *sc)
239 {
240 switch (sc->hw.mac.type) {
241 case e1000_82576:
242 case e1000_i350:
243 return (true);
244 default:
245 return (false);
246 }
247 }
248
249 bool
igb_iov_enabled(const struct e1000_softc * sc)250 igb_iov_enabled(const struct e1000_softc *sc)
251 {
252 return (sc->num_vfs != 0);
253 }
254
255 int
igb_iov_attach(struct e1000_softc * sc)256 igb_iov_attach(struct e1000_softc *sc)
257 {
258 nvlist_t *pf_schema, *vf_schema;
259 int error, iov_pos;
260
261 if (!igb_iov_supported(sc))
262 return (0);
263 if (pci_find_extcap(sc->dev, PCIZ_SRIOV, &iov_pos) != 0)
264 return (0);
265
266 pf_schema = pci_iov_schema_alloc_node();
267 vf_schema = pci_iov_schema_alloc_node();
268 pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL);
269 pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof",
270 IOV_SCHEMA_HASDEFAULT, true);
271 pci_iov_schema_add_bool(vf_schema, "allow-set-mac",
272 IOV_SCHEMA_HASDEFAULT, false);
273 pci_iov_schema_add_bool(vf_schema, "allow-promisc",
274 IOV_SCHEMA_HASDEFAULT, false);
275 pci_iov_schema_add_vlan(vf_schema, "vlan", IOV_SCHEMA_HASDEFAULT,
276 VF_VLAN_TRUNK);
277
278 error = pci_iov_attach(sc->dev, pf_schema, vf_schema);
279 if (error != 0)
280 device_printf(sc->dev,
281 "failed to attach SR-IOV configuration interface: %d\n",
282 error);
283 else {
284 callout_init(&sc->iov_mbx_retry, 1);
285 sc->iov_mbx_retry_initialized = true;
286 }
287 return (error);
288 }
289
290 void
igb_iov_detach(struct e1000_softc * sc)291 igb_iov_detach(struct e1000_softc *sc)
292 {
293
294 if (!sc->iov_mbx_retry_initialized)
295 return;
296 callout_drain(&sc->iov_mbx_retry);
297 sc->iov_mbx_retry_initialized = false;
298 }
299
300 static u32
igb_iov_active_mask(struct e1000_softc * sc)301 igb_iov_active_mask(struct e1000_softc *sc)
302 {
303 u32 mask;
304 int i;
305
306 mask = 0;
307 for (i = 0; i < sc->num_vfs; i++)
308 if (sc->vfs[i].flags & IGB_VF_ACTIVE)
309 mask |= 1U << i;
310 return (mask);
311 }
312
313 static void
igb_iov_map_rar(struct e1000_softc * sc,u16 rar,const u8 * mac,u16 pool)314 igb_iov_map_rar(struct e1000_softc *sc, u16 rar, const u8 *mac, u16 pool)
315 {
316 struct e1000_hw *hw;
317 u32 rah;
318
319 hw = &sc->hw;
320 e1000_rar_set(hw, __DECONST(u8 *, mac), rar);
321 rah = E1000_READ_REG(hw, E1000_RAH(rar));
322 rah &= ~IGB_IOV_RAH_POOLSEL_MASK;
323 rah |= 1U << (IGB_IOV_RAH_POOLSEL_SHIFT + pool);
324 E1000_WRITE_REG(hw, E1000_RAH(rar), rah);
325 }
326
327 static void
igb_iov_clear_rar(struct e1000_softc * sc,u16 rar)328 igb_iov_clear_rar(struct e1000_softc *sc, u16 rar)
329 {
330 u8 zero[ETHER_ADDR_LEN] = {};
331
332 e1000_rar_set(&sc->hw, zero, rar);
333 }
334
335 static void
igb_iov_clear_mac_filters(struct e1000_softc * sc,const struct igb_vf * vf)336 igb_iov_clear_mac_filters(struct e1000_softc *sc, const struct igb_vf *vf)
337 {
338 struct igb_vf_mac_filter *filter;
339 int i;
340
341 for (i = 0; i < sc->num_vf_mac_filters; i++) {
342 filter = &sc->vf_mac_filters[i];
343 if (!filter->active || filter->pool != vf->pool)
344 continue;
345 igb_iov_clear_rar(sc, filter->rar_index);
346 filter->active = false;
347 memset(filter->mac, 0, sizeof(filter->mac));
348 }
349 }
350
351 static u32
igb_iov_switch_reg(struct e1000_softc * sc)352 igb_iov_switch_reg(struct e1000_softc *sc)
353 {
354 return (sc->hw.mac.type == e1000_82576 ?
355 E1000_DTXSWC : E1000_TXSWC);
356 }
357
358 static void
igb_iov_set_anti_spoof(struct e1000_softc * sc,struct igb_vf * vf)359 igb_iov_set_anti_spoof(struct e1000_softc *sc, struct igb_vf *vf)
360 {
361 struct e1000_hw *hw;
362 u32 reg, value;
363
364 hw = &sc->hw;
365 reg = igb_iov_switch_reg(sc);
366 value = E1000_READ_REG(hw, reg);
367 value &= ~((1U << vf->pool) |
368 (1U << (vf->pool + E1000_DTXSWC_VLAN_SPOOF_SHIFT)));
369 if (vf->flags & IGB_VF_MAC_ANTI_SPOOF)
370 value |= 1U << vf->pool;
371 if (vf->flags & IGB_VF_ACTIVE)
372 value |= 1U <<
373 (vf->pool + E1000_DTXSWC_VLAN_SPOOF_SHIFT);
374 E1000_WRITE_REG(hw, reg, value);
375 }
376
377 static void
igb_iov_set_uta(struct e1000_softc * sc)378 igb_iov_set_uta(struct e1000_softc *sc)
379 {
380 struct e1000_hw *hw;
381 bool enable;
382 int i;
383
384 if (!igb_iov_enabled(sc) || sc->hw.mac.type != e1000_82576)
385 return;
386
387 hw = &sc->hw;
388 enable = (E1000_READ_REG(hw, E1000_VMOLR(sc->pool)) &
389 E1000_VMOLR_ROPE) != 0;
390 for (i = 0; i < sc->num_vfs; i++)
391 if ((sc->vfs[i].flags &
392 (IGB_VF_ACTIVE | IGB_VF_UCAST_PROMISC)) ==
393 (IGB_VF_ACTIVE | IGB_VF_UCAST_PROMISC)) {
394 enable = true;
395 break;
396 }
397
398 for (i = 0; i < MAX_MTA_REG; i++)
399 E1000_WRITE_REG_ARRAY(hw, E1000_UTA, i,
400 enable ? 0xffffffffU : 0);
401 }
402
403 static void
igb_iov_configure_dvmolr(struct e1000_softc * sc,u16 pool,bool strip_vlan,bool hide_vlan,bool vf_pool)404 igb_iov_configure_dvmolr(struct e1000_softc *sc, u16 pool,
405 bool strip_vlan, bool hide_vlan, bool vf_pool)
406 {
407 struct e1000_hw *hw;
408 u32 dvmolr;
409
410 hw = &sc->hw;
411 if (hw->mac.type != e1000_i350)
412 return;
413
414 dvmolr = E1000_READ_REG(hw, E1000_DVMOLR(pool));
415 dvmolr &= ~(E1000_DVMOLR_HIDVLAN | E1000_DVMOLR_STRVLAN |
416 E1000_DVMOLR_STRCRC);
417 if (hide_vlan)
418 dvmolr |= E1000_DVMOLR_HIDVLAN;
419 if (strip_vlan)
420 dvmolr |= E1000_DVMOLR_STRVLAN;
421 if (vf_pool || strip_vlan ||
422 (E1000_READ_REG(hw, E1000_RCTL) & E1000_RCTL_SECRC) != 0)
423 dvmolr |= E1000_DVMOLR_STRCRC;
424 E1000_WRITE_REG(hw, E1000_DVMOLR(pool), dvmolr);
425 }
426
427 static void
igb_iov_configure_vmolr(struct e1000_softc * sc,struct igb_vf * vf)428 igb_iov_configure_vmolr(struct e1000_softc *sc, struct igb_vf *vf)
429 {
430 struct e1000_hw *hw;
431 u32 max_frame_size, vmolr, vmvir;
432
433 hw = &sc->hw;
434 max_frame_size = vf->max_frame_size;
435 if (vf->vlan_count != 0)
436 max_frame_size = min(max_frame_size + VLAN_TAG_SIZE,
437 IGB_IOV_MAX_FRAME_SIZE);
438 vmolr = E1000_READ_REG(hw, E1000_VMOLR(vf->pool));
439 vmolr &= ~(E1000_VMOLR_RLPML_MASK | E1000_VMOLR_RSSE |
440 E1000_VMOLR_VPE | E1000_VMOLR_UPE | E1000_VMOLR_ROMPE |
441 E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_STRVLAN);
442 vmolr |= E1000_VMOLR_BAM | E1000_VMOLR_LPE |
443 (max_frame_size & E1000_VMOLR_RLPML_MASK);
444 if (vf->default_vlan == 0)
445 vmolr |= E1000_VMOLR_AUPE;
446 if (vf->mc_count != 0 &&
447 (vf->flags & (IGB_VF_MCAST_PROMISC |
448 IGB_VF_MCAST_OVERFLOW)) == 0)
449 vmolr |= E1000_VMOLR_ROMPE;
450 if (hw->mac.type == e1000_82576)
451 vmolr |= IGB_82576_VMOLR_RSV;
452
453 if (vf->flags & IGB_VF_UCAST_PROMISC) {
454 if (hw->mac.type == e1000_82576)
455 vmolr |= E1000_VMOLR_ROPE;
456 else
457 vmolr |= E1000_VMOLR_UPE;
458 }
459 /*
460 * The mailbox can describe only 30 hashes. Fall back to receiving all
461 * multicast within the VF's VLAN membership when that list overflows.
462 */
463 if ((vf->flags & (IGB_VF_MCAST_PROMISC |
464 IGB_VF_MCAST_OVERFLOW)) != 0)
465 vmolr |= E1000_VMOLR_MPME;
466 if (hw->mac.type == e1000_82576 && vf->vlan_count != 0)
467 vmolr |= E1000_VMOLR_STRVLAN;
468 /* A nonzero default VLAN makes this VF an untagged access port. */
469 if (vf->default_vlan == 0)
470 vmvir = 0;
471 else
472 vmvir = vf->default_vlan | E1000_VMVIR_VLANA_DEFAULT;
473
474 E1000_WRITE_REG(hw, E1000_VMOLR(vf->pool), vmolr);
475 E1000_WRITE_REG(hw, E1000_VMVIR(vf->pool), vmvir);
476 igb_iov_configure_dvmolr(sc, vf->pool, vf->vlan_count != 0,
477 vf->default_vlan != 0, true);
478 }
479
480 static void
igb_iov_configure_pf_vmolr(struct e1000_softc * sc)481 igb_iov_configure_pf_vmolr(struct e1000_softc *sc)
482 {
483 struct e1000_hw *hw;
484 if_t ifp;
485 bool strip_vlan;
486 u32 max_frame_size;
487 u32 old_vmolr, vmolr;
488
489 hw = &sc->hw;
490 ifp = iflib_get_ifp(sc->ctx);
491 max_frame_size = min(sc->shared->isc_max_frame_size + VLAN_TAG_SIZE,
492 IGB_IOV_MAX_FRAME_SIZE);
493 strip_vlan = (E1000_READ_REG(hw, E1000_CTRL) & E1000_CTRL_VME) != 0;
494 old_vmolr = E1000_READ_REG(hw, E1000_VMOLR(sc->pool));
495 vmolr = E1000_VMOLR_BAM | E1000_VMOLR_AUPE |
496 E1000_VMOLR_LPE |
497 (max_frame_size & E1000_VMOLR_RLPML_MASK);
498 if (hw->mac.type == e1000_82576) {
499 vmolr |= IGB_82576_VMOLR_RSV;
500 if (strip_vlan)
501 vmolr |= E1000_VMOLR_STRVLAN;
502 } else
503 vmolr |= old_vmolr & E1000_VMOLR_VPE;
504
505 if (if_getflags(ifp) & IFF_PROMISC) {
506 if (hw->mac.type == e1000_82576)
507 vmolr |= E1000_VMOLR_ROPE;
508 else
509 vmolr |= E1000_VMOLR_UPE | E1000_VMOLR_VPE;
510 vmolr |= E1000_VMOLR_MPME;
511 } else if ((if_getflags(ifp) & IFF_ALLMULTI) != 0 ||
512 if_llmaddr_count(ifp) >= MAX_NUM_MULTICAST_ADDRESSES)
513 vmolr |= E1000_VMOLR_MPME;
514 else if (if_llmaddr_count(ifp) != 0)
515 vmolr |= E1000_VMOLR_ROMPE;
516
517 E1000_WRITE_REG(hw, E1000_VMOLR(sc->pool), vmolr);
518 igb_iov_configure_dvmolr(sc, sc->pool, strip_vlan, false, false);
519 }
520
521 void
igb_iov_update_pf_vmolr(struct e1000_softc * sc)522 igb_iov_update_pf_vmolr(struct e1000_softc *sc)
523 {
524 if (!igb_iov_enabled(sc))
525 return;
526
527 igb_iov_configure_pf_vmolr(sc);
528 igb_iov_set_uta(sc);
529 }
530
531 u32
igb_iov_intr_mask(const struct e1000_softc * sc)532 igb_iov_intr_mask(const struct e1000_softc *sc)
533 {
534 if (!sc->iov_hw_active)
535 return (0);
536 return (E1000_IMS_VMMB | E1000_IMS_MDDET);
537 }
538
539 void
igb_iov_intr_drain_stale(struct e1000_softc * sc)540 igb_iov_intr_drain_stale(struct e1000_softc *sc)
541 {
542 struct e1000_hw *hw;
543 u32 icr;
544
545 if (atomic_readandclear_32(&sc->iov_intr_drain_pending) == 0)
546 return;
547 hw = &sc->hw;
548 /*
549 * Consume setup-time diagnostic state at the actual transition from
550 * masked to armed. Read ICR last so an event arriving after the drain
551 * remains pending and is delivered when the caller enables MDDET.
552 */
553 (void)E1000_READ_REG(hw, E1000_LVMMC);
554 if (hw->mac.type == e1000_82576)
555 (void)E1000_READ_REG(hw, E1000_WVBR);
556 icr = E1000_READ_REG(hw, E1000_ICR);
557 /*
558 * em_if_init() injects LSC after IOV setup to close the post-reset
559 * link race. Preserve that cause across this MDDET-specific drain.
560 */
561 if (__predict_true(icr != 0xffffffff) &&
562 (icr & E1000_ICR_LSC) != 0)
563 E1000_WRITE_REG(hw, E1000_ICS, E1000_ICS_LSC);
564 }
565
566 static void
igb_iov_vfta_shadow_invalidate(struct e1000_softc * sc)567 igb_iov_vfta_shadow_invalidate(struct e1000_softc *sc)
568 {
569
570 /*
571 * I350 erratum 20 makes VFTA reads unreliable while VMDq loopback or
572 * anti-spoofing is active. The shadow is therefore authoritative
573 * until a reset or another independent hardware writer invalidates
574 * it. Readback cannot reliably audit a stale-but-valid shadow on
575 * this part, so keep all shadow mutation in these two helpers.
576 */
577 memset(sc->iov_vfta, 0, sizeof(sc->iov_vfta));
578 sc->iov_vfta_valid = false;
579 }
580
581 static void
igb_iov_vfta_shadow_store(struct e1000_softc * sc,const u32 * vfta)582 igb_iov_vfta_shadow_store(struct e1000_softc *sc, const u32 *vfta)
583 {
584
585 memcpy(sc->iov_vfta, vfta, sizeof(sc->iov_vfta));
586 sc->iov_vfta_valid = true;
587 }
588
589 static void
igb_iov_notify_vfs_reset(struct e1000_softc * sc)590 igb_iov_notify_vfs_reset(struct e1000_softc *sc)
591 {
592 struct igb_vf *vf;
593 struct e1000_hw *hw;
594 sbintime_t deadline;
595 u32 msg, pending, undelivered;
596 int i;
597
598 hw = &sc->hw;
599 /*
600 * Process VFLRs first and wait only for VFs that completed their
601 * mailbox handshake. An unattached VF has nobody who can acknowledge.
602 */
603 igb_iov_handle_mbx(sc);
604 pending = 0;
605 for (i = 0; i < sc->num_vfs; i++) {
606 vf = &sc->vfs[i];
607 if ((vf->flags & (IGB_VF_ACTIVE | IGB_VF_CTS)) ==
608 (IGB_VF_ACTIVE | IGB_VF_CTS))
609 pending |= 1U << i;
610 }
611 if (pending == 0)
612 return;
613
614 /*
615 * I350 SDM section 4.6.11.2.3 requires each VF to acknowledge a
616 * mailbox warning before the PF asserts CTRL.RST.
617 *
618 * The mailbox pass above drained requests and stale acknowledgements.
619 * A VF read of the new notification sets its ACK bit.
620 */
621 undelivered = 0;
622 for (i = 0; i < sc->num_vfs; i++) {
623 if ((pending & (1U << i)) == 0)
624 continue;
625 msg = E1000_PF_CONTROL_MSG;
626 if (e1000_write_mbx(hw, &msg, 1, i) != 0) {
627 undelivered |= 1U << i;
628 pending &= ~(1U << i);
629 }
630 }
631 if (undelivered != 0)
632 device_printf(sc->dev,
633 "could not deliver reset warning to VF mask %#x\n",
634 undelivered);
635
636 deadline = getsbinuptime() + IGB_I350_RESET_ACK_TIMEOUT;
637 while (pending != 0 && getsbinuptime() < deadline) {
638 for (i = 0; i < sc->num_vfs; i++) {
639 if ((pending & (1U << i)) != 0 &&
640 e1000_check_for_ack(hw, i) == 0)
641 pending &= ~(1U << i);
642 }
643 if (pending != 0)
644 pause_sbt("igback", SBT_1MS, 0, C_HARDCLOCK);
645 }
646 if (pending != 0)
647 device_printf(sc->dev,
648 "VF reset acknowledgement timed out for mask %#x\n",
649 pending);
650 }
651
652 void
igb_iov_reset_prepare(struct e1000_softc * sc)653 igb_iov_reset_prepare(struct e1000_softc *sc)
654 {
655 struct e1000_hw *hw;
656 u32 mask;
657
658 if (sc->iov_hw_active) {
659 hw = &sc->hw;
660 if (atomic_load_acq_32(&sc->iov_teardown) == 0) {
661 if (hw->mac.type == e1000_i350)
662 igb_iov_notify_vfs_reset(sc);
663 else
664 igb_iov_ping_all_vfs(sc);
665 }
666
667 /* Stop VF DMA before the PF asserts CTRL.RST. */
668 mask = 1U << sc->pool;
669 E1000_WRITE_REG(hw, E1000_VFRE, mask);
670 E1000_WRITE_REG(hw, E1000_VFTE, mask);
671 E1000_WRITE_FLUSH(hw);
672 }
673 sc->iov_hw_active = false;
674 if (sc->iov_mbx_retry_initialized)
675 callout_stop(&sc->iov_mbx_retry);
676 sc->iov_mta_valid = false;
677 igb_iov_vfta_shadow_invalidate(sc);
678 atomic_readandclear_32(&sc->iov_mdd_cause);
679 atomic_readandclear_32(&sc->iov_pending);
680 atomic_readandclear_32(&sc->iov_spoof_pending);
681 atomic_readandclear_32(&sc->iov_blocked_pending);
682 /*
683 * Normal iflib initialization prepares the reset before
684 * igb_iov_initialize() requests this drain. Preserve a still-pending
685 * I350 request across a later stop or repeated preparation so the next
686 * interrupt arm consumes it. Other families retain the ordinary
687 * stop-time cleanup.
688 */
689 if (sc->hw.mac.type != e1000_i350)
690 atomic_readandclear_32(&sc->iov_intr_drain_pending);
691 }
692
693 void
igb_iov_rebuild_mta(struct e1000_softc * sc)694 igb_iov_rebuild_mta(struct e1000_softc *sc)
695 {
696 struct e1000_hw *hw;
697 struct igb_vf *vf;
698 u32 hash_bit, hash_reg, hash_value;
699 u32 mta[MAX_MTA_REG] = {};
700 u16 hash;
701 bool changed;
702 int i, j, mcnt;
703
704 if (!igb_iov_enabled(sc))
705 return;
706
707 hw = &sc->hw;
708 memset(sc->mta, 0,
709 ETHER_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES);
710 mcnt = if_foreach_llmaddr(iflib_get_ifp(sc->ctx),
711 igb_iov_copy_maddr, sc->mta);
712 mcnt = min(mcnt, MAX_NUM_MULTICAST_ADDRESSES);
713 for (i = 0; i < mcnt; i++) {
714 hash_value = e1000_hash_mc_addr(hw,
715 &sc->mta[i * ETHER_ADDR_LEN]);
716 hash_reg = (hash_value >> 5) &
717 (hw->mac.mta_reg_count - 1);
718 hash_bit = hash_value & 0x1f;
719 mta[hash_reg] |= 1U << hash_bit;
720 }
721 for (i = 0; i < sc->num_vfs; i++) {
722 vf = &sc->vfs[i];
723 if (!(vf->flags & IGB_VF_ACTIVE))
724 continue;
725 for (j = 0; j < vf->mc_count; j++) {
726 hash = vf->mc_hashes[j] & 0xfff;
727 mta[(hash >> 5) & (hw->mac.mta_reg_count - 1)] |=
728 1U << (hash & 0x1f);
729 }
730 }
731
732 changed = false;
733 for (i = hw->mac.mta_reg_count - 1; i >= 0; i--) {
734 if (sc->iov_mta_valid && hw->mac.mta_shadow[i] == mta[i])
735 continue;
736 hw->mac.mta_shadow[i] = mta[i];
737 E1000_WRITE_REG_ARRAY(hw, E1000_MTA, i, mta[i]);
738 changed = true;
739 }
740 if (changed)
741 E1000_WRITE_FLUSH(hw);
742 sc->iov_mta_valid = true;
743 }
744
745 static int
igb_iov_vlvf_add(u32 * vlvf,const u32 * old_vlvf,u16 vid,u16 pool,bool preserve_only)746 igb_iov_vlvf_add(u32 *vlvf, const u32 *old_vlvf, u16 vid, u16 pool,
747 bool preserve_only)
748 {
749 int free_slot, i;
750
751 free_slot = -1;
752 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
753 if ((vlvf[i] & E1000_VLVF_VLANID_ENABLE) != 0 &&
754 (vlvf[i] & E1000_VLVF_VLANID_MASK) == vid) {
755 vlvf[i] |= 1U << (E1000_VLVF_POOLSEL_SHIFT + pool);
756 return (0);
757 }
758 if (free_slot == -1 &&
759 (vlvf[i] & E1000_VLVF_VLANID_ENABLE) == 0)
760 free_slot = i;
761 }
762 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
763 if ((old_vlvf[i] & E1000_VLVF_VLANID_ENABLE) != 0 &&
764 (old_vlvf[i] & E1000_VLVF_VLANID_MASK) == vid &&
765 (vlvf[i] & E1000_VLVF_VLANID_ENABLE) == 0) {
766 free_slot = i;
767 break;
768 }
769 if (preserve_only && (i == E1000_VLVF_ARRAY_SIZE))
770 return (ENOENT);
771 if (free_slot == -1)
772 return (ENOSPC);
773
774 vlvf[free_slot] = E1000_VLVF_VLANID_ENABLE | vid |
775 (1U << (E1000_VLVF_POOLSEL_SHIFT + pool));
776 return (0);
777 }
778
779 void
igb_iov_rebuild_vlan(struct e1000_softc * sc)780 igb_iov_rebuild_vlan(struct e1000_softc *sc)
781 {
782 struct e1000_hw *hw;
783 struct igb_vf *vf;
784 u32 old_vlvf[E1000_VLVF_ARRAY_SIZE];
785 u32 effective_vfta[EM_VFTA_SIZE], vfta[EM_VFTA_SIZE];
786 u32 vlvf[E1000_VLVF_ARRAY_SIZE];
787 u32 old_vfta, rctl, vmolr;
788 bool force_vfta, pf_overflow, pf_vlan_promisc, preserve_pf;
789 bool vfta_changed, vlvf_changed;
790 int i, vid;
791
792 if (!igb_iov_enabled(sc))
793 return;
794
795 hw = &sc->hw;
796 rctl = E1000_READ_REG(hw, E1000_RCTL);
797 rctl &= ~E1000_RCTL_CFIEN;
798 rctl |= E1000_RCTL_VFE;
799 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
800 memcpy(vfta, sc->shadow_vfta, sizeof(vfta));
801 memset(vlvf, 0, sizeof(vlvf));
802 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
803 old_vlvf[i] = E1000_READ_REG(hw, E1000_VLVF(i));
804
805 pf_vlan_promisc = igb_iov_pf_vlan_promisc(sc);
806 pf_overflow = !pf_vlan_promisc && hw->mac.type == e1000_i350 &&
807 igb_iov_vlan_unique_count(sc, true) > E1000_VLVF_ARRAY_SIZE;
808 preserve_pf = !pf_vlan_promisc && !pf_overflow;
809
810 /* First keep every surviving VF mapping in its current slot. */
811 for (i = 0; i < sc->num_vfs; i++) {
812 vf = &sc->vfs[i];
813 if (!(vf->flags & IGB_VF_ACTIVE))
814 continue;
815 for (vid = 0; vid < 4096; vid++) {
816 if ((vf->vlans[vid >> 5] & (1U << (vid & 0x1f))) ==
817 0)
818 continue;
819 (void)igb_iov_vlvf_add(vlvf, old_vlvf, vid,
820 vf->pool, true);
821 }
822 }
823
824 /*
825 * Preserve PF mappings unless I350 needs their slots for VFs.
826 * PF-only VLANs on 82576 intentionally have no VLVF mapping and
827 * reach the default PF pool after passing the global VFTA.
828 */
829 if (preserve_pf)
830 for (vid = 0; vid < 4096; vid++) {
831 if ((sc->shadow_vfta[vid >> 5] &
832 (1U << (vid & 0x1f))) == 0)
833 continue;
834 if (hw->mac.type == e1000_82576 &&
835 !igb_iov_vlan_present(sc, vid, false))
836 continue;
837 (void)igb_iov_vlvf_add(vlvf, old_vlvf, vid,
838 sc->pool, true);
839 }
840
841 /* Allocate new VF mappings before PF mappings. */
842 for (i = 0; i < sc->num_vfs; i++) {
843 vf = &sc->vfs[i];
844 if (!(vf->flags & IGB_VF_ACTIVE))
845 continue;
846 for (vid = 0; vid < 4096; vid++) {
847 if ((vf->vlans[vid >> 5] & (1U << (vid & 0x1f))) ==
848 0)
849 continue;
850 if (igb_iov_vlvf_add(vlvf, old_vlvf, vid,
851 vf->pool, false) == 0)
852 vfta[vid >> 5] |= 1U << (vid & 0x1f);
853 }
854 igb_iov_configure_vmolr(sc, vf);
855 }
856 if (!pf_vlan_promisc)
857 for (vid = 0; vid < 4096; vid++) {
858 if ((sc->shadow_vfta[vid >> 5] &
859 (1U << (vid & 0x1f))) == 0)
860 continue;
861 /*
862 * With no VLVF match, 82576 sends a globally admitted
863 * VLAN to the default PF pool. A VLVF entry is needed
864 * only when this VLAN is also assigned to a VF.
865 */
866 if (hw->mac.type == e1000_82576 &&
867 !igb_iov_vlan_present(sc, vid, false))
868 continue;
869 if (igb_iov_vlvf_add(vlvf, old_vlvf, vid,
870 sc->pool, false) != 0)
871 pf_overflow = true;
872 }
873
874 if (pf_vlan_promisc) {
875 memset(vfta, 0xff, sizeof(vfta));
876 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
877 if ((vlvf[i] & E1000_VLVF_VLANID_ENABLE) != 0)
878 vlvf[i] |= 1U <<
879 (E1000_VLVF_POOLSEL_SHIFT + sc->pool);
880 }
881
882 /*
883 * Establish the PF fallback before an overflowing I350 rebuild can
884 * displace one of its old VLVF mappings.
885 */
886 vmolr = E1000_READ_REG(hw, E1000_VMOLR(sc->pool));
887 vmolr &= ~E1000_VMOLR_VPE;
888 if (hw->mac.type == e1000_i350 &&
889 (pf_overflow || pf_vlan_promisc))
890 vmolr |= E1000_VMOLR_VPE;
891 E1000_WRITE_REG(hw, E1000_VMOLR(sc->pool), vmolr);
892
893 /*
894 * Remove global VFTA membership before removing a VLAN entirely, and
895 * add a VLVF mapping before globally admitting a new VF VLAN. A
896 * transition to a PF-only VLAN deliberately retains VFTA membership
897 * and falls through to the default PF pool.
898 */
899 force_vfta = hw->mac.type == e1000_i350 &&
900 !sc->iov_vfta_valid;
901 vfta_changed = false;
902 for (i = 0; i < EM_VFTA_SIZE; i++) {
903 /*
904 * I350 erratum 20 makes VFTA reads unreliable while VMDq
905 * loopback or anti-spoofing is active. Its ten-write
906 * workaround is already in e1000_write_vfta_i350(). Force a
907 * complete clear when the authoritative shadow is invalid;
908 * 82576 can safely diff against its live register contents.
909 */
910 if (hw->mac.type == e1000_i350)
911 old_vfta = force_vfta ? 0 : sc->iov_vfta[i];
912 else
913 old_vfta =
914 E1000_READ_REG_ARRAY(hw, E1000_VFTA, i);
915 effective_vfta[i] = old_vfta & vfta[i];
916 if (force_vfta || effective_vfta[i] != old_vfta) {
917 SDT_PROBE3(igb_iov, vlan, rebuild, vfta_clear,
918 sc, i, effective_vfta[i]);
919 e1000_write_vfta(hw, i, effective_vfta[i]);
920 vfta_changed = true;
921 }
922 }
923 if (vfta_changed)
924 E1000_WRITE_FLUSH(hw);
925 vlvf_changed = false;
926 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
927 if (vlvf[i] != old_vlvf[i]) {
928 SDT_PROBE3(igb_iov, vlan, rebuild, vlvf_write,
929 sc, i, vlvf[i]);
930 E1000_WRITE_REG(hw, E1000_VLVF(i), vlvf[i]);
931 vlvf_changed = true;
932 }
933 if (vlvf_changed)
934 E1000_WRITE_FLUSH(hw);
935 vfta_changed = false;
936 for (i = 0; i < EM_VFTA_SIZE; i++)
937 if (vfta[i] != effective_vfta[i]) {
938 SDT_PROBE3(igb_iov, vlan, rebuild, vfta_set,
939 sc, i, vfta[i]);
940 e1000_write_vfta(hw, i, vfta[i]);
941 vfta_changed = true;
942 }
943 if (vfta_changed)
944 E1000_WRITE_FLUSH(hw);
945 SDT_PROBE3(igb_iov, vlan, rebuild, state, sc, vfta, vlvf);
946 igb_iov_vfta_shadow_store(sc, vfta);
947 }
948
949 static bool
igb_iov_vlan_present(struct e1000_softc * sc,u16 vid,bool include_pf)950 igb_iov_vlan_present(struct e1000_softc *sc, u16 vid, bool include_pf)
951 {
952 int i;
953
954 if (include_pf &&
955 (sc->shadow_vfta[vid >> 5] & (1U << (vid & 0x1f))) != 0)
956 return (true);
957 for (i = 0; i < sc->num_vfs; i++)
958 if ((sc->vfs[i].flags & IGB_VF_ACTIVE) != 0 &&
959 (sc->vfs[i].vlans[vid >> 5] &
960 (1U << (vid & 0x1f))) != 0)
961 return (true);
962 return (false);
963 }
964
965 static int
igb_iov_vlan_unique_count(struct e1000_softc * sc,bool include_pf)966 igb_iov_vlan_unique_count(struct e1000_softc *sc, bool include_pf)
967 {
968 u32 vlans;
969 int count, i, word;
970
971 count = 0;
972 for (word = 0; word < EM_VFTA_SIZE; word++) {
973 vlans = include_pf ? sc->shadow_vfta[word] : 0;
974 for (i = 0; i < sc->num_vfs; i++)
975 if ((sc->vfs[i].flags & IGB_VF_ACTIVE) != 0)
976 vlans |= sc->vfs[i].vlans[word];
977 count += bitcount32(vlans);
978 }
979 return (count);
980 }
981
982 static int
igb_iov_set_vlan(struct e1000_softc * sc,struct igb_vf * vf,u16 vid,bool add)983 igb_iov_set_vlan(struct e1000_softc *sc, struct igb_vf *vf, u16 vid,
984 bool add)
985 {
986 u32 bit;
987 bool present;
988
989 bit = 1U << (vid & 0x1f);
990 present = (vf->vlans[vid >> 5] & bit) != 0;
991 if (vid == 0) {
992 if (!present) {
993 vf->vlans[0] |= 1U;
994 igb_iov_rebuild_vlan(sc);
995 }
996 return (0);
997 }
998 if (add == present)
999 return (0);
1000
1001 /*
1002 * Removals always reduce privilege and remain available. Charge only
1003 * additions, which a hostile VF must alternate with removals to force
1004 * repeated global VLAN rebuilds.
1005 */
1006 if (add && !igb_iov_vlan_present(sc, vid, false) &&
1007 igb_iov_vlan_unique_count(sc, false) >=
1008 E1000_VLVF_ARRAY_SIZE)
1009 return (ENOSPC);
1010 if (add && !igb_iov_vlan_add_allowed(vf))
1011 return (EBUSY);
1012
1013 if (add) {
1014 vf->vlans[vid >> 5] |= bit;
1015 vf->vlan_count++;
1016 } else {
1017 vf->vlans[vid >> 5] &= ~bit;
1018 vf->vlan_count--;
1019 }
1020 igb_iov_rebuild_vlan(sc);
1021 return (0);
1022 }
1023
1024 static void
igb_iov_reset_vf_state(struct e1000_softc * sc,struct igb_vf * vf)1025 igb_iov_reset_vf_state(struct e1000_softc *sc, struct igb_vf *vf)
1026 {
1027 bool update_uta;
1028
1029 update_uta = (vf->flags & IGB_VF_UCAST_PROMISC) != 0;
1030 vf->flags &= ~(IGB_VF_CTS | IGB_VF_UCAST_PROMISC |
1031 IGB_VF_MCAST_PROMISC | IGB_VF_MCAST_OVERFLOW |
1032 IGB_VF_MBX_PENDING | IGB_VF_MBX_GAVE_UP |
1033 IGB_VF_MDD_NOTIFY_PENDING);
1034 vf->mbx_retry_at = 0;
1035 vf->mdd_notify_at = 0;
1036 vf->mbx_retry_count = 0;
1037 /*
1038 * A reset starts a new mailbox epoch. Permit one immediate NACK so a
1039 * premature non-reset request does not wait for its posted-read
1040 * timeout.
1041 */
1042 memset(&vf->last_nack, 0, sizeof(vf->last_nack));
1043 vf->max_frame_size = ETHER_MAX_LEN;
1044 vf->mc_count = 0;
1045 vf->vlan_count = 0;
1046 memset(vf->mc_hashes, 0, sizeof(vf->mc_hashes));
1047 memset(vf->vlans, 0, sizeof(vf->vlans));
1048 /* Preserve the administrative access VLAN across VF and PF resets. */
1049 if (vf->default_vlan == 0)
1050 vf->vlans[0] = 1U;
1051 else {
1052 vf->vlans[vf->default_vlan >> 5] =
1053 1U << (vf->default_vlan & 0x1f);
1054 vf->vlan_count = 1;
1055 }
1056 igb_iov_configure_vmolr(sc, vf);
1057 if (update_uta)
1058 igb_iov_set_uta(sc);
1059 }
1060
1061 static bool
igb_iov_vf_vlan_is_default(const struct igb_vf * vf)1062 igb_iov_vf_vlan_is_default(const struct igb_vf *vf)
1063 {
1064 u32 expected;
1065 int i;
1066
1067 for (i = 0; i < EM_VFTA_SIZE; i++) {
1068 expected = 0;
1069 if (i == vf->default_vlan >> 5)
1070 expected = 1U << (vf->default_vlan & 0x1f);
1071 if (vf->vlans[i] != expected)
1072 return (false);
1073 }
1074 return (true);
1075 }
1076
1077 static bool
igb_iov_sanitize_vf_queues(struct e1000_softc * sc,struct igb_vf * vf)1078 igb_iov_sanitize_vf_queues(struct e1000_softc *sc,
1079 struct igb_vf *vf)
1080 {
1081 struct e1000_hw *hw;
1082 u16 qid[IGB_IOV_VF_QUEUES_MAX];
1083 u32 rxdctl, txdctl;
1084 int i, nqueues, retry;
1085
1086 hw = &sc->hw;
1087 switch (hw->mac.type) {
1088 case e1000_82576:
1089 nqueues = IGB_82576_VF_QUEUES;
1090 qid[0] = vf->pool;
1091 qid[1] = vf->pool + IGB_82576_VF_QUEUE_STRIDE;
1092 break;
1093 case e1000_i350:
1094 nqueues = IGB_I350_VF_QUEUES;
1095 qid[0] = vf->pool;
1096 break;
1097 default:
1098 return (true);
1099 }
1100
1101 /*
1102 * I350 maps pool n to queue n. 82576 gives VF n physical queues n
1103 * and n + 8, so both retained queue configurations must be cleared.
1104 */
1105 for (i = 0; i < nqueues; i++)
1106 KASSERT(qid[i] < (hw->mac.type == e1000_82576 ?
1107 IGB_82576_NUM_QUEUES : IGB_I350_NUM_QUEUES),
1108 ("%s: invalid VF queue %u", __func__, qid[i]));
1109
1110 /*
1111 * The 82576 and I350 specification updates, Software Clarification 3,
1112 * note that VFLR does not reset the VF queue configuration. Clear the
1113 * PF-programmable state before acknowledging the reset so a new VF
1114 * owner cannot inherit it, particularly a descriptor-head write-back
1115 * DMA address. The new VF driver initializes its active ring pointers
1116 * during queue setup.
1117 *
1118 * Disable every queue first, then wait for outstanding DMA activity to
1119 * stop before clearing TDWBAL/H and the remaining retained state.
1120 * Spin only for the normal fast transition, then sleep so a VF that
1121 * keeps asserting QUEUE_ENABLE cannot busy-wait the PF for 10 ms.
1122 */
1123 for (i = 0; i < nqueues; i++) {
1124 E1000_WRITE_REG(hw, E1000_RXDCTL(qid[i]), 0);
1125 E1000_WRITE_REG(hw, E1000_TXDCTL(qid[i]), 0);
1126 }
1127 E1000_WRITE_FLUSH(hw);
1128 for (retry = 0; retry < IGB_IOV_QUEUE_DISABLE_RETRIES; retry++) {
1129 for (i = 0; i < nqueues; i++) {
1130 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(qid[i]));
1131 txdctl = E1000_READ_REG(hw, E1000_TXDCTL(qid[i]));
1132 if ((rxdctl & E1000_RXDCTL_QUEUE_ENABLE) != 0 ||
1133 (txdctl & E1000_TXDCTL_QUEUE_ENABLE) != 0)
1134 break;
1135 }
1136 if (i == nqueues)
1137 break;
1138 if (retry + 1 < IGB_IOV_QUEUE_DISABLE_RETRIES) {
1139 if (retry < IGB_IOV_QUEUE_DISABLE_BUSY_RETRIES)
1140 DELAY(IGB_IOV_QUEUE_DISABLE_DELAY_US);
1141 else
1142 pause_sbt("igbqds",
1143 IGB_IOV_QUEUE_DISABLE_PAUSE, 0,
1144 C_PREL(1));
1145 }
1146 }
1147 if (retry == IGB_IOV_QUEUE_DISABLE_RETRIES) {
1148 if (ratecheck(&vf->last_queue_log,
1149 &igb_iov_mbx_log_interval))
1150 device_printf(sc->dev,
1151 "could not disable queues for VF %u; "
1152 "reset deferred\n", vf->pool);
1153 return (false);
1154 }
1155
1156 for (i = 0; i < nqueues; i++) {
1157 E1000_WRITE_REG(hw, E1000_SRRCTL(qid[i]), 0);
1158 E1000_WRITE_REG(hw, E1000_DCA_RXCTRL(qid[i]), 0);
1159 E1000_WRITE_REG(hw, E1000_TDWBAL(qid[i]), 0);
1160 E1000_WRITE_REG(hw, E1000_TDWBAH(qid[i]), 0);
1161 E1000_WRITE_REG(hw, E1000_DCA_TXCTRL(qid[i]), 0);
1162 }
1163 E1000_WRITE_REG(hw, E1000_PSRTYPE(vf->pool), 0);
1164 E1000_WRITE_FLUSH(hw);
1165 return (true);
1166 }
1167
1168 static bool
igb_iov_reset_event_common(struct e1000_softc * sc,struct igb_vf * vf,bool reset_intrs)1169 igb_iov_reset_event_common(struct e1000_softc *sc, struct igb_vf *vf,
1170 bool reset_intrs)
1171 {
1172 struct e1000_hw *hw;
1173 bool rebuild_mta, rebuild_vlan, sanitized;
1174 u32 reg;
1175
1176 hw = &sc->hw;
1177 rebuild_mta = vf->mc_count != 0;
1178 rebuild_vlan = !igb_iov_vf_vlan_is_default(vf);
1179 reg = E1000_READ_REG(hw, E1000_VFTE);
1180 E1000_WRITE_REG(hw, E1000_VFTE, reg & ~(1U << vf->pool));
1181 reg = E1000_READ_REG(hw, E1000_VFRE);
1182 E1000_WRITE_REG(hw, E1000_VFRE, reg & ~(1U << vf->pool));
1183 if (reset_intrs)
1184 E1000_WRITE_REG(hw, E1000_VTCTRL(vf->pool),
1185 E1000_VTCTRL_RST);
1186 sanitized = igb_iov_sanitize_vf_queues(sc, vf);
1187 E1000_WRITE_REG(hw, E1000_VMVIR(vf->pool), 0);
1188 igb_iov_clear_mac_filters(sc, vf);
1189 igb_iov_clear_rar(sc, vf->rar_index);
1190 igb_iov_reset_vf_state(sc, vf);
1191 if (rebuild_mta)
1192 igb_iov_rebuild_mta(sc);
1193 if (rebuild_vlan)
1194 igb_iov_rebuild_vlan(sc);
1195 return (sanitized);
1196 }
1197
1198 static bool
igb_iov_reset_event(struct e1000_softc * sc,struct igb_vf * vf)1199 igb_iov_reset_event(struct e1000_softc *sc, struct igb_vf *vf)
1200 {
1201 return (igb_iov_reset_event_common(sc, vf, true));
1202 }
1203
1204 static void
igb_iov_mdd_reset_event(struct e1000_softc * sc,struct igb_vf * vf)1205 igb_iov_mdd_reset_event(struct e1000_softc *sc, struct igb_vf *vf)
1206 {
1207 /*
1208 * VTCTRL.RST clears the VF's queue-enable and interrupt registers
1209 * (I350 section 8.28.1). It therefore also removes the admin-vector
1210 * route needed to deliver the reset notification below. MDD recovery
1211 * explicitly permits toggling VFTE instead (section 7.8.3.8.3).
1212 *
1213 * Leave the interrupt registers intact, keep VFTE/VFRE disabled until
1214 * the VF completes a new reset handshake, and use the no-CTS control
1215 * message to make the guest reinitialize. FreeBSD and DPDK consume
1216 * that message directly; Linux ACKs it and the PF's non-CTS ACK path
1217 * replies with the NACK that schedules igbvf's reset task.
1218 *
1219 * Sanitization failure leaves the pool disabled. The VF reset
1220 * handshake retries it and is NACKed while a queue remains active.
1221 */
1222 (void)igb_iov_reset_event_common(sc, vf, false);
1223 }
1224
1225 static void
igb_iov_reset_msg(struct e1000_softc * sc,struct igb_vf * vf)1226 igb_iov_reset_msg(struct e1000_softc *sc, struct igb_vf *vf)
1227 {
1228 struct e1000_hw *hw;
1229 u32 msg[3], reg;
1230
1231 hw = &sc->hw;
1232 if (!igb_iov_reset_event(sc, vf)) {
1233 msg[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_NACK;
1234 e1000_write_mbx(hw, msg, 1, vf->pool);
1235 return;
1236 }
1237 igb_iov_map_rar(sc, vf->rar_index, vf->mac, vf->pool);
1238 igb_iov_set_anti_spoof(sc, vf);
1239
1240 reg = E1000_READ_REG(hw, E1000_VFTE);
1241 E1000_WRITE_REG(hw, E1000_VFTE, reg | (1U << vf->pool));
1242 reg = E1000_READ_REG(hw, E1000_VFRE);
1243 E1000_WRITE_REG(hw, E1000_VFRE, reg | (1U << vf->pool));
1244 /*
1245 * 82576's WVBR blocked bitmap is read-clear, so the reset handshake
1246 * completes that event's lifetime. I350 MDFB might be read-only;
1247 * re-arm its edge latch only after a valid MDFB sample reads clear.
1248 */
1249 if (hw->mac.type == e1000_82576)
1250 vf->flags &= ~IGB_VF_MDD_BLOCKED;
1251 vf->flags |= IGB_VF_CTS;
1252
1253 memset(msg, 0, sizeof(msg));
1254 msg[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
1255 memcpy(&msg[1], vf->mac, ETHER_ADDR_LEN);
1256 e1000_write_mbx(hw, msg, 3, vf->pool);
1257 }
1258
1259 static int
igb_iov_set_mac_filter(struct e1000_softc * sc,struct igb_vf * vf,u32 * msg)1260 igb_iov_set_mac_filter(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg)
1261 {
1262 struct igb_vf_mac_filter *filter, *free_filter;
1263 const u8 *mac;
1264 u32 info;
1265 int count, i;
1266
1267 info = msg[0] & E1000_VT_MSGINFO_MASK;
1268 if (info == E1000_VF_MAC_FILTER_CLR) {
1269 igb_iov_clear_mac_filters(sc, vf);
1270 return (0);
1271 }
1272 if (info != E1000_VF_MAC_FILTER_ADD)
1273 return (EINVAL);
1274 if ((vf->flags & IGB_VF_CAP_MAC) == 0)
1275 return (EPERM);
1276
1277 mac = (const u8 *)&msg[1];
1278 if (!igb_iov_mac_valid(mac))
1279 return (EINVAL);
1280 if (memcmp(mac, vf->mac, ETHER_ADDR_LEN) == 0)
1281 return (0);
1282
1283 count = 0;
1284 free_filter = NULL;
1285 for (i = 0; i < sc->num_vf_mac_filters; i++) {
1286 filter = &sc->vf_mac_filters[i];
1287 if (!filter->active) {
1288 if (free_filter == NULL)
1289 free_filter = filter;
1290 continue;
1291 }
1292 if (memcmp(filter->mac, mac, ETHER_ADDR_LEN) != 0)
1293 continue;
1294 return (filter->pool == vf->pool ? 0 : EADDRINUSE);
1295 }
1296 for (i = 0; i < sc->num_vf_mac_filters; i++)
1297 if (sc->vf_mac_filters[i].active &&
1298 sc->vf_mac_filters[i].pool == vf->pool)
1299 count++;
1300 if (igb_iov_mac_in_use(sc, mac, vf))
1301 return (EADDRINUSE);
1302 if (count >= IGB_IOV_MAX_MAC_FILTERS)
1303 return (ENOSPC);
1304 if (free_filter == NULL)
1305 return (ENOSPC);
1306
1307 free_filter->active = true;
1308 free_filter->pool = vf->pool;
1309 memcpy(free_filter->mac, mac, ETHER_ADDR_LEN);
1310 igb_iov_map_rar(sc, free_filter->rar_index, free_filter->mac, vf->pool);
1311 return (0);
1312 }
1313
1314 static int
igb_iov_set_mac(struct e1000_softc * sc,struct igb_vf * vf,u32 * msg)1315 igb_iov_set_mac(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg)
1316 {
1317 u8 *mac;
1318
1319 if ((msg[0] & E1000_VT_MSGINFO_MASK) != 0)
1320 return (igb_iov_set_mac_filter(sc, vf, msg));
1321
1322 mac = (u8 *)&msg[1];
1323 if (!igb_iov_mac_valid(mac))
1324 return (EINVAL);
1325 if (memcmp(mac, vf->mac, ETHER_ADDR_LEN) != 0 &&
1326 !(vf->flags & IGB_VF_CAP_MAC))
1327 return (EPERM);
1328 if (memcmp(mac, vf->mac, ETHER_ADDR_LEN) != 0 &&
1329 igb_iov_mac_in_use(sc, mac, vf))
1330 return (EADDRINUSE);
1331
1332 memcpy(vf->mac, mac, ETHER_ADDR_LEN);
1333 igb_iov_map_rar(sc, vf->rar_index, vf->mac, vf->pool);
1334 return (0);
1335 }
1336
1337 static int
igb_iov_set_multicast(struct e1000_softc * sc,struct igb_vf * vf,u32 * msg)1338 igb_iov_set_multicast(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg)
1339 {
1340 u16 hashes[IGB_IOV_MAX_MC_HASHES] = {};
1341 bool overflow;
1342 int count, i;
1343
1344 count = (msg[0] & E1000_VF_SET_MULTICAST_COUNT_MASK) >>
1345 E1000_VT_MSGINFO_SHIFT;
1346 overflow = count > IGB_IOV_MAX_MC_HASHES ||
1347 (msg[0] & E1000_VF_SET_MULTICAST_OVERFLOW) != 0;
1348 count = min(count, IGB_IOV_MAX_MC_HASHES);
1349 for (i = 0; i < count; i++)
1350 hashes[i] =
1351 (msg[1 + i / 2] >> ((i & 1) * 16)) & 0xffff;
1352 if (vf->mc_count == count &&
1353 ((vf->flags & IGB_VF_MCAST_OVERFLOW) != 0) == overflow &&
1354 memcmp(vf->mc_hashes, hashes, sizeof(hashes)) == 0)
1355 return (0);
1356 memcpy(vf->mc_hashes, hashes, sizeof(vf->mc_hashes));
1357 vf->mc_count = count;
1358 if (overflow)
1359 vf->flags |= IGB_VF_MCAST_OVERFLOW;
1360 else
1361 vf->flags &= ~IGB_VF_MCAST_OVERFLOW;
1362 if (overflow &&
1363 (vf->flags & IGB_VF_MCAST_OVERFLOW_WARNED) == 0) {
1364 vf->flags |= IGB_VF_MCAST_OVERFLOW_WARNED;
1365 device_printf(sc->dev,
1366 "VF %u multicast list exceeds 30 entries; "
1367 "enabling all-multicast reception\n", vf->pool);
1368 }
1369 igb_iov_configure_vmolr(sc, vf);
1370 igb_iov_rebuild_mta(sc);
1371 return (0);
1372 }
1373
1374 static int
igb_iov_set_lpe(struct e1000_softc * sc,struct igb_vf * vf,u32 * msg)1375 igb_iov_set_lpe(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg)
1376 {
1377 u32 size;
1378
1379 size = msg[1];
1380 if (size < ETHER_MIN_LEN)
1381 return (EINVAL);
1382 vf->max_frame_size = min(size, IGB_IOV_MAX_FRAME_SIZE);
1383 igb_iov_configure_vmolr(sc, vf);
1384 return (0);
1385 }
1386
1387 static int
igb_iov_set_promisc(struct e1000_softc * sc,struct igb_vf * vf,u32 msg)1388 igb_iov_set_promisc(struct e1000_softc *sc, struct igb_vf *vf, u32 msg)
1389 {
1390 u32 mode;
1391
1392 mode = msg & E1000_VT_MSGINFO_MASK;
1393 if (mode & ~(E1000_VF_SET_PROMISC_UNICAST |
1394 E1000_VF_SET_PROMISC_MULTICAST))
1395 return (EINVAL);
1396 if (mode != 0 && !(vf->flags & IGB_VF_ALLOW_PROMISC))
1397 return (EPERM);
1398
1399 vf->flags &= ~(IGB_VF_UCAST_PROMISC | IGB_VF_MCAST_PROMISC);
1400 if (mode & E1000_VF_SET_PROMISC_UNICAST)
1401 vf->flags |= IGB_VF_UCAST_PROMISC;
1402 if (mode & E1000_VF_SET_PROMISC_MULTICAST)
1403 vf->flags |= IGB_VF_MCAST_PROMISC;
1404 igb_iov_configure_vmolr(sc, vf);
1405 igb_iov_set_uta(sc);
1406 return (0);
1407 }
1408
1409 static bool
igb_iov_process_msg(struct e1000_softc * sc,struct igb_vf * vf)1410 igb_iov_process_msg(struct e1000_softc *sc, struct igb_vf *vf)
1411 {
1412 struct e1000_hw *hw;
1413 u32 msg[E1000_VFMAILBOX_SIZE], type;
1414 int error;
1415
1416 hw = &sc->hw;
1417 memset(msg, 0, sizeof(msg));
1418 if (e1000_read_mbx(hw, msg, nitems(msg), vf->pool, false) != 0)
1419 return (false);
1420 vf->flags &= ~IGB_VF_MBX_PENDING;
1421 vf->mbx_retry_at = 0;
1422 vf->mbx_retry_count = 0;
1423
1424 if (msg[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK)) {
1425 e1000_unlock_mbx(hw, vf->pool);
1426 return (true);
1427 }
1428 if (msg[0] == E1000_VF_RESET) {
1429 igb_iov_reset_msg(sc, vf);
1430 return (true);
1431 }
1432 if (!(vf->flags & IGB_VF_CTS)) {
1433 if (igb_iov_nack_allowed(vf)) {
1434 msg[0] = igb_iov_reply_header(msg[0], false, false);
1435 e1000_write_mbx(hw, msg, 1, vf->pool);
1436 } else
1437 e1000_unlock_mbx(hw, vf->pool);
1438 return (true);
1439 }
1440
1441 type = msg[0] & 0xffff;
1442 switch (type) {
1443 case E1000_VF_SET_MAC_ADDR:
1444 error = igb_iov_set_mac(sc, vf, msg);
1445 break;
1446 case E1000_VF_SET_MULTICAST:
1447 error = igb_iov_set_multicast(sc, vf, msg);
1448 break;
1449 case E1000_VF_SET_VLAN:
1450 if (vf->default_vlan != 0)
1451 error = EPERM;
1452 else if ((msg[1] & ~E1000_VLVF_VLANID_MASK) != 0)
1453 error = EINVAL;
1454 else
1455 error = igb_iov_set_vlan(sc, vf,
1456 msg[1] & E1000_VLVF_VLANID_MASK,
1457 (msg[0] & E1000_VF_SET_VLAN_ADD) != 0);
1458 break;
1459 case E1000_VF_SET_LPE:
1460 error = igb_iov_set_lpe(sc, vf, msg);
1461 break;
1462 case E1000_VF_SET_PROMISC:
1463 error = igb_iov_set_promisc(sc, vf, msg[0]);
1464 break;
1465 default:
1466 error = EOPNOTSUPP;
1467 break;
1468 }
1469
1470 msg[0] = igb_iov_reply_header(msg[0], true, error == 0);
1471 e1000_write_mbx(hw, msg, 1, vf->pool);
1472 return (true);
1473 }
1474
1475 static sbintime_t
igb_iov_service_pending_mbx(struct e1000_softc * sc,struct igb_vf * vf,sbintime_t now)1476 igb_iov_service_pending_mbx(struct e1000_softc *sc, struct igb_vf *vf,
1477 sbintime_t now)
1478 {
1479 sbintime_t delay;
1480
1481 if ((vf->flags & IGB_VF_MBX_PENDING) == 0)
1482 return (0);
1483 if (vf->mbx_retry_at != 0 && now < vf->mbx_retry_at)
1484 return (vf->mbx_retry_at);
1485 if (igb_iov_process_msg(sc, vf))
1486 return (0);
1487
1488 now = getsbinuptime();
1489 if (vf->mbx_retry_count < IGB_IOV_MBX_RETRY_COUNT) {
1490 delay = igb_iov_mbx_retry_delay[vf->mbx_retry_count++];
1491 vf->mbx_retry_at = now + delay;
1492 return (vf->mbx_retry_at);
1493 }
1494
1495 vf->flags &= ~(IGB_VF_CTS | IGB_VF_MBX_PENDING);
1496 vf->flags |= IGB_VF_MBX_GAVE_UP;
1497 vf->mbx_retry_at = 0;
1498 if (ratecheck(&vf->last_mbx_log, &igb_iov_mbx_log_interval))
1499 device_printf(sc->dev,
1500 "mailbox remained busy for VF %u; CTS revoked\n",
1501 vf->pool);
1502 return (0);
1503 }
1504
1505 void
igb_iov_handle_mbx(struct e1000_softc * sc)1506 igb_iov_handle_mbx(struct e1000_softc *sc)
1507 {
1508 struct e1000_hw *hw;
1509 struct igb_vf *vf;
1510 sbintime_t delay, next_retry_at, now, retry_at;
1511 u32 msg;
1512 int i;
1513
1514 if (!sc->iov_hw_active)
1515 return;
1516
1517 hw = &sc->hw;
1518 next_retry_at = 0;
1519 for (i = 0; i < sc->num_vfs; i++) {
1520 vf = &sc->vfs[i];
1521 if (!(vf->flags & IGB_VF_ACTIVE))
1522 continue;
1523 now = getsbinuptime();
1524 if (e1000_check_for_rst(hw, vf->pool) == 0) {
1525 /*
1526 * The old VF is gone. A new owner's reset handshake
1527 * reruns sanitization before enabling its pool.
1528 */
1529 (void)igb_iov_reset_event(sc, vf);
1530 }
1531 if ((vf->flags &
1532 (IGB_VF_MBX_PENDING | IGB_VF_MBX_GAVE_UP)) == 0 &&
1533 e1000_check_for_msg(hw, vf->pool) == 0) {
1534 vf->flags |= IGB_VF_MBX_PENDING;
1535 vf->mbx_retry_at = 0;
1536 vf->mbx_retry_count = 0;
1537 }
1538 retry_at = igb_iov_service_pending_mbx(sc, vf, now);
1539 if (retry_at != 0 &&
1540 (next_retry_at == 0 || retry_at < next_retry_at))
1541 next_retry_at = retry_at;
1542 if (e1000_check_for_ack(hw, vf->pool) == 0 &&
1543 !(vf->flags & IGB_VF_CTS) && igb_iov_nack_allowed(vf)) {
1544 msg = E1000_VT_MSGTYPE_NACK;
1545 e1000_write_mbx(hw, &msg, 1, vf->pool);
1546 }
1547 }
1548 if (next_retry_at != 0) {
1549 delay = next_retry_at - getsbinuptime();
1550 if (delay <= 0)
1551 delay = SBT_1MS;
1552 callout_reset_sbt(&sc->iov_mbx_retry, delay, 0,
1553 igb_iov_mbx_retry_callout, sc, C_PREL(1));
1554 }
1555 }
1556
1557 static bool
igb_iov_notify_vf_mdd_reset(struct e1000_softc * sc,struct igb_vf * vf)1558 igb_iov_notify_vf_mdd_reset(struct e1000_softc *sc, struct igb_vf *vf)
1559 {
1560 u32 msg;
1561
1562 /*
1563 * MDD recovery preserves the VF's admin-vector configuration. Send
1564 * the same no-CTS control message used for PF reset notification so
1565 * the VF discards its state and completes a new reset handshake.
1566 * A failed write is retried from the timer-driven admin pass; the VF's
1567 * transmit watchdog remains the final fallback when traffic is still
1568 * queued and notification never succeeds.
1569 */
1570 msg = E1000_PF_CONTROL_MSG;
1571 if (e1000_write_mbx(&sc->hw, &msg, 1, vf->pool) != 0) {
1572 vf->mdd_notify_at =
1573 getsbinuptime() + igb_iov_mdd_notify_retry;
1574 if (ratecheck(&vf->last_mbx_log,
1575 &igb_iov_mbx_log_interval))
1576 device_printf(sc->dev,
1577 "could not notify VF %u of malicious-driver "
1578 "reset; will retry\n", vf->pool);
1579 return (false);
1580 }
1581 vf->flags &= ~IGB_VF_MDD_NOTIFY_PENDING;
1582 vf->mdd_notify_at = 0;
1583 return (true);
1584 }
1585
1586 void
igb_iov_handle_mdd(struct e1000_softc * sc)1587 igb_iov_handle_mdd(struct e1000_softc *sc)
1588 {
1589 struct igb_vf *vf;
1590 u32 blocked, cleared, handled, lvmmc;
1591 u32 readback, spoofed;
1592 bool mdfb_valid, pending;
1593 int i;
1594
1595 pending = atomic_readandclear_32(&sc->iov_pending) != 0;
1596 lvmmc = pending ?
1597 atomic_readandclear_32(&sc->iov_mdd_cause) : 0;
1598 if (!sc->iov_hw_active) {
1599 atomic_readandclear_32(&sc->iov_spoof_pending);
1600 atomic_readandclear_32(&sc->iov_blocked_pending);
1601 return;
1602 }
1603
1604 blocked = 0;
1605 handled = 0;
1606 mdfb_valid = false;
1607 if (sc->hw.mac.type == e1000_i350) {
1608 u32 mdfb;
1609
1610 spoofed = atomic_readandclear_32(&sc->iov_spoof_pending);
1611 /*
1612 * I350 reports ordinary MAC/VLAN spoofing through the
1613 * interrupt-time LVMMC snapshot rather than WVBR. The
1614 * filter accumulates Last_Q into iov_spoof_pending so events
1615 * from different VFs coalesce safely until this timer-driven
1616 * admin pass.
1617 */
1618 spoofed &= IGB_I350_QUEUE_MASK;
1619 /*
1620 * Sample MDFB on every admin pass so a blocked queue is not
1621 * mislabeled as an ordinary spoof when no MDDET observation
1622 * is pending.
1623 */
1624 mdfb = E1000_READ_REG(&sc->hw, E1000_MDFB);
1625 if (__predict_false(mdfb == 0xffffffff))
1626 mdfb = 0;
1627 else {
1628 mdfb &= IGB_I350_QUEUE_MASK;
1629 mdfb_valid = true;
1630 }
1631 /*
1632 * I350 SDM sections 8.14.10 and 8.14.11: WVBR reports
1633 * spoof and malicious-driver events, while MDFB identifies
1634 * the queues actually blocked for malicious behavior.
1635 */
1636 spoofed &= ~mdfb;
1637 blocked = mdfb;
1638 if (blocked != 0 && lvmmc == 0)
1639 lvmmc = E1000_READ_REG(&sc->hw, E1000_LVMMC);
1640 /*
1641 * A failed diagnostic read does not invalidate the
1642 * blocked-queue bitmap that was read successfully above.
1643 */
1644 if (__predict_false(lvmmc == 0xffffffff))
1645 lvmmc = 0;
1646 /*
1647 * MDFB is authoritative for queues stopped by malicious-driver
1648 * detection. LVMMC reports causes such as VLAN IERR and
1649 * Mal_PF, but its Last_Q field does not establish that a queue
1650 * was blocked. Do not manufacture a blocked bit when MDFB is
1651 * clear.
1652 */
1653 } else {
1654 if (!pending)
1655 return;
1656 /*
1657 * WVBR is read-clear and does not preserve every queue across
1658 * multiple MDDET interrupts. The interrupt filter snapshots and
1659 * accumulates its pool bitmaps before this deferred admin pass.
1660 */
1661 spoofed = atomic_readandclear_32(&sc->iov_spoof_pending);
1662 blocked = atomic_readandclear_32(&sc->iov_blocked_pending);
1663 /* A blocked-queue classification dominates its WVBR low bit. */
1664 spoofed &= ~blocked;
1665 }
1666
1667 for (i = 0; i < sc->num_vfs; i++) {
1668 vf = &sc->vfs[i];
1669 if (!(vf->flags & IGB_VF_ACTIVE))
1670 continue;
1671 if ((vf->flags & IGB_VF_MDD_NOTIFY_PENDING) != 0 &&
1672 getsbinuptime() >= vf->mdd_notify_at)
1673 (void)igb_iov_notify_vf_mdd_reset(sc, vf);
1674 /*
1675 * An invalid MDFB sample must neither report a new edge nor
1676 * masquerade as evidence that an old edge has cleared.
1677 */
1678 if (sc->hw.mac.type == e1000_i350 && mdfb_valid &&
1679 (blocked & (1U << i)) == 0)
1680 vf->flags &= ~IGB_VF_MDD_BLOCKED;
1681 if ((spoofed & (1U << i)) != 0 &&
1682 ratecheck(&vf->last_spoof_log,
1683 &igb_iov_spoof_log_interval))
1684 device_printf(sc->dev,
1685 "spoof event detected from VF %u; packet dropped\n",
1686 vf->pool);
1687 if ((blocked & (1U << i)) == 0)
1688 continue;
1689 if ((vf->flags & IGB_VF_MDD_BLOCKED) != 0)
1690 continue;
1691 vf->flags |= IGB_VF_MDD_BLOCKED;
1692 if (ratecheck(&vf->last_mdd_log, &igb_iov_mdd_log_interval))
1693 device_printf(sc->dev,
1694 "malicious-driver event 0x%08x from VF %u; "
1695 "resetting VF\n", lvmmc, vf->pool);
1696 igb_iov_mdd_reset_event(sc, vf);
1697 vf->flags |= IGB_VF_MDD_NOTIFY_PENDING;
1698 (void)igb_iov_notify_vf_mdd_reset(sc, vf);
1699 handled |= 1U << i;
1700 }
1701 if (sc->hw.mac.type == e1000_i350 && mdfb_valid &&
1702 (blocked & (1U << sc->pool)) == 0)
1703 sc->iov_pf_mdd_blocked = false;
1704 if ((blocked & (1U << sc->pool)) != 0 &&
1705 (sc->hw.mac.type != e1000_i350 ||
1706 !sc->iov_pf_mdd_blocked)) {
1707 if (sc->hw.mac.type == e1000_i350)
1708 sc->iov_pf_mdd_blocked = true;
1709 if (ratecheck(&sc->iov_last_mdd_log,
1710 &igb_iov_mdd_log_interval))
1711 device_printf(sc->dev,
1712 "malicious-driver event 0x%08x from PF queue; "
1713 "resetting PF\n", lvmmc);
1714 iflib_request_reset(sc->ctx);
1715 iflib_admin_intr_deferred(sc->ctx);
1716 handled |= 1U << sc->pool;
1717 }
1718 if (sc->hw.mac.type == e1000_i350 && handled != 0) {
1719 /*
1720 * I350 documentation conflicts: the register summary calls
1721 * MDFB RWS while the detailed field table calls it RO. I350
1722 * silicon clears a blocked bit when software writes it back.
1723 * Write only bits whose recovery was initiated. If a revision
1724 * instead implements MDFB as RO, the edge latch above prevents
1725 * a reset loop and this one transition-time write is harmless.
1726 */
1727 E1000_WRITE_REG(&sc->hw, E1000_MDFB, handled);
1728 E1000_WRITE_FLUSH(&sc->hw);
1729 /*
1730 * Rearm from observed hardware state instead of waiting for
1731 * the next admin pass. The PF context lock prevents a reset
1732 * handshake from re-enabling the VF before this readback. A
1733 * write-to-clear part reports zero; a read-only part retains
1734 * the bit and therefore retains the one-shot edge latch.
1735 */
1736 readback = E1000_READ_REG(&sc->hw, E1000_MDFB);
1737 if (__predict_false(readback == 0xffffffff))
1738 cleared = 0;
1739 else
1740 cleared = handled &
1741 ~(readback & IGB_I350_QUEUE_MASK);
1742 for (i = 0; i < sc->num_vfs; i++)
1743 if ((cleared & (1U << i)) != 0)
1744 sc->vfs[i].flags &= ~IGB_VF_MDD_BLOCKED;
1745 if ((cleared & (1U << sc->pool)) != 0)
1746 sc->iov_pf_mdd_blocked = false;
1747 }
1748 if (sc->hw.mac.type == e1000_i350) {
1749 /*
1750 * I350 can retain EICR.OTHER without delivering the admin MSI-X
1751 * even though its EIMS and legacy IMS bits remain enabled. Kick
1752 * the already-enabled vector on each admin pass so its filter
1753 * consumes any retained ICR/LVMMC cause. A synthetic interrupt
1754 * with no legacy cause is handled entirely by the filter.
1755 */
1756 E1000_WRITE_REG(&sc->hw, E1000_EICS, sc->link_mask);
1757 E1000_WRITE_FLUSH(&sc->hw);
1758 }
1759 }
1760
1761 void
igb_iov_mdd_event(struct e1000_softc * sc)1762 igb_iov_mdd_event(struct e1000_softc *sc)
1763 {
1764 u32 blocked, cause, queues, queue, spoofed, wvbr;
1765
1766 /*
1767 * LVMMC is clear-on-read. Preserve it in the interrupt filter, as
1768 * Intel's igb driver does, rather than deferring the only copy.
1769 */
1770 cause = E1000_READ_REG(&sc->hw, E1000_LVMMC);
1771 if (__predict_false(cause == 0xffffffff))
1772 return;
1773 if (sc->hw.mac.type == e1000_82576) {
1774 /*
1775 * Snapshot WVBR in the interrupt filter. Waiting for the admin
1776 * task loses all but the last of back-to-back VF MDD events on
1777 * 82576. Convert the staggered queue map into pool bits and OR
1778 * each observation into software latches for deferred recovery.
1779 */
1780 wvbr = E1000_READ_REG(&sc->hw, E1000_WVBR);
1781 if (__predict_false(wvbr == 0xffffffff)) {
1782 spoofed = 0;
1783 blocked = 0;
1784 } else {
1785 queues = wvbr & IGB_82576_QUEUE_MASK;
1786 spoofed = (queues & 0xff) |
1787 (queues >> IGB_82576_STAGGERED_QUEUE_SHIFT);
1788 queues = (wvbr >> 16) & IGB_82576_QUEUE_MASK;
1789 blocked = (queues & 0xff) |
1790 (queues >> IGB_82576_STAGGERED_QUEUE_SHIFT);
1791 }
1792 SDT_PROBE4(igb_iov, mdd, sample, wvbr, sc, wvbr, spoofed,
1793 blocked);
1794 /*
1795 * 82576 can report a coalesced block-class event with all affected
1796 * queues in WVBR's low half and no high-half blocked bits. If an
1797 * ordinary spoof shares that snapshot, the register has no per-queue
1798 * cause information. Deliberately fail closed by recovering every
1799 * low-half queue; this can reset a spoof-only sibling, but avoids
1800 * stranding a blocked VF. LVMMC.Last_Q identifies only the final
1801 * event and lost simultaneous blocked VFs on tested silicon.
1802 */
1803 if (blocked == 0 &&
1804 (cause & IGB_82576_LVMMC_BLOCK_MASK) != 0) {
1805 blocked = spoofed;
1806 if (blocked == 0) {
1807 queue = (cause >> 16) & 0xf;
1808 blocked = 1U << (queue & 0x7);
1809 }
1810 }
1811 if (spoofed != 0)
1812 atomic_set_32(&sc->iov_spoof_pending, spoofed);
1813 if (blocked != 0)
1814 atomic_set_32(&sc->iov_blocked_pending, blocked);
1815 }
1816 if (sc->hw.mac.type == e1000_i350 &&
1817 (cause & IGB_I350_LVMMC_MAC_VLAN_SPOOF) != 0) {
1818 queue = (cause >> IGB_I350_LVMMC_LAST_Q_SHIFT) &
1819 IGB_I350_LVMMC_LAST_Q_MASK;
1820 /*
1821 * FreeBSD assigns one queue to each VF pool, so Last_Q is
1822 * also the VF number. Preserve all VFs observed before the
1823 * timer pass, and do not overwrite an unrelated blocked
1824 * queue's diagnostic with this non-blocking spoof event.
1825 */
1826 atomic_set_32(&sc->iov_spoof_pending, 1U << queue);
1827 return;
1828 }
1829 atomic_store_rel_32(&sc->iov_mdd_cause, cause);
1830 atomic_set_32(&sc->iov_pending, 1);
1831 }
1832
1833 void
igb_iov_ping_all_vfs(struct e1000_softc * sc)1834 igb_iov_ping_all_vfs(struct e1000_softc *sc)
1835 {
1836 struct igb_vf *vf;
1837 u32 msg;
1838 int i;
1839
1840 if (!sc->iov_hw_active)
1841 return;
1842
1843 for (i = 0; i < sc->num_vfs; i++) {
1844 vf = &sc->vfs[i];
1845 if (!(vf->flags & IGB_VF_ACTIVE))
1846 continue;
1847 msg = E1000_PF_CONTROL_MSG;
1848 if (vf->flags & IGB_VF_CTS)
1849 msg |= E1000_VT_MSGTYPE_CTS;
1850 e1000_write_mbx(&sc->hw, &msg, 1, vf->pool);
1851 }
1852 }
1853
1854 void
igb_iov_initialize(struct e1000_softc * sc)1855 igb_iov_initialize(struct e1000_softc *sc)
1856 {
1857 struct e1000_hw *hw;
1858 struct igb_vf *vf;
1859 u32 ctrl_ext, dtxctl, mask, rctl, rplolr, vt_ctl;
1860 int i;
1861
1862 if (sc->num_vfs == 0)
1863 return;
1864
1865 hw = &sc->hw;
1866 atomic_readandclear_32(&sc->iov_mdd_cause);
1867 atomic_readandclear_32(&sc->iov_pending);
1868 atomic_readandclear_32(&sc->iov_spoof_pending);
1869 atomic_readandclear_32(&sc->iov_blocked_pending);
1870 /* Plain VMDq keeps every 82576 PF/VF pool on queue zero. */
1871 E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_ENABLE_VMDQ);
1872
1873 vt_ctl = E1000_READ_REG(hw, E1000_VT_CTL);
1874 vt_ctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
1875 E1000_VT_CTL_DISABLE_DEF_POOL);
1876 vt_ctl |= sc->pool << E1000_VT_CTL_DEFAULT_POOL_SHIFT;
1877 vt_ctl |= E1000_VT_CTL_VM_REPL_EN;
1878 E1000_WRITE_REG(hw, E1000_VT_CTL, vt_ctl);
1879
1880 mask = 1U << sc->pool;
1881 E1000_WRITE_REG(hw, E1000_VFRE, mask);
1882 E1000_WRITE_REG(hw, E1000_VFTE, mask);
1883 /* A VF without RX descriptors must not block any other pool. */
1884 E1000_WRITE_REG(hw, E1000_QDE,
1885 hw->mac.type == e1000_i350 ? IGB_I350_QUEUE_MASK : ALL_QUEUES);
1886 e1000_vmdq_set_loopback_pf(hw, true);
1887 dtxctl = E1000_READ_REG(hw, E1000_DTXCTL);
1888 dtxctl |= E1000_DTXCTL_MDP_EN;
1889 if (hw->mac.type == e1000_82576) {
1890 dtxctl |= E1000_DTXCTL_VLAN_ADDED |
1891 E1000_DTXCTL_SPOOF_INT;
1892 rplolr = E1000_READ_REG(hw, E1000_RPLOLR);
1893 rplolr |= E1000_RPLOLR_STRVLAN;
1894 E1000_WRITE_REG(hw, E1000_RPLOLR, rplolr);
1895 } else {
1896 /*
1897 * I350 SDM section 8.12.5 defines this field with inverted
1898 * polarity: setting it keeps an ordinary spoof from disabling
1899 * the VF queue. Enable its notification as well. I350
1900 * hardware reports the VF in LVMMC.Last_Q (WVBR remains zero);
1901 * the moderated admin vector captures that value, while
1902 * timer-driven administration and per-VF ratecheck bound the
1903 * work and console output.
1904 */
1905 dtxctl |= E1000_DTXCTL_SPOOF_INT |
1906 IGB_I350_DTXCTL_ENABLE_SPOOF_QUEUE;
1907 }
1908 E1000_WRITE_REG(hw, E1000_DTXCTL, dtxctl);
1909
1910 igb_iov_map_rar(sc, 0, hw->mac.addr, sc->pool);
1911 igb_iov_configure_pf_vmolr(sc);
1912 igb_iov_set_uta(sc);
1913 for (i = 0; i < sc->num_vfs; i++) {
1914 vf = &sc->vfs[i];
1915 if (!(vf->flags & IGB_VF_ACTIVE))
1916 continue;
1917 /*
1918 * A PF-wide reset is trusted and can require a complete guest
1919 * replay. Guest-controlled RESET and VFLR do not refill this
1920 * allowance.
1921 */
1922 igb_iov_reset_vlan_rate(vf);
1923 igb_iov_clear_mac_filters(sc, vf);
1924 igb_iov_reset_vf_state(sc, vf);
1925 igb_iov_clear_rar(sc, vf->rar_index);
1926 igb_iov_set_anti_spoof(sc, vf);
1927 }
1928 igb_iov_rebuild_mta(sc);
1929 igb_iov_rebuild_vlan(sc);
1930
1931 rctl = E1000_READ_REG(hw, E1000_RCTL);
1932 E1000_WRITE_REG(hw, E1000_RCTL, rctl | E1000_RCTL_VFE);
1933 E1000_WRITE_REG(hw, E1000_MBVFIMR, igb_iov_active_mask(sc));
1934
1935 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1936 E1000_WRITE_REG(hw, E1000_CTRL_EXT,
1937 ctrl_ext | E1000_CTRL_EXT_PFRSTD);
1938 E1000_WRITE_FLUSH(hw);
1939 /*
1940 * MDDET remains masked until iov_hw_active is published and iflib
1941 * rearms the admin vector. Programming the per-pool policy above can
1942 * leave a setup-time MDDET observation in the read-clear registers.
1943 * If that stale cause is carried across the unmask, a later ordinary
1944 * spoof can update LVMMC without generating a new interrupt edge.
1945 *
1946 * Drain only after all IOV policy is installed and before exposing the
1947 * active state. Mailbox requests are also serviced by the periodic
1948 * admin pass, and ping_all_vfs() below supplies a fresh notification.
1949 */
1950 /*
1951 * Clear the setup-time interrupt latch before its diagnostic state.
1952 * I350 does not reliably generate the next MDDET edge when LVMMC is
1953 * consumed while ICR.MDDET remains latched. This differs deliberately
1954 * from the final arm-time drain, where ICR is read last so a later event
1955 * remains pending for the unmask.
1956 */
1957 if (hw->mac.type == e1000_i350)
1958 (void)E1000_READ_REG(hw, E1000_ICR);
1959 (void)E1000_READ_REG(hw, E1000_LVMMC);
1960 if (hw->mac.type == e1000_82576)
1961 (void)E1000_READ_REG(hw, E1000_WVBR);
1962 if (hw->mac.type != e1000_i350)
1963 (void)E1000_READ_REG(hw, E1000_ICR);
1964 atomic_readandclear_32(&sc->iov_mdd_cause);
1965 atomic_readandclear_32(&sc->iov_pending);
1966 atomic_readandclear_32(&sc->iov_spoof_pending);
1967 atomic_readandclear_32(&sc->iov_blocked_pending);
1968 atomic_store_rel_32(&sc->iov_intr_drain_pending, 1);
1969 sc->iov_hw_active = true;
1970 igb_iov_ping_all_vfs(sc);
1971 }
1972
1973 int
igb_iov_validate(struct e1000_softc * sc,u16 num_vfs)1974 igb_iov_validate(struct e1000_softc *sc, u16 num_vfs)
1975 {
1976 if (!igb_iov_supported(sc))
1977 return (ENXIO);
1978 /* One of the eight hardware pools is reserved for the PF. */
1979 if (num_vfs == 0 || num_vfs > MAX_NUM_VFS)
1980 return (EINVAL);
1981 if (sc->vfs != NULL)
1982 return (EBUSY);
1983 if (sc->intr_type != IFLIB_INTR_MSIX) {
1984 device_printf(sc->dev, "SR-IOV requires MSI-X\n");
1985 return (ENOTSUP);
1986 }
1987 if (sc->tx_num_queues != 1 || sc->rx_num_queues != 1) {
1988 device_printf(sc->dev,
1989 "SR-IOV requires one PF TX and RX queue; set "
1990 "dev.igb.%d.iflib.override_ntxqs=1 and "
1991 "dev.igb.%d.iflib.override_nrxqs=1 before attach\n",
1992 device_get_unit(sc->dev), device_get_unit(sc->dev));
1993 return (EINVAL);
1994 }
1995 return (0);
1996 }
1997
1998 int
igb_if_vf_status(if_ctx_t ctx,struct if_vf_status ** statusp)1999 igb_if_vf_status(if_ctx_t ctx, struct if_vf_status **statusp)
2000 {
2001 struct e1000_softc *sc;
2002 struct igb_vf *vf;
2003 struct if_vf_info *info;
2004 struct if_vf_status *status;
2005 u_int num_queues;
2006 int i;
2007
2008 sc = iflib_get_softc(ctx);
2009 if (!sc->iov_mbx_retry_initialized)
2010 return (EOPNOTSUPP);
2011 num_queues = sc->hw.mac.type == e1000_82576 ?
2012 IGB_82576_VF_QUEUES : IGB_I350_VF_QUEUES;
2013 status = if_vf_status_alloc(sc->num_vfs);
2014 if (status == NULL)
2015 return (ENOMEM);
2016 for (i = 0; i < sc->num_vfs; i++) {
2017 vf = &sc->vfs[i];
2018 info = &status->vfs[i];
2019 info->fields = IFVF_F_CONFIGURED | IFVF_F_INITIALIZED |
2020 IFVF_F_VLAN_MODE | IFVF_F_VLAN_COUNT |
2021 IFVF_F_NUM_TX_QUEUES | IFVF_F_NUM_RX_QUEUES |
2022 IFVF_F_ALLOW_SET_MAC |
2023 IFVF_F_ALLOW_SET_VLAN | IFVF_F_MAC_ANTI_SPOOF |
2024 IFVF_F_ALLOW_PROMISC | IFVF_F_TRAFFIC_ALLOWED |
2025 IFVF_F_FAULT_BLOCKED;
2026 info->index = i;
2027 info->configured = (vf->flags & IGB_VF_ACTIVE) != 0;
2028 info->initialized = sc->iov_hw_active &&
2029 (vf->flags & IGB_VF_CTS) != 0;
2030 if (!ETHER_IS_ZERO(vf->mac)) {
2031 memcpy(info->mac, vf->mac, sizeof(info->mac));
2032 info->fields |= IFVF_F_MAC;
2033 }
2034 if (vf->default_vlan == 0)
2035 info->vlan_mode = IFVF_VLAN_TRUNK;
2036 else {
2037 info->vlan_mode = IFVF_VLAN_ACCESS;
2038 info->vlan = vf->default_vlan;
2039 info->vlan_pcp = 0;
2040 info->vlan_proto = ETHERTYPE_VLAN;
2041 info->fields |= IFVF_F_VLAN | IFVF_F_VLAN_PCP |
2042 IFVF_F_VLAN_PROTO;
2043 }
2044 info->vlan_count = vf->vlan_count;
2045 info->tx_queue_count = num_queues;
2046 info->rx_queue_count = num_queues;
2047 info->allow_set_mac = (vf->flags & IGB_VF_CAP_MAC) != 0;
2048 /* Access VFs cannot manage VLAN membership through the mailbox. */
2049 info->allow_set_vlan = vf->default_vlan == 0;
2050 info->mac_anti_spoof =
2051 (vf->flags & IGB_VF_MAC_ANTI_SPOOF) != 0;
2052 info->allow_promisc =
2053 (vf->flags & IGB_VF_ALLOW_PROMISC) != 0;
2054 info->fault_blocked =
2055 (vf->flags & IGB_VF_MDD_BLOCKED) != 0;
2056 info->traffic_allowed = info->configured &&
2057 !info->fault_blocked;
2058 }
2059 *statusp = status;
2060 return (0);
2061 }
2062
2063 int
igb_if_iov_init(if_ctx_t ctx,u16 num_vfs,const nvlist_t * config)2064 igb_if_iov_init(if_ctx_t ctx, u16 num_vfs, const nvlist_t *config)
2065 {
2066 struct e1000_softc *sc;
2067 int error, i;
2068
2069 sc = iflib_get_softc(ctx);
2070 (void)config;
2071 /*
2072 * This callback may run while the PF is down. Record the software
2073 * layout here; igb_iov_initialize() programs it during interface init.
2074 */
2075 atomic_store_rel_32(&sc->iov_teardown, 0);
2076 error = igb_iov_validate(sc, num_vfs);
2077 if (error != 0)
2078 return (error);
2079
2080 sc->vfs = mallocarray(num_vfs, sizeof(*sc->vfs), M_IGB_IOV,
2081 M_WAITOK | M_ZERO);
2082 sc->num_vf_mac_filters =
2083 sc->hw.mac.rar_entry_count - num_vfs - 1;
2084 sc->vf_mac_filters = mallocarray(sc->num_vf_mac_filters,
2085 sizeof(*sc->vf_mac_filters), M_IGB_IOV, M_WAITOK | M_ZERO);
2086 for (i = 0; i < sc->num_vf_mac_filters; i++)
2087 sc->vf_mac_filters[i].rar_index = i + 1;
2088 sc->pool = num_vfs;
2089 sc->iov_mta_valid = false;
2090 sc->iov_pf_mdd_blocked = false;
2091 sc->tx_queues[0].txr.me = sc->pool;
2092 sc->rx_queues[0].rxr.me = sc->pool;
2093 e1000_init_mbx_params_pf(&sc->hw);
2094 sc->num_vfs = num_vfs;
2095 return (0);
2096 }
2097
2098 void
igb_if_iov_uninit(if_ctx_t ctx)2099 igb_if_iov_uninit(if_ctx_t ctx)
2100 {
2101 struct e1000_softc *sc;
2102 struct e1000_hw *hw;
2103 u32 mask, rah;
2104 int error, i, iov_pos;
2105 u16 iov_ctl;
2106
2107 sc = iflib_get_softc(ctx);
2108 if (sc->vfs == NULL)
2109 return;
2110 hw = &sc->hw;
2111 sc->iov_hw_active = false;
2112 if (sc->iov_mbx_retry_initialized)
2113 callout_drain(&sc->iov_mbx_retry);
2114
2115 E1000_WRITE_REG(hw, E1000_MBVFIMR, 0);
2116 mask = 1U << sc->pool;
2117 E1000_WRITE_REG(hw, E1000_VFRE, mask);
2118 E1000_WRITE_REG(hw, E1000_VFTE, mask);
2119
2120 /*
2121 * pci_iov(4) invokes the driver before it clears VF Enable. Quiesce
2122 * the VFs and clear it here so that 82576's queue-reuse interval is
2123 * measured from the actual IOV-disable event.
2124 */
2125 error = pci_find_extcap(sc->dev, PCIZ_SRIOV, &iov_pos);
2126 if (error == 0) {
2127 iov_ctl = pci_read_config(sc->dev,
2128 iov_pos + PCIR_SRIOV_CTL, 2);
2129 iov_ctl &= ~(PCIM_SRIOV_VF_EN | PCIM_SRIOV_VF_MSE);
2130 pci_write_config(sc->dev, iov_pos + PCIR_SRIOV_CTL,
2131 iov_ctl, 2);
2132 if (hw->mac.type == e1000_82576) {
2133 pause("igbiov", MAX(1, howmany(hz, 10)));
2134 E1000_WRITE_REG(hw, E1000_IOVCTL,
2135 E1000_IOVCTL_REUSE_VFQ);
2136 E1000_WRITE_FLUSH(hw);
2137 pause("igbiov", MAX(1, howmany(hz, 10)));
2138 }
2139 } else
2140 device_printf(sc->dev,
2141 "could not disable PCI SR-IOV before queue reuse: %d\n",
2142 error);
2143
2144 E1000_WRITE_REG(hw, E1000_VT_CTL, 0);
2145 e1000_vmdq_set_loopback_pf(hw, false);
2146 e1000_vmdq_set_anti_spoofing_pf(hw, false, 0);
2147 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
2148 E1000_WRITE_REG(hw, E1000_VLVF(i), 0);
2149 for (i = 0; i < sc->num_vfs; i++)
2150 if (sc->vfs[i].flags & IGB_VF_ACTIVE)
2151 igb_iov_clear_rar(sc, sc->vfs[i].rar_index);
2152 for (i = 0; i < sc->num_vf_mac_filters; i++)
2153 if (sc->vf_mac_filters[i].active)
2154 igb_iov_clear_rar(sc, sc->vf_mac_filters[i].rar_index);
2155 rah = E1000_READ_REG(hw, E1000_RAH(0));
2156 rah &= ~IGB_IOV_RAH_POOLSEL_MASK;
2157 E1000_WRITE_REG(hw, E1000_RAH(0), rah);
2158
2159 free(sc->vfs, M_IGB_IOV);
2160 free(sc->vf_mac_filters, M_IGB_IOV);
2161 sc->vfs = NULL;
2162 sc->vf_mac_filters = NULL;
2163 sc->num_vfs = 0;
2164 sc->num_vf_mac_filters = 0;
2165 sc->pool = 0;
2166 sc->iov_mta_valid = false;
2167 sc->iov_pf_mdd_blocked = false;
2168 sc->iov_pf_vlan_promisc = false;
2169 igb_iov_vfta_shadow_invalidate(sc);
2170 sc->tx_queues[0].txr.me = 0;
2171 sc->rx_queues[0].rxr.me = 0;
2172 atomic_readandclear_32(&sc->iov_mdd_cause);
2173 atomic_readandclear_32(&sc->iov_pending);
2174 atomic_readandclear_32(&sc->iov_spoof_pending);
2175 atomic_readandclear_32(&sc->iov_blocked_pending);
2176 atomic_readandclear_32(&sc->iov_intr_drain_pending);
2177 atomic_store_rel_32(&sc->iov_teardown, 0);
2178 }
2179
2180 static bool
igb_iov_mac_in_use(struct e1000_softc * sc,const u8 * mac,const struct igb_vf * skip)2181 igb_iov_mac_in_use(struct e1000_softc *sc, const u8 *mac,
2182 const struct igb_vf *skip)
2183 {
2184 int i;
2185
2186 if (memcmp(sc->hw.mac.addr, mac, ETHER_ADDR_LEN) == 0)
2187 return (true);
2188 for (i = 0; i < sc->num_vfs; i++)
2189 if (&sc->vfs[i] != skip &&
2190 (sc->vfs[i].flags & IGB_VF_ACTIVE) != 0 &&
2191 memcmp(sc->vfs[i].mac, mac, ETHER_ADDR_LEN) == 0)
2192 return (true);
2193 for (i = 0; i < sc->num_vf_mac_filters; i++)
2194 if (sc->vf_mac_filters[i].active &&
2195 memcmp(sc->vf_mac_filters[i].mac, mac,
2196 ETHER_ADDR_LEN) == 0)
2197 return (true);
2198 return (false);
2199 }
2200
2201 int
igb_if_iov_vf_add(if_ctx_t ctx,u16 vfnum,const nvlist_t * config)2202 igb_if_iov_vf_add(if_ctx_t ctx, u16 vfnum, const nvlist_t *config)
2203 {
2204 struct e1000_softc *sc;
2205 struct igb_vf *vf;
2206 struct ether_addr generated;
2207 const void *mac;
2208 char nameunit[IFNAMSIZ + sizeof("-vf65535")];
2209 size_t mac_size;
2210 uint64_t configured_vlan;
2211 u16 vlan;
2212
2213 sc = iflib_get_softc(ctx);
2214 if (vfnum >= sc->num_vfs)
2215 return (EINVAL);
2216 vf = &sc->vfs[vfnum];
2217 if (vf->flags & IGB_VF_ACTIVE)
2218 return (EBUSY);
2219
2220 configured_vlan = nvlist_get_number(config, "vlan");
2221 if (configured_vlan > VF_VLAN_TRUNK)
2222 return (EINVAL);
2223 vlan = configured_vlan;
2224 if (vlan == 0)
2225 return (ENOTSUP);
2226 if (vlan == VF_VLAN_TRUNK)
2227 vlan = 0;
2228 if (!igb_iov_vlan_present(sc, vlan, false) &&
2229 igb_iov_vlan_unique_count(sc, false) >=
2230 E1000_VLVF_ARRAY_SIZE)
2231 return (ENOSPC);
2232
2233 vf->pool = vfnum;
2234 vf->rar_index = sc->hw.mac.rar_entry_count - (vfnum + 1);
2235 vf->max_frame_size = ETHER_MAX_LEN;
2236 vf->default_vlan = vlan;
2237 igb_iov_reset_vlan_rate(vf);
2238 if (nvlist_exists_binary(config, "mac-addr")) {
2239 mac = nvlist_get_binary(config, "mac-addr", &mac_size);
2240 if (mac_size != ETHER_ADDR_LEN || !igb_iov_mac_valid(mac))
2241 return (EINVAL);
2242 if (igb_iov_mac_in_use(sc, mac, vf))
2243 return (EADDRINUSE);
2244 memcpy(vf->mac, mac, ETHER_ADDR_LEN);
2245 } else {
2246 snprintf(nameunit, sizeof(nameunit), "%s-vf%u",
2247 device_get_nameunit(sc->dev), vfnum);
2248 ether_gen_addr_byname(nameunit, &generated);
2249 memcpy(vf->mac, generated.octet, ETHER_ADDR_LEN);
2250 if (igb_iov_mac_in_use(sc, vf->mac, vf))
2251 return (EADDRINUSE);
2252 }
2253 if (nvlist_get_bool(config, "allow-set-mac"))
2254 vf->flags |= IGB_VF_CAP_MAC;
2255 if (nvlist_get_bool(config, "mac-anti-spoof"))
2256 vf->flags |= IGB_VF_MAC_ANTI_SPOOF;
2257 if (nvlist_get_bool(config, "allow-promisc"))
2258 vf->flags |= IGB_VF_ALLOW_PROMISC;
2259 vf->flags |= IGB_VF_ACTIVE;
2260
2261 igb_iov_reset_vf_state(sc, vf);
2262 igb_iov_set_anti_spoof(sc, vf);
2263 igb_iov_rebuild_vlan(sc);
2264 E1000_WRITE_REG(&sc->hw, E1000_MBVFIMR, igb_iov_active_mask(sc));
2265 return (0);
2266 }
2267
2268 #endif /* PCI_IOV */
2269