1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 2010-2016, Intel Corporation
5 * Copyright (c) 2026 Kevin Bowling <kbowling@FreeBSD.org>
6 */
7
8 #include "if_em.h"
9 #include "if_igb_iov.h"
10
11 #ifdef PCI_IOV
12
13 #include <sys/iov.h>
14 #include <sys/sdt.h>
15 #include <sys/time.h>
16
17 #define IGB_IOV_RAH_POOLSEL_SHIFT 18
18 #define IGB_IOV_RAH_POOLSEL_MASK (0xffU << IGB_IOV_RAH_POOLSEL_SHIFT)
19 #define IGB_IOV_MAX_MAC_FILTERS 3
20 #define IGB_IOV_MAX_MC_HASHES 30
21 #define IGB_IOV_MBX_RETRY_COUNT 6
22 /* Allow two complete 31-VID replays, then sustain eight additions/second. */
23 #define IGB_IOV_VLAN_CHANGE_BURST 64
24 #define IGB_IOV_VLAN_CHANGE_INTERVAL (SBT_1S / 8)
25 /* 82576 Datasheet rev. 2.0, Section 8.14.16: VMOLR[31] must be one. */
26 #define IGB_82576_VMOLR_RSV (1U << 31)
27 #define IGB_82576_LVMMC_BLOCK_MASK 0x1c
28 #define IGB_82576_NUM_QUEUES 16
29 #define IGB_82576_QUEUE_MASK 0xffff
30 #define IGB_82576_STAGGERED_QUEUE_SHIFT 8
31 #define IGB_82576_VF_QUEUE_STRIDE 8
32 #define IGB_82576_VF_QUEUES 2
33 #define IGB_I350_DTXCTL_ENABLE_SPOOF_QUEUE (1U << 2)
34 #define IGB_I350_LVMMC_MAC_VLAN_SPOOF (1U << 25)
35 #define IGB_I350_LVMMC_LAST_Q_SHIFT 29
36 #define IGB_I350_LVMMC_LAST_Q_MASK 0x7
37 #define IGB_I350_NUM_QUEUES 8
38 #define IGB_I350_QUEUE_MASK 0xff
39 #define IGB_I350_RESET_ACK_TIMEOUT (100 * SBT_1MS)
40 #define IGB_I350_VF_QUEUES 1
41 #define IGB_IOV_QUEUE_DISABLE_BUSY_RETRIES 10
42 #define IGB_IOV_QUEUE_DISABLE_DELAY_US 10
43 #define IGB_IOV_QUEUE_DISABLE_PAUSE (100 * SBT_1US)
44 #define IGB_IOV_QUEUE_DISABLE_RETRIES 20
45 #define IGB_IOV_VF_QUEUES_MAX 2
46
47 #define IGB_VF_CTS (1U << 0)
48 #define IGB_VF_CAP_MAC (1U << 1)
49 #define IGB_VF_ACTIVE (1U << 2)
50 #define IGB_VF_MAC_ANTI_SPOOF (1U << 3)
51 #define IGB_VF_ALLOW_PROMISC (1U << 4)
52 #define IGB_VF_UCAST_PROMISC (1U << 5)
53 #define IGB_VF_MCAST_PROMISC (1U << 6)
54 #define IGB_VF_MCAST_OVERFLOW (1U << 7)
55 #define IGB_VF_MCAST_OVERFLOW_WARNED (1U << 8)
56 #define IGB_VF_MDD_BLOCKED (1U << 9)
57 #define IGB_VF_MBX_PENDING (1U << 10)
58 /*
59 * After bounded PFU retries, suppress future or overlapping VF requests until
60 * RST/VFLR starts a new mailbox epoch. Intel VF drivers assert CTRL.RST
61 * before sending their mailbox reset request.
62 */
63 #define IGB_VF_MBX_GAVE_UP (1U << 11)
64 #define IGB_VF_MDD_NOTIFY_PENDING (1U << 12)
65
66 struct igb_vf {
67 u32 flags;
68 struct timeval last_nack;
69 struct timeval last_mbx_log;
70 struct timeval last_spoof_log;
71 struct timeval last_mdd_log;
72 struct timeval last_queue_log;
73 sbintime_t mbx_retry_at;
74 sbintime_t mdd_notify_at;
75 sbintime_t vlan_token_time;
76 u16 pool;
77 u16 rar_index;
78 u16 max_frame_size;
79 u16 mc_count;
80 u16 vlan_count;
81 u16 vlan_tokens;
82 u16 default_vlan;
83 u8 mbx_retry_count;
84 u8 mac[ETHER_ADDR_LEN];
85 u16 mc_hashes[IGB_IOV_MAX_MC_HASHES];
86 u32 vlans[EM_VFTA_SIZE];
87 };
88
89 struct igb_vf_mac_filter {
90 bool active;
91 u16 pool;
92 u16 rar_index;
93 u8 mac[ETHER_ADDR_LEN];
94 };
95
96 MALLOC_DEFINE(M_IGB_IOV, "igb_iov", "igb SR-IOV allocations");
97
98 /*
99 * These logical-write probes let hardware tests verify the elision policy.
100 * e1000_write_vfta_i350() expands one VFTA call into ten physical writes, so
101 * the probes intentionally count calls made by the rebuild rather than MMIO
102 * transactions. The state probe exposes the final software images while the
103 * stack arrays are still live.
104 */
105 SDT_PROVIDER_DEFINE(igb_iov);
106 SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, vfta_clear,
107 "struct e1000_softc *", "u_int", "uint32_t");
108 SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, vlvf_write,
109 "struct e1000_softc *", "u_int", "uint32_t");
110 SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, vfta_set,
111 "struct e1000_softc *", "u_int", "uint32_t");
112 SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, state,
113 "struct e1000_softc *", "uint32_t *", "uint32_t *");
114 SDT_PROBE_DEFINE4(igb_iov, mdd, sample, wvbr,
115 "struct e1000_softc *", "uint32_t", "uint32_t", "uint32_t");
116
117 static const struct timeval igb_iov_nack_interval = { 2, 0 };
118 static const struct timeval igb_iov_mbx_log_interval = { 2, 0 };
119 static const struct timeval igb_iov_spoof_log_interval = { 2, 0 };
120 static const struct timeval igb_iov_mdd_log_interval = { 2, 0 };
121 static const sbintime_t igb_iov_mdd_notify_retry = SBT_1S / 2;
122 static const sbintime_t igb_iov_mbx_retry_delay[IGB_IOV_MBX_RETRY_COUNT] = {
123 SBT_1MS,
124 2 * SBT_1MS,
125 4 * SBT_1MS,
126 8 * SBT_1MS,
127 16 * SBT_1MS,
128 32 * SBT_1MS,
129 };
130
131 static void igb_iov_clear_mac_filters(struct e1000_softc *,
132 const struct igb_vf *);
133 static bool igb_iov_mac_in_use(struct e1000_softc *, const u8 *,
134 const struct igb_vf *);
135 static bool igb_iov_vlan_present(struct e1000_softc *, u16, bool);
136 static int igb_iov_vlan_unique_count(struct e1000_softc *, bool);
137
138 static void
igb_iov_mbx_retry_callout(void * arg)139 igb_iov_mbx_retry_callout(void *arg)
140 {
141 struct e1000_softc *sc;
142
143 sc = arg;
144 /*
145 * Mailbox service is serialized by iflib's context lock. The
146 * callout only re-enters through the ordinary admin task.
147 */
148 iflib_admin_intr_deferred(sc->ctx);
149 }
150
151 static u_int
igb_iov_copy_maddr(void * arg,struct sockaddr_dl * sdl,u_int idx)152 igb_iov_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int idx)
153 {
154 u8 *mta;
155
156 if (idx == MAX_NUM_MULTICAST_ADDRESSES)
157 return (0);
158 mta = arg;
159 memcpy(&mta[idx * ETHER_ADDR_LEN], LLADDR(sdl), ETHER_ADDR_LEN);
160 return (1);
161 }
162
163 static bool
igb_iov_pf_vlan_promisc(struct e1000_softc * sc)164 igb_iov_pf_vlan_promisc(struct e1000_softc *sc)
165 {
166 if_t ifp;
167
168 ifp = iflib_get_ifp(sc->ctx);
169 return (sc->iov_pf_vlan_promisc ||
170 (if_getflags(ifp) & IFF_PROMISC) != 0);
171 }
172
173 static bool
igb_iov_mac_valid(const u8 * mac)174 igb_iov_mac_valid(const u8 *mac)
175 {
176 static const u8 zero[ETHER_ADDR_LEN];
177
178 return (!ETHER_IS_MULTICAST(mac) &&
179 memcmp(mac, zero, ETHER_ADDR_LEN) != 0);
180 }
181
182 static bool
igb_iov_nack_allowed(struct igb_vf * vf)183 igb_iov_nack_allowed(struct igb_vf *vf)
184 {
185 return (ratecheck(&vf->last_nack, &igb_iov_nack_interval) != 0);
186 }
187
188 static void
igb_iov_reset_vlan_rate(struct igb_vf * vf)189 igb_iov_reset_vlan_rate(struct igb_vf *vf)
190 {
191
192 vf->vlan_token_time = getsbinuptime();
193 vf->vlan_tokens = IGB_IOV_VLAN_CHANGE_BURST;
194 }
195
196 static bool
igb_iov_vlan_add_allowed(struct igb_vf * vf)197 igb_iov_vlan_add_allowed(struct igb_vf *vf)
198 {
199 sbintime_t elapsed, now;
200 uint64_t refill;
201
202 now = getsbinuptime();
203 elapsed = now - vf->vlan_token_time;
204 if (elapsed >= IGB_IOV_VLAN_CHANGE_INTERVAL) {
205 refill = elapsed / IGB_IOV_VLAN_CHANGE_INTERVAL;
206 vf->vlan_tokens = min((uint64_t)IGB_IOV_VLAN_CHANGE_BURST,
207 vf->vlan_tokens + refill);
208 vf->vlan_token_time = now;
209 }
210 if (vf->vlan_tokens == 0)
211 return (false);
212 vf->vlan_tokens--;
213 return (true);
214 }
215
216 static u32
igb_iov_reply_header(u32 request,bool cts,bool ack)217 igb_iov_reply_header(u32 request, bool cts, bool ack)
218 {
219 u32 reply, type;
220
221 type = request & 0xffff;
222 if (type == E1000_VF_SET_MAC_ADDR &&
223 (request & E1000_VT_MSGINFO_MASK) != 0)
224 reply = request;
225 else
226 reply = type;
227 reply &= ~(E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK |
228 E1000_VT_MSGTYPE_CTS);
229 if (cts)
230 reply |= E1000_VT_MSGTYPE_CTS;
231 reply |= ack ? E1000_VT_MSGTYPE_ACK : E1000_VT_MSGTYPE_NACK;
232 return (reply);
233 }
234
235 bool
igb_iov_supported(const struct e1000_softc * sc)236 igb_iov_supported(const struct e1000_softc *sc)
237 {
238 switch (sc->hw.mac.type) {
239 case e1000_82576:
240 case e1000_i350:
241 return (true);
242 default:
243 return (false);
244 }
245 }
246
247 bool
igb_iov_enabled(const struct e1000_softc * sc)248 igb_iov_enabled(const struct e1000_softc *sc)
249 {
250 return (sc->num_vfs != 0);
251 }
252
253 int
igb_iov_attach(struct e1000_softc * sc)254 igb_iov_attach(struct e1000_softc *sc)
255 {
256 nvlist_t *pf_schema, *vf_schema;
257 int error, iov_pos;
258
259 if (!igb_iov_supported(sc))
260 return (0);
261 if (pci_find_extcap(sc->dev, PCIZ_SRIOV, &iov_pos) != 0)
262 return (0);
263
264 pf_schema = pci_iov_schema_alloc_node();
265 vf_schema = pci_iov_schema_alloc_node();
266 pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL);
267 pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof",
268 IOV_SCHEMA_HASDEFAULT, true);
269 pci_iov_schema_add_bool(vf_schema, "allow-set-mac",
270 IOV_SCHEMA_HASDEFAULT, false);
271 pci_iov_schema_add_bool(vf_schema, "allow-promisc",
272 IOV_SCHEMA_HASDEFAULT, false);
273 pci_iov_schema_add_vlan(vf_schema, "vlan", IOV_SCHEMA_HASDEFAULT,
274 VF_VLAN_TRUNK);
275
276 error = pci_iov_attach(sc->dev, pf_schema, vf_schema);
277 if (error != 0)
278 device_printf(sc->dev,
279 "failed to attach SR-IOV configuration interface: %d\n",
280 error);
281 else {
282 callout_init(&sc->iov_mbx_retry, 1);
283 sc->iov_mbx_retry_initialized = true;
284 }
285 return (error);
286 }
287
288 void
igb_iov_detach(struct e1000_softc * sc)289 igb_iov_detach(struct e1000_softc *sc)
290 {
291
292 if (!sc->iov_mbx_retry_initialized)
293 return;
294 callout_drain(&sc->iov_mbx_retry);
295 sc->iov_mbx_retry_initialized = false;
296 }
297
298 static u32
igb_iov_active_mask(struct e1000_softc * sc)299 igb_iov_active_mask(struct e1000_softc *sc)
300 {
301 u32 mask;
302 int i;
303
304 mask = 0;
305 for (i = 0; i < sc->num_vfs; i++)
306 if (sc->vfs[i].flags & IGB_VF_ACTIVE)
307 mask |= 1U << i;
308 return (mask);
309 }
310
311 static void
igb_iov_map_rar(struct e1000_softc * sc,u16 rar,const u8 * mac,u16 pool)312 igb_iov_map_rar(struct e1000_softc *sc, u16 rar, const u8 *mac, u16 pool)
313 {
314 struct e1000_hw *hw;
315 u32 rah;
316
317 hw = &sc->hw;
318 e1000_rar_set(hw, __DECONST(u8 *, mac), rar);
319 rah = E1000_READ_REG(hw, E1000_RAH(rar));
320 rah &= ~IGB_IOV_RAH_POOLSEL_MASK;
321 rah |= 1U << (IGB_IOV_RAH_POOLSEL_SHIFT + pool);
322 E1000_WRITE_REG(hw, E1000_RAH(rar), rah);
323 }
324
325 static void
igb_iov_clear_rar(struct e1000_softc * sc,u16 rar)326 igb_iov_clear_rar(struct e1000_softc *sc, u16 rar)
327 {
328 u8 zero[ETHER_ADDR_LEN] = {};
329
330 e1000_rar_set(&sc->hw, zero, rar);
331 }
332
333 static void
igb_iov_clear_mac_filters(struct e1000_softc * sc,const struct igb_vf * vf)334 igb_iov_clear_mac_filters(struct e1000_softc *sc, const struct igb_vf *vf)
335 {
336 struct igb_vf_mac_filter *filter;
337 int i;
338
339 for (i = 0; i < sc->num_vf_mac_filters; i++) {
340 filter = &sc->vf_mac_filters[i];
341 if (!filter->active || filter->pool != vf->pool)
342 continue;
343 igb_iov_clear_rar(sc, filter->rar_index);
344 filter->active = false;
345 memset(filter->mac, 0, sizeof(filter->mac));
346 }
347 }
348
349 static u32
igb_iov_switch_reg(struct e1000_softc * sc)350 igb_iov_switch_reg(struct e1000_softc *sc)
351 {
352 return (sc->hw.mac.type == e1000_82576 ?
353 E1000_DTXSWC : E1000_TXSWC);
354 }
355
356 static void
igb_iov_set_anti_spoof(struct e1000_softc * sc,struct igb_vf * vf)357 igb_iov_set_anti_spoof(struct e1000_softc *sc, struct igb_vf *vf)
358 {
359 struct e1000_hw *hw;
360 u32 reg, value;
361
362 hw = &sc->hw;
363 reg = igb_iov_switch_reg(sc);
364 value = E1000_READ_REG(hw, reg);
365 value &= ~((1U << vf->pool) |
366 (1U << (vf->pool + E1000_DTXSWC_VLAN_SPOOF_SHIFT)));
367 if (vf->flags & IGB_VF_MAC_ANTI_SPOOF)
368 value |= 1U << vf->pool;
369 if (vf->flags & IGB_VF_ACTIVE)
370 value |= 1U <<
371 (vf->pool + E1000_DTXSWC_VLAN_SPOOF_SHIFT);
372 E1000_WRITE_REG(hw, reg, value);
373 }
374
375 static void
igb_iov_set_uta(struct e1000_softc * sc)376 igb_iov_set_uta(struct e1000_softc *sc)
377 {
378 struct e1000_hw *hw;
379 bool enable;
380 int i;
381
382 if (!igb_iov_enabled(sc) || sc->hw.mac.type != e1000_82576)
383 return;
384
385 hw = &sc->hw;
386 enable = (E1000_READ_REG(hw, E1000_VMOLR(sc->pool)) &
387 E1000_VMOLR_ROPE) != 0;
388 for (i = 0; i < sc->num_vfs; i++)
389 if ((sc->vfs[i].flags &
390 (IGB_VF_ACTIVE | IGB_VF_UCAST_PROMISC)) ==
391 (IGB_VF_ACTIVE | IGB_VF_UCAST_PROMISC)) {
392 enable = true;
393 break;
394 }
395
396 for (i = 0; i < MAX_MTA_REG; i++)
397 E1000_WRITE_REG_ARRAY(hw, E1000_UTA, i,
398 enable ? 0xffffffffU : 0);
399 }
400
401 static void
igb_iov_configure_dvmolr(struct e1000_softc * sc,u16 pool,bool strip_vlan,bool hide_vlan,bool vf_pool)402 igb_iov_configure_dvmolr(struct e1000_softc *sc, u16 pool,
403 bool strip_vlan, bool hide_vlan, bool vf_pool)
404 {
405 struct e1000_hw *hw;
406 u32 dvmolr;
407
408 hw = &sc->hw;
409 if (hw->mac.type != e1000_i350)
410 return;
411
412 dvmolr = E1000_READ_REG(hw, E1000_DVMOLR(pool));
413 dvmolr &= ~(E1000_DVMOLR_HIDVLAN | E1000_DVMOLR_STRVLAN |
414 E1000_DVMOLR_STRCRC);
415 if (hide_vlan)
416 dvmolr |= E1000_DVMOLR_HIDVLAN;
417 if (strip_vlan)
418 dvmolr |= E1000_DVMOLR_STRVLAN;
419 if (vf_pool || strip_vlan ||
420 (E1000_READ_REG(hw, E1000_RCTL) & E1000_RCTL_SECRC) != 0)
421 dvmolr |= E1000_DVMOLR_STRCRC;
422 E1000_WRITE_REG(hw, E1000_DVMOLR(pool), dvmolr);
423 }
424
425 static void
igb_iov_configure_vmolr(struct e1000_softc * sc,struct igb_vf * vf)426 igb_iov_configure_vmolr(struct e1000_softc *sc, struct igb_vf *vf)
427 {
428 struct e1000_hw *hw;
429 u32 max_frame_size, vmolr, vmvir;
430
431 hw = &sc->hw;
432 max_frame_size = vf->max_frame_size;
433 if (vf->vlan_count != 0)
434 max_frame_size = min(max_frame_size + VLAN_TAG_SIZE,
435 IGB_IOV_MAX_FRAME_SIZE);
436 vmolr = E1000_READ_REG(hw, E1000_VMOLR(vf->pool));
437 vmolr &= ~(E1000_VMOLR_RLPML_MASK | E1000_VMOLR_RSSE |
438 E1000_VMOLR_VPE | E1000_VMOLR_UPE | E1000_VMOLR_ROMPE |
439 E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_STRVLAN);
440 vmolr |= E1000_VMOLR_BAM | E1000_VMOLR_LPE |
441 (max_frame_size & E1000_VMOLR_RLPML_MASK);
442 if (vf->default_vlan == 0)
443 vmolr |= E1000_VMOLR_AUPE;
444 if (vf->mc_count != 0 &&
445 (vf->flags & (IGB_VF_MCAST_PROMISC |
446 IGB_VF_MCAST_OVERFLOW)) == 0)
447 vmolr |= E1000_VMOLR_ROMPE;
448 if (hw->mac.type == e1000_82576)
449 vmolr |= IGB_82576_VMOLR_RSV;
450
451 if (vf->flags & IGB_VF_UCAST_PROMISC) {
452 if (hw->mac.type == e1000_82576)
453 vmolr |= E1000_VMOLR_ROPE;
454 else
455 vmolr |= E1000_VMOLR_UPE;
456 }
457 /*
458 * The mailbox can describe only 30 hashes. Fall back to receiving all
459 * multicast within the VF's VLAN membership when that list overflows.
460 */
461 if ((vf->flags & (IGB_VF_MCAST_PROMISC |
462 IGB_VF_MCAST_OVERFLOW)) != 0)
463 vmolr |= E1000_VMOLR_MPME;
464 if (hw->mac.type == e1000_82576 && vf->vlan_count != 0)
465 vmolr |= E1000_VMOLR_STRVLAN;
466 /* A nonzero default VLAN makes this VF an untagged access port. */
467 if (vf->default_vlan == 0)
468 vmvir = 0;
469 else
470 vmvir = vf->default_vlan | E1000_VMVIR_VLANA_DEFAULT;
471
472 E1000_WRITE_REG(hw, E1000_VMOLR(vf->pool), vmolr);
473 E1000_WRITE_REG(hw, E1000_VMVIR(vf->pool), vmvir);
474 igb_iov_configure_dvmolr(sc, vf->pool, vf->vlan_count != 0,
475 vf->default_vlan != 0, true);
476 }
477
478 static void
igb_iov_configure_pf_vmolr(struct e1000_softc * sc)479 igb_iov_configure_pf_vmolr(struct e1000_softc *sc)
480 {
481 struct e1000_hw *hw;
482 if_t ifp;
483 bool strip_vlan;
484 u32 max_frame_size;
485 u32 old_vmolr, vmolr;
486
487 hw = &sc->hw;
488 ifp = iflib_get_ifp(sc->ctx);
489 max_frame_size = min(sc->shared->isc_max_frame_size + VLAN_TAG_SIZE,
490 IGB_IOV_MAX_FRAME_SIZE);
491 strip_vlan = (E1000_READ_REG(hw, E1000_CTRL) & E1000_CTRL_VME) != 0;
492 old_vmolr = E1000_READ_REG(hw, E1000_VMOLR(sc->pool));
493 vmolr = E1000_VMOLR_BAM | E1000_VMOLR_AUPE |
494 E1000_VMOLR_LPE |
495 (max_frame_size & E1000_VMOLR_RLPML_MASK);
496 if (hw->mac.type == e1000_82576) {
497 vmolr |= IGB_82576_VMOLR_RSV;
498 if (strip_vlan)
499 vmolr |= E1000_VMOLR_STRVLAN;
500 } else
501 vmolr |= old_vmolr & E1000_VMOLR_VPE;
502
503 if (if_getflags(ifp) & IFF_PROMISC) {
504 if (hw->mac.type == e1000_82576)
505 vmolr |= E1000_VMOLR_ROPE;
506 else
507 vmolr |= E1000_VMOLR_UPE | E1000_VMOLR_VPE;
508 vmolr |= E1000_VMOLR_MPME;
509 } else if ((if_getflags(ifp) & IFF_ALLMULTI) != 0 ||
510 if_llmaddr_count(ifp) >= MAX_NUM_MULTICAST_ADDRESSES)
511 vmolr |= E1000_VMOLR_MPME;
512 else if (if_llmaddr_count(ifp) != 0)
513 vmolr |= E1000_VMOLR_ROMPE;
514
515 E1000_WRITE_REG(hw, E1000_VMOLR(sc->pool), vmolr);
516 igb_iov_configure_dvmolr(sc, sc->pool, strip_vlan, false, false);
517 }
518
519 void
igb_iov_update_pf_vmolr(struct e1000_softc * sc)520 igb_iov_update_pf_vmolr(struct e1000_softc *sc)
521 {
522 if (!igb_iov_enabled(sc))
523 return;
524
525 igb_iov_configure_pf_vmolr(sc);
526 igb_iov_set_uta(sc);
527 }
528
529 u32
igb_iov_intr_mask(const struct e1000_softc * sc)530 igb_iov_intr_mask(const struct e1000_softc *sc)
531 {
532 if (!sc->iov_hw_active)
533 return (0);
534 return (E1000_IMS_VMMB | E1000_IMS_MDDET);
535 }
536
537 void
igb_iov_intr_drain_stale(struct e1000_softc * sc)538 igb_iov_intr_drain_stale(struct e1000_softc *sc)
539 {
540 struct e1000_hw *hw;
541 u32 icr;
542
543 if (atomic_readandclear_32(&sc->iov_intr_drain_pending) == 0)
544 return;
545 hw = &sc->hw;
546 /*
547 * Consume setup-time diagnostic state at the actual transition from
548 * masked to armed. Read ICR last so an event arriving after the drain
549 * remains pending and is delivered when the caller enables MDDET.
550 */
551 (void)E1000_READ_REG(hw, E1000_LVMMC);
552 if (hw->mac.type == e1000_82576)
553 (void)E1000_READ_REG(hw, E1000_WVBR);
554 icr = E1000_READ_REG(hw, E1000_ICR);
555 /*
556 * em_if_init() injects LSC after IOV setup to close the post-reset
557 * link race. Preserve that cause across this MDDET-specific drain.
558 */
559 if (__predict_true(icr != 0xffffffff) &&
560 (icr & E1000_ICR_LSC) != 0)
561 E1000_WRITE_REG(hw, E1000_ICS, E1000_ICS_LSC);
562 }
563
564 static void
igb_iov_vfta_shadow_invalidate(struct e1000_softc * sc)565 igb_iov_vfta_shadow_invalidate(struct e1000_softc *sc)
566 {
567
568 /*
569 * I350 erratum 20 makes VFTA reads unreliable while VMDq loopback or
570 * anti-spoofing is active. The shadow is therefore authoritative
571 * until a reset or another independent hardware writer invalidates
572 * it. Readback cannot reliably audit a stale-but-valid shadow on
573 * this part, so keep all shadow mutation in these two helpers.
574 */
575 memset(sc->iov_vfta, 0, sizeof(sc->iov_vfta));
576 sc->iov_vfta_valid = false;
577 }
578
579 static void
igb_iov_vfta_shadow_store(struct e1000_softc * sc,const u32 * vfta)580 igb_iov_vfta_shadow_store(struct e1000_softc *sc, const u32 *vfta)
581 {
582
583 memcpy(sc->iov_vfta, vfta, sizeof(sc->iov_vfta));
584 sc->iov_vfta_valid = true;
585 }
586
587 static void
igb_iov_notify_vfs_reset(struct e1000_softc * sc)588 igb_iov_notify_vfs_reset(struct e1000_softc *sc)
589 {
590 struct igb_vf *vf;
591 struct e1000_hw *hw;
592 sbintime_t deadline;
593 u32 msg, pending, undelivered;
594 int i;
595
596 hw = &sc->hw;
597 /*
598 * Process VFLRs first and wait only for VFs that completed their
599 * mailbox handshake. An unattached VF has nobody who can acknowledge.
600 */
601 igb_iov_handle_mbx(sc);
602 pending = 0;
603 for (i = 0; i < sc->num_vfs; i++) {
604 vf = &sc->vfs[i];
605 if ((vf->flags & (IGB_VF_ACTIVE | IGB_VF_CTS)) ==
606 (IGB_VF_ACTIVE | IGB_VF_CTS))
607 pending |= 1U << i;
608 }
609 if (pending == 0)
610 return;
611
612 /*
613 * I350 SDM section 4.6.11.2.3 requires each VF to acknowledge a
614 * mailbox warning before the PF asserts CTRL.RST.
615 *
616 * The mailbox pass above drained requests and stale acknowledgements.
617 * A VF read of the new notification sets its ACK bit.
618 */
619 undelivered = 0;
620 for (i = 0; i < sc->num_vfs; i++) {
621 if ((pending & (1U << i)) == 0)
622 continue;
623 msg = E1000_PF_CONTROL_MSG;
624 if (e1000_write_mbx(hw, &msg, 1, i) != 0) {
625 undelivered |= 1U << i;
626 pending &= ~(1U << i);
627 }
628 }
629 if (undelivered != 0)
630 device_printf(sc->dev,
631 "could not deliver reset warning to VF mask %#x\n",
632 undelivered);
633
634 deadline = getsbinuptime() + IGB_I350_RESET_ACK_TIMEOUT;
635 while (pending != 0 && getsbinuptime() < deadline) {
636 for (i = 0; i < sc->num_vfs; i++) {
637 if ((pending & (1U << i)) != 0 &&
638 e1000_check_for_ack(hw, i) == 0)
639 pending &= ~(1U << i);
640 }
641 if (pending != 0)
642 pause_sbt("igback", SBT_1MS, 0, C_HARDCLOCK);
643 }
644 if (pending != 0)
645 device_printf(sc->dev,
646 "VF reset acknowledgement timed out for mask %#x\n",
647 pending);
648 }
649
650 void
igb_iov_reset_prepare(struct e1000_softc * sc)651 igb_iov_reset_prepare(struct e1000_softc *sc)
652 {
653 struct e1000_hw *hw;
654 u32 mask;
655
656 if (sc->iov_hw_active) {
657 hw = &sc->hw;
658 if (atomic_load_acq_32(&sc->iov_teardown) == 0) {
659 if (hw->mac.type == e1000_i350)
660 igb_iov_notify_vfs_reset(sc);
661 else
662 igb_iov_ping_all_vfs(sc);
663 }
664
665 /* Stop VF DMA before the PF asserts CTRL.RST. */
666 mask = 1U << sc->pool;
667 E1000_WRITE_REG(hw, E1000_VFRE, mask);
668 E1000_WRITE_REG(hw, E1000_VFTE, mask);
669 E1000_WRITE_FLUSH(hw);
670 }
671 sc->iov_hw_active = false;
672 if (sc->iov_mbx_retry_initialized)
673 callout_stop(&sc->iov_mbx_retry);
674 sc->iov_mta_valid = false;
675 igb_iov_vfta_shadow_invalidate(sc);
676 atomic_readandclear_32(&sc->iov_mdd_cause);
677 atomic_readandclear_32(&sc->iov_pending);
678 atomic_readandclear_32(&sc->iov_spoof_pending);
679 atomic_readandclear_32(&sc->iov_blocked_pending);
680 /*
681 * Normal iflib initialization prepares the reset before
682 * igb_iov_initialize() requests this drain. Preserve a still-pending
683 * I350 request across a later stop or repeated preparation so the next
684 * interrupt arm consumes it. Other families retain the ordinary
685 * stop-time cleanup.
686 */
687 if (sc->hw.mac.type != e1000_i350)
688 atomic_readandclear_32(&sc->iov_intr_drain_pending);
689 }
690
691 void
igb_iov_rebuild_mta(struct e1000_softc * sc)692 igb_iov_rebuild_mta(struct e1000_softc *sc)
693 {
694 struct e1000_hw *hw;
695 struct igb_vf *vf;
696 u32 hash_bit, hash_reg, hash_value;
697 u32 mta[MAX_MTA_REG] = {};
698 u16 hash;
699 bool changed;
700 int i, j, mcnt;
701
702 if (!igb_iov_enabled(sc))
703 return;
704
705 hw = &sc->hw;
706 memset(sc->mta, 0,
707 ETHER_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES);
708 mcnt = if_foreach_llmaddr(iflib_get_ifp(sc->ctx),
709 igb_iov_copy_maddr, sc->mta);
710 mcnt = min(mcnt, MAX_NUM_MULTICAST_ADDRESSES);
711 for (i = 0; i < mcnt; i++) {
712 hash_value = e1000_hash_mc_addr(hw,
713 &sc->mta[i * ETHER_ADDR_LEN]);
714 hash_reg = (hash_value >> 5) &
715 (hw->mac.mta_reg_count - 1);
716 hash_bit = hash_value & 0x1f;
717 mta[hash_reg] |= 1U << hash_bit;
718 }
719 for (i = 0; i < sc->num_vfs; i++) {
720 vf = &sc->vfs[i];
721 if (!(vf->flags & IGB_VF_ACTIVE))
722 continue;
723 for (j = 0; j < vf->mc_count; j++) {
724 hash = vf->mc_hashes[j] & 0xfff;
725 mta[(hash >> 5) & (hw->mac.mta_reg_count - 1)] |=
726 1U << (hash & 0x1f);
727 }
728 }
729
730 changed = false;
731 for (i = hw->mac.mta_reg_count - 1; i >= 0; i--) {
732 if (sc->iov_mta_valid && hw->mac.mta_shadow[i] == mta[i])
733 continue;
734 hw->mac.mta_shadow[i] = mta[i];
735 E1000_WRITE_REG_ARRAY(hw, E1000_MTA, i, mta[i]);
736 changed = true;
737 }
738 if (changed)
739 E1000_WRITE_FLUSH(hw);
740 sc->iov_mta_valid = true;
741 }
742
743 static int
igb_iov_vlvf_add(u32 * vlvf,const u32 * old_vlvf,u16 vid,u16 pool,bool preserve_only)744 igb_iov_vlvf_add(u32 *vlvf, const u32 *old_vlvf, u16 vid, u16 pool,
745 bool preserve_only)
746 {
747 int free_slot, i;
748
749 free_slot = -1;
750 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
751 if ((vlvf[i] & E1000_VLVF_VLANID_ENABLE) != 0 &&
752 (vlvf[i] & E1000_VLVF_VLANID_MASK) == vid) {
753 vlvf[i] |= 1U << (E1000_VLVF_POOLSEL_SHIFT + pool);
754 return (0);
755 }
756 if (free_slot == -1 &&
757 (vlvf[i] & E1000_VLVF_VLANID_ENABLE) == 0)
758 free_slot = i;
759 }
760 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
761 if ((old_vlvf[i] & E1000_VLVF_VLANID_ENABLE) != 0 &&
762 (old_vlvf[i] & E1000_VLVF_VLANID_MASK) == vid &&
763 (vlvf[i] & E1000_VLVF_VLANID_ENABLE) == 0) {
764 free_slot = i;
765 break;
766 }
767 if (preserve_only && (i == E1000_VLVF_ARRAY_SIZE))
768 return (ENOENT);
769 if (free_slot == -1)
770 return (ENOSPC);
771
772 vlvf[free_slot] = E1000_VLVF_VLANID_ENABLE | vid |
773 (1U << (E1000_VLVF_POOLSEL_SHIFT + pool));
774 return (0);
775 }
776
777 void
igb_iov_rebuild_vlan(struct e1000_softc * sc)778 igb_iov_rebuild_vlan(struct e1000_softc *sc)
779 {
780 struct e1000_hw *hw;
781 struct igb_vf *vf;
782 u32 old_vlvf[E1000_VLVF_ARRAY_SIZE];
783 u32 effective_vfta[EM_VFTA_SIZE], vfta[EM_VFTA_SIZE];
784 u32 vlvf[E1000_VLVF_ARRAY_SIZE];
785 u32 old_vfta, rctl, vmolr;
786 bool force_vfta, pf_overflow, pf_vlan_promisc, preserve_pf;
787 bool vfta_changed, vlvf_changed;
788 int i, vid;
789
790 if (!igb_iov_enabled(sc))
791 return;
792
793 hw = &sc->hw;
794 rctl = E1000_READ_REG(hw, E1000_RCTL);
795 rctl &= ~E1000_RCTL_CFIEN;
796 rctl |= E1000_RCTL_VFE;
797 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
798 memcpy(vfta, sc->shadow_vfta, sizeof(vfta));
799 memset(vlvf, 0, sizeof(vlvf));
800 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
801 old_vlvf[i] = E1000_READ_REG(hw, E1000_VLVF(i));
802
803 pf_vlan_promisc = igb_iov_pf_vlan_promisc(sc);
804 pf_overflow = !pf_vlan_promisc && hw->mac.type == e1000_i350 &&
805 igb_iov_vlan_unique_count(sc, true) > E1000_VLVF_ARRAY_SIZE;
806 preserve_pf = !pf_vlan_promisc && !pf_overflow;
807
808 /* First keep every surviving VF mapping in its current slot. */
809 for (i = 0; i < sc->num_vfs; i++) {
810 vf = &sc->vfs[i];
811 if (!(vf->flags & IGB_VF_ACTIVE))
812 continue;
813 for (vid = 0; vid < 4096; vid++) {
814 if ((vf->vlans[vid >> 5] & (1U << (vid & 0x1f))) ==
815 0)
816 continue;
817 (void)igb_iov_vlvf_add(vlvf, old_vlvf, vid,
818 vf->pool, true);
819 }
820 }
821
822 /*
823 * Preserve PF mappings unless I350 needs their slots for VFs.
824 * PF-only VLANs on 82576 intentionally have no VLVF mapping and
825 * reach the default PF pool after passing the global VFTA.
826 */
827 if (preserve_pf)
828 for (vid = 0; vid < 4096; vid++) {
829 if ((sc->shadow_vfta[vid >> 5] &
830 (1U << (vid & 0x1f))) == 0)
831 continue;
832 if (hw->mac.type == e1000_82576 &&
833 !igb_iov_vlan_present(sc, vid, false))
834 continue;
835 (void)igb_iov_vlvf_add(vlvf, old_vlvf, vid,
836 sc->pool, true);
837 }
838
839 /* Allocate new VF mappings before PF mappings. */
840 for (i = 0; i < sc->num_vfs; i++) {
841 vf = &sc->vfs[i];
842 if (!(vf->flags & IGB_VF_ACTIVE))
843 continue;
844 for (vid = 0; vid < 4096; vid++) {
845 if ((vf->vlans[vid >> 5] & (1U << (vid & 0x1f))) ==
846 0)
847 continue;
848 if (igb_iov_vlvf_add(vlvf, old_vlvf, vid,
849 vf->pool, false) == 0)
850 vfta[vid >> 5] |= 1U << (vid & 0x1f);
851 }
852 igb_iov_configure_vmolr(sc, vf);
853 }
854 if (!pf_vlan_promisc)
855 for (vid = 0; vid < 4096; vid++) {
856 if ((sc->shadow_vfta[vid >> 5] &
857 (1U << (vid & 0x1f))) == 0)
858 continue;
859 /*
860 * With no VLVF match, 82576 sends a globally admitted
861 * VLAN to the default PF pool. A VLVF entry is needed
862 * only when this VLAN is also assigned to a VF.
863 */
864 if (hw->mac.type == e1000_82576 &&
865 !igb_iov_vlan_present(sc, vid, false))
866 continue;
867 if (igb_iov_vlvf_add(vlvf, old_vlvf, vid,
868 sc->pool, false) != 0)
869 pf_overflow = true;
870 }
871
872 if (pf_vlan_promisc) {
873 memset(vfta, 0xff, sizeof(vfta));
874 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
875 if ((vlvf[i] & E1000_VLVF_VLANID_ENABLE) != 0)
876 vlvf[i] |= 1U <<
877 (E1000_VLVF_POOLSEL_SHIFT + sc->pool);
878 }
879
880 /*
881 * Establish the PF fallback before an overflowing I350 rebuild can
882 * displace one of its old VLVF mappings.
883 */
884 vmolr = E1000_READ_REG(hw, E1000_VMOLR(sc->pool));
885 vmolr &= ~E1000_VMOLR_VPE;
886 if (hw->mac.type == e1000_i350 &&
887 (pf_overflow || pf_vlan_promisc))
888 vmolr |= E1000_VMOLR_VPE;
889 E1000_WRITE_REG(hw, E1000_VMOLR(sc->pool), vmolr);
890
891 /*
892 * Remove global VFTA membership before removing a VLAN entirely, and
893 * add a VLVF mapping before globally admitting a new VF VLAN. A
894 * transition to a PF-only VLAN deliberately retains VFTA membership
895 * and falls through to the default PF pool.
896 */
897 force_vfta = hw->mac.type == e1000_i350 &&
898 !sc->iov_vfta_valid;
899 vfta_changed = false;
900 for (i = 0; i < EM_VFTA_SIZE; i++) {
901 /*
902 * I350 erratum 20 makes VFTA reads unreliable while VMDq
903 * loopback or anti-spoofing is active. Its ten-write
904 * workaround is already in e1000_write_vfta_i350(). Force a
905 * complete clear when the authoritative shadow is invalid;
906 * 82576 can safely diff against its live register contents.
907 */
908 if (hw->mac.type == e1000_i350)
909 old_vfta = force_vfta ? 0 : sc->iov_vfta[i];
910 else
911 old_vfta =
912 E1000_READ_REG_ARRAY(hw, E1000_VFTA, i);
913 effective_vfta[i] = old_vfta & vfta[i];
914 if (force_vfta || effective_vfta[i] != old_vfta) {
915 SDT_PROBE3(igb_iov, vlan, rebuild, vfta_clear,
916 sc, i, effective_vfta[i]);
917 e1000_write_vfta(hw, i, effective_vfta[i]);
918 vfta_changed = true;
919 }
920 }
921 if (vfta_changed)
922 E1000_WRITE_FLUSH(hw);
923 vlvf_changed = false;
924 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
925 if (vlvf[i] != old_vlvf[i]) {
926 SDT_PROBE3(igb_iov, vlan, rebuild, vlvf_write,
927 sc, i, vlvf[i]);
928 E1000_WRITE_REG(hw, E1000_VLVF(i), vlvf[i]);
929 vlvf_changed = true;
930 }
931 if (vlvf_changed)
932 E1000_WRITE_FLUSH(hw);
933 vfta_changed = false;
934 for (i = 0; i < EM_VFTA_SIZE; i++)
935 if (vfta[i] != effective_vfta[i]) {
936 SDT_PROBE3(igb_iov, vlan, rebuild, vfta_set,
937 sc, i, vfta[i]);
938 e1000_write_vfta(hw, i, vfta[i]);
939 vfta_changed = true;
940 }
941 if (vfta_changed)
942 E1000_WRITE_FLUSH(hw);
943 SDT_PROBE3(igb_iov, vlan, rebuild, state, sc, vfta, vlvf);
944 igb_iov_vfta_shadow_store(sc, vfta);
945 }
946
947 static bool
igb_iov_vlan_present(struct e1000_softc * sc,u16 vid,bool include_pf)948 igb_iov_vlan_present(struct e1000_softc *sc, u16 vid, bool include_pf)
949 {
950 int i;
951
952 if (include_pf &&
953 (sc->shadow_vfta[vid >> 5] & (1U << (vid & 0x1f))) != 0)
954 return (true);
955 for (i = 0; i < sc->num_vfs; i++)
956 if ((sc->vfs[i].flags & IGB_VF_ACTIVE) != 0 &&
957 (sc->vfs[i].vlans[vid >> 5] &
958 (1U << (vid & 0x1f))) != 0)
959 return (true);
960 return (false);
961 }
962
963 static int
igb_iov_vlan_unique_count(struct e1000_softc * sc,bool include_pf)964 igb_iov_vlan_unique_count(struct e1000_softc *sc, bool include_pf)
965 {
966 u32 vlans;
967 int count, i, word;
968
969 count = 0;
970 for (word = 0; word < EM_VFTA_SIZE; word++) {
971 vlans = include_pf ? sc->shadow_vfta[word] : 0;
972 for (i = 0; i < sc->num_vfs; i++)
973 if ((sc->vfs[i].flags & IGB_VF_ACTIVE) != 0)
974 vlans |= sc->vfs[i].vlans[word];
975 count += bitcount32(vlans);
976 }
977 return (count);
978 }
979
980 static int
igb_iov_set_vlan(struct e1000_softc * sc,struct igb_vf * vf,u16 vid,bool add)981 igb_iov_set_vlan(struct e1000_softc *sc, struct igb_vf *vf, u16 vid,
982 bool add)
983 {
984 u32 bit;
985 bool present;
986
987 bit = 1U << (vid & 0x1f);
988 present = (vf->vlans[vid >> 5] & bit) != 0;
989 if (vid == 0) {
990 if (!present) {
991 vf->vlans[0] |= 1U;
992 igb_iov_rebuild_vlan(sc);
993 }
994 return (0);
995 }
996 if (add == present)
997 return (0);
998
999 /*
1000 * Removals always reduce privilege and remain available. Charge only
1001 * additions, which a hostile VF must alternate with removals to force
1002 * repeated global VLAN rebuilds.
1003 */
1004 if (add && !igb_iov_vlan_present(sc, vid, false) &&
1005 igb_iov_vlan_unique_count(sc, false) >=
1006 E1000_VLVF_ARRAY_SIZE)
1007 return (ENOSPC);
1008 if (add && !igb_iov_vlan_add_allowed(vf))
1009 return (EBUSY);
1010
1011 if (add) {
1012 vf->vlans[vid >> 5] |= bit;
1013 vf->vlan_count++;
1014 } else {
1015 vf->vlans[vid >> 5] &= ~bit;
1016 vf->vlan_count--;
1017 }
1018 igb_iov_rebuild_vlan(sc);
1019 return (0);
1020 }
1021
1022 static void
igb_iov_reset_vf_state(struct e1000_softc * sc,struct igb_vf * vf)1023 igb_iov_reset_vf_state(struct e1000_softc *sc, struct igb_vf *vf)
1024 {
1025 bool update_uta;
1026
1027 update_uta = (vf->flags & IGB_VF_UCAST_PROMISC) != 0;
1028 vf->flags &= ~(IGB_VF_CTS | IGB_VF_UCAST_PROMISC |
1029 IGB_VF_MCAST_PROMISC | IGB_VF_MCAST_OVERFLOW |
1030 IGB_VF_MBX_PENDING | IGB_VF_MBX_GAVE_UP |
1031 IGB_VF_MDD_NOTIFY_PENDING);
1032 vf->mbx_retry_at = 0;
1033 vf->mdd_notify_at = 0;
1034 vf->mbx_retry_count = 0;
1035 /*
1036 * A reset starts a new mailbox epoch. Permit one immediate NACK so a
1037 * premature non-reset request does not wait for its posted-read
1038 * timeout.
1039 */
1040 memset(&vf->last_nack, 0, sizeof(vf->last_nack));
1041 vf->max_frame_size = ETHER_MAX_LEN;
1042 vf->mc_count = 0;
1043 vf->vlan_count = 0;
1044 memset(vf->mc_hashes, 0, sizeof(vf->mc_hashes));
1045 memset(vf->vlans, 0, sizeof(vf->vlans));
1046 /* Preserve the administrative access VLAN across VF and PF resets. */
1047 if (vf->default_vlan == 0)
1048 vf->vlans[0] = 1U;
1049 else {
1050 vf->vlans[vf->default_vlan >> 5] =
1051 1U << (vf->default_vlan & 0x1f);
1052 vf->vlan_count = 1;
1053 }
1054 igb_iov_configure_vmolr(sc, vf);
1055 if (update_uta)
1056 igb_iov_set_uta(sc);
1057 }
1058
1059 static bool
igb_iov_vf_vlan_is_default(const struct igb_vf * vf)1060 igb_iov_vf_vlan_is_default(const struct igb_vf *vf)
1061 {
1062 u32 expected;
1063 int i;
1064
1065 for (i = 0; i < EM_VFTA_SIZE; i++) {
1066 expected = 0;
1067 if (i == vf->default_vlan >> 5)
1068 expected = 1U << (vf->default_vlan & 0x1f);
1069 if (vf->vlans[i] != expected)
1070 return (false);
1071 }
1072 return (true);
1073 }
1074
1075 static bool
igb_iov_sanitize_vf_queues(struct e1000_softc * sc,struct igb_vf * vf)1076 igb_iov_sanitize_vf_queues(struct e1000_softc *sc,
1077 struct igb_vf *vf)
1078 {
1079 struct e1000_hw *hw;
1080 u16 qid[IGB_IOV_VF_QUEUES_MAX];
1081 u32 rxdctl, txdctl;
1082 int i, nqueues, retry;
1083
1084 hw = &sc->hw;
1085 switch (hw->mac.type) {
1086 case e1000_82576:
1087 nqueues = IGB_82576_VF_QUEUES;
1088 qid[0] = vf->pool;
1089 qid[1] = vf->pool + IGB_82576_VF_QUEUE_STRIDE;
1090 break;
1091 case e1000_i350:
1092 nqueues = IGB_I350_VF_QUEUES;
1093 qid[0] = vf->pool;
1094 break;
1095 default:
1096 return (true);
1097 }
1098
1099 /*
1100 * I350 maps pool n to queue n. 82576 gives VF n physical queues n
1101 * and n + 8, so both retained queue configurations must be cleared.
1102 */
1103 for (i = 0; i < nqueues; i++)
1104 KASSERT(qid[i] < (hw->mac.type == e1000_82576 ?
1105 IGB_82576_NUM_QUEUES : IGB_I350_NUM_QUEUES),
1106 ("%s: invalid VF queue %u", __func__, qid[i]));
1107
1108 /*
1109 * The 82576 and I350 specification updates, Software Clarification 3,
1110 * note that VFLR does not reset the VF queue configuration. Clear the
1111 * PF-programmable state before acknowledging the reset so a new VF
1112 * owner cannot inherit it, particularly a descriptor-head write-back
1113 * DMA address. The new VF driver initializes its active ring pointers
1114 * during queue setup.
1115 *
1116 * Disable every queue first, then wait for outstanding DMA activity to
1117 * stop before clearing TDWBAL/H and the remaining retained state.
1118 * Spin only for the normal fast transition, then sleep so a VF that
1119 * keeps asserting QUEUE_ENABLE cannot busy-wait the PF for 10 ms.
1120 */
1121 for (i = 0; i < nqueues; i++) {
1122 E1000_WRITE_REG(hw, E1000_RXDCTL(qid[i]), 0);
1123 E1000_WRITE_REG(hw, E1000_TXDCTL(qid[i]), 0);
1124 }
1125 E1000_WRITE_FLUSH(hw);
1126 for (retry = 0; retry < IGB_IOV_QUEUE_DISABLE_RETRIES; retry++) {
1127 for (i = 0; i < nqueues; i++) {
1128 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(qid[i]));
1129 txdctl = E1000_READ_REG(hw, E1000_TXDCTL(qid[i]));
1130 if ((rxdctl & E1000_RXDCTL_QUEUE_ENABLE) != 0 ||
1131 (txdctl & E1000_TXDCTL_QUEUE_ENABLE) != 0)
1132 break;
1133 }
1134 if (i == nqueues)
1135 break;
1136 if (retry + 1 < IGB_IOV_QUEUE_DISABLE_RETRIES) {
1137 if (retry < IGB_IOV_QUEUE_DISABLE_BUSY_RETRIES)
1138 DELAY(IGB_IOV_QUEUE_DISABLE_DELAY_US);
1139 else
1140 pause_sbt("igbqds",
1141 IGB_IOV_QUEUE_DISABLE_PAUSE, 0,
1142 C_PREL(1));
1143 }
1144 }
1145 if (retry == IGB_IOV_QUEUE_DISABLE_RETRIES) {
1146 if (ratecheck(&vf->last_queue_log,
1147 &igb_iov_mbx_log_interval))
1148 device_printf(sc->dev,
1149 "could not disable queues for VF %u; "
1150 "reset deferred\n", vf->pool);
1151 return (false);
1152 }
1153
1154 for (i = 0; i < nqueues; i++) {
1155 E1000_WRITE_REG(hw, E1000_SRRCTL(qid[i]), 0);
1156 E1000_WRITE_REG(hw, E1000_DCA_RXCTRL(qid[i]), 0);
1157 E1000_WRITE_REG(hw, E1000_TDWBAL(qid[i]), 0);
1158 E1000_WRITE_REG(hw, E1000_TDWBAH(qid[i]), 0);
1159 E1000_WRITE_REG(hw, E1000_DCA_TXCTRL(qid[i]), 0);
1160 }
1161 E1000_WRITE_REG(hw, E1000_PSRTYPE(vf->pool), 0);
1162 E1000_WRITE_FLUSH(hw);
1163 return (true);
1164 }
1165
1166 static bool
igb_iov_reset_event_common(struct e1000_softc * sc,struct igb_vf * vf,bool reset_intrs)1167 igb_iov_reset_event_common(struct e1000_softc *sc, struct igb_vf *vf,
1168 bool reset_intrs)
1169 {
1170 struct e1000_hw *hw;
1171 bool rebuild_mta, rebuild_vlan, sanitized;
1172 u32 reg;
1173
1174 hw = &sc->hw;
1175 rebuild_mta = vf->mc_count != 0;
1176 rebuild_vlan = !igb_iov_vf_vlan_is_default(vf);
1177 reg = E1000_READ_REG(hw, E1000_VFTE);
1178 E1000_WRITE_REG(hw, E1000_VFTE, reg & ~(1U << vf->pool));
1179 reg = E1000_READ_REG(hw, E1000_VFRE);
1180 E1000_WRITE_REG(hw, E1000_VFRE, reg & ~(1U << vf->pool));
1181 if (reset_intrs)
1182 E1000_WRITE_REG(hw, E1000_VTCTRL(vf->pool),
1183 E1000_VTCTRL_RST);
1184 sanitized = igb_iov_sanitize_vf_queues(sc, vf);
1185 E1000_WRITE_REG(hw, E1000_VMVIR(vf->pool), 0);
1186 igb_iov_clear_mac_filters(sc, vf);
1187 igb_iov_clear_rar(sc, vf->rar_index);
1188 igb_iov_reset_vf_state(sc, vf);
1189 if (rebuild_mta)
1190 igb_iov_rebuild_mta(sc);
1191 if (rebuild_vlan)
1192 igb_iov_rebuild_vlan(sc);
1193 return (sanitized);
1194 }
1195
1196 static bool
igb_iov_reset_event(struct e1000_softc * sc,struct igb_vf * vf)1197 igb_iov_reset_event(struct e1000_softc *sc, struct igb_vf *vf)
1198 {
1199 return (igb_iov_reset_event_common(sc, vf, true));
1200 }
1201
1202 static void
igb_iov_mdd_reset_event(struct e1000_softc * sc,struct igb_vf * vf)1203 igb_iov_mdd_reset_event(struct e1000_softc *sc, struct igb_vf *vf)
1204 {
1205 /*
1206 * VTCTRL.RST clears the VF's queue-enable and interrupt registers
1207 * (I350 section 8.28.1). It therefore also removes the admin-vector
1208 * route needed to deliver the reset notification below. MDD recovery
1209 * explicitly permits toggling VFTE instead (section 7.8.3.8.3).
1210 *
1211 * Leave the interrupt registers intact, keep VFTE/VFRE disabled until
1212 * the VF completes a new reset handshake, and use the no-CTS control
1213 * message to make the guest reinitialize. FreeBSD and DPDK consume
1214 * that message directly; Linux ACKs it and the PF's non-CTS ACK path
1215 * replies with the NACK that schedules igbvf's reset task.
1216 *
1217 * Sanitization failure leaves the pool disabled. The VF reset
1218 * handshake retries it and is NACKed while a queue remains active.
1219 */
1220 (void)igb_iov_reset_event_common(sc, vf, false);
1221 }
1222
1223 static void
igb_iov_reset_msg(struct e1000_softc * sc,struct igb_vf * vf)1224 igb_iov_reset_msg(struct e1000_softc *sc, struct igb_vf *vf)
1225 {
1226 struct e1000_hw *hw;
1227 u32 msg[3], reg;
1228
1229 hw = &sc->hw;
1230 if (!igb_iov_reset_event(sc, vf)) {
1231 msg[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_NACK;
1232 e1000_write_mbx(hw, msg, 1, vf->pool);
1233 return;
1234 }
1235 igb_iov_map_rar(sc, vf->rar_index, vf->mac, vf->pool);
1236 igb_iov_set_anti_spoof(sc, vf);
1237
1238 reg = E1000_READ_REG(hw, E1000_VFTE);
1239 E1000_WRITE_REG(hw, E1000_VFTE, reg | (1U << vf->pool));
1240 reg = E1000_READ_REG(hw, E1000_VFRE);
1241 E1000_WRITE_REG(hw, E1000_VFRE, reg | (1U << vf->pool));
1242 /*
1243 * 82576's WVBR blocked bitmap is read-clear, so the reset handshake
1244 * completes that event's lifetime. I350 MDFB might be read-only;
1245 * re-arm its edge latch only after a valid MDFB sample reads clear.
1246 */
1247 if (hw->mac.type == e1000_82576)
1248 vf->flags &= ~IGB_VF_MDD_BLOCKED;
1249 vf->flags |= IGB_VF_CTS;
1250
1251 memset(msg, 0, sizeof(msg));
1252 msg[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
1253 memcpy(&msg[1], vf->mac, ETHER_ADDR_LEN);
1254 e1000_write_mbx(hw, msg, 3, vf->pool);
1255 }
1256
1257 static int
igb_iov_set_mac_filter(struct e1000_softc * sc,struct igb_vf * vf,u32 * msg)1258 igb_iov_set_mac_filter(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg)
1259 {
1260 struct igb_vf_mac_filter *filter, *free_filter;
1261 const u8 *mac;
1262 u32 info;
1263 int count, i;
1264
1265 info = msg[0] & E1000_VT_MSGINFO_MASK;
1266 if (info == E1000_VF_MAC_FILTER_CLR) {
1267 igb_iov_clear_mac_filters(sc, vf);
1268 return (0);
1269 }
1270 if (info != E1000_VF_MAC_FILTER_ADD)
1271 return (EINVAL);
1272 if ((vf->flags & IGB_VF_CAP_MAC) == 0)
1273 return (EPERM);
1274
1275 mac = (const u8 *)&msg[1];
1276 if (!igb_iov_mac_valid(mac))
1277 return (EINVAL);
1278 if (memcmp(mac, vf->mac, ETHER_ADDR_LEN) == 0)
1279 return (0);
1280
1281 count = 0;
1282 free_filter = NULL;
1283 for (i = 0; i < sc->num_vf_mac_filters; i++) {
1284 filter = &sc->vf_mac_filters[i];
1285 if (!filter->active) {
1286 if (free_filter == NULL)
1287 free_filter = filter;
1288 continue;
1289 }
1290 if (memcmp(filter->mac, mac, ETHER_ADDR_LEN) != 0)
1291 continue;
1292 return (filter->pool == vf->pool ? 0 : EADDRINUSE);
1293 }
1294 for (i = 0; i < sc->num_vf_mac_filters; i++)
1295 if (sc->vf_mac_filters[i].active &&
1296 sc->vf_mac_filters[i].pool == vf->pool)
1297 count++;
1298 if (igb_iov_mac_in_use(sc, mac, vf))
1299 return (EADDRINUSE);
1300 if (count >= IGB_IOV_MAX_MAC_FILTERS)
1301 return (ENOSPC);
1302 if (free_filter == NULL)
1303 return (ENOSPC);
1304
1305 free_filter->active = true;
1306 free_filter->pool = vf->pool;
1307 memcpy(free_filter->mac, mac, ETHER_ADDR_LEN);
1308 igb_iov_map_rar(sc, free_filter->rar_index, free_filter->mac, vf->pool);
1309 return (0);
1310 }
1311
1312 static int
igb_iov_set_mac(struct e1000_softc * sc,struct igb_vf * vf,u32 * msg)1313 igb_iov_set_mac(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg)
1314 {
1315 u8 *mac;
1316
1317 if ((msg[0] & E1000_VT_MSGINFO_MASK) != 0)
1318 return (igb_iov_set_mac_filter(sc, vf, msg));
1319
1320 mac = (u8 *)&msg[1];
1321 if (!igb_iov_mac_valid(mac))
1322 return (EINVAL);
1323 if (memcmp(mac, vf->mac, ETHER_ADDR_LEN) != 0 &&
1324 !(vf->flags & IGB_VF_CAP_MAC))
1325 return (EPERM);
1326 if (memcmp(mac, vf->mac, ETHER_ADDR_LEN) != 0 &&
1327 igb_iov_mac_in_use(sc, mac, vf))
1328 return (EADDRINUSE);
1329
1330 memcpy(vf->mac, mac, ETHER_ADDR_LEN);
1331 igb_iov_map_rar(sc, vf->rar_index, vf->mac, vf->pool);
1332 return (0);
1333 }
1334
1335 static int
igb_iov_set_multicast(struct e1000_softc * sc,struct igb_vf * vf,u32 * msg)1336 igb_iov_set_multicast(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg)
1337 {
1338 u16 hashes[IGB_IOV_MAX_MC_HASHES] = {};
1339 bool overflow;
1340 int count, i;
1341
1342 count = (msg[0] & E1000_VF_SET_MULTICAST_COUNT_MASK) >>
1343 E1000_VT_MSGINFO_SHIFT;
1344 overflow = count > IGB_IOV_MAX_MC_HASHES ||
1345 (msg[0] & E1000_VF_SET_MULTICAST_OVERFLOW) != 0;
1346 count = min(count, IGB_IOV_MAX_MC_HASHES);
1347 for (i = 0; i < count; i++)
1348 hashes[i] =
1349 (msg[1 + i / 2] >> ((i & 1) * 16)) & 0xffff;
1350 if (vf->mc_count == count &&
1351 ((vf->flags & IGB_VF_MCAST_OVERFLOW) != 0) == overflow &&
1352 memcmp(vf->mc_hashes, hashes, sizeof(hashes)) == 0)
1353 return (0);
1354 memcpy(vf->mc_hashes, hashes, sizeof(vf->mc_hashes));
1355 vf->mc_count = count;
1356 if (overflow)
1357 vf->flags |= IGB_VF_MCAST_OVERFLOW;
1358 else
1359 vf->flags &= ~IGB_VF_MCAST_OVERFLOW;
1360 if (overflow &&
1361 (vf->flags & IGB_VF_MCAST_OVERFLOW_WARNED) == 0) {
1362 vf->flags |= IGB_VF_MCAST_OVERFLOW_WARNED;
1363 device_printf(sc->dev,
1364 "VF %u multicast list exceeds 30 entries; "
1365 "enabling all-multicast reception\n", vf->pool);
1366 }
1367 igb_iov_configure_vmolr(sc, vf);
1368 igb_iov_rebuild_mta(sc);
1369 return (0);
1370 }
1371
1372 static int
igb_iov_set_lpe(struct e1000_softc * sc,struct igb_vf * vf,u32 * msg)1373 igb_iov_set_lpe(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg)
1374 {
1375 u32 size;
1376
1377 size = msg[1];
1378 if (size < ETHER_MIN_LEN)
1379 return (EINVAL);
1380 vf->max_frame_size = min(size, IGB_IOV_MAX_FRAME_SIZE);
1381 igb_iov_configure_vmolr(sc, vf);
1382 return (0);
1383 }
1384
1385 static int
igb_iov_set_promisc(struct e1000_softc * sc,struct igb_vf * vf,u32 msg)1386 igb_iov_set_promisc(struct e1000_softc *sc, struct igb_vf *vf, u32 msg)
1387 {
1388 u32 mode;
1389
1390 mode = msg & E1000_VT_MSGINFO_MASK;
1391 if (mode & ~(E1000_VF_SET_PROMISC_UNICAST |
1392 E1000_VF_SET_PROMISC_MULTICAST))
1393 return (EINVAL);
1394 if (mode != 0 && !(vf->flags & IGB_VF_ALLOW_PROMISC))
1395 return (EPERM);
1396
1397 vf->flags &= ~(IGB_VF_UCAST_PROMISC | IGB_VF_MCAST_PROMISC);
1398 if (mode & E1000_VF_SET_PROMISC_UNICAST)
1399 vf->flags |= IGB_VF_UCAST_PROMISC;
1400 if (mode & E1000_VF_SET_PROMISC_MULTICAST)
1401 vf->flags |= IGB_VF_MCAST_PROMISC;
1402 igb_iov_configure_vmolr(sc, vf);
1403 igb_iov_set_uta(sc);
1404 return (0);
1405 }
1406
1407 static bool
igb_iov_process_msg(struct e1000_softc * sc,struct igb_vf * vf)1408 igb_iov_process_msg(struct e1000_softc *sc, struct igb_vf *vf)
1409 {
1410 struct e1000_hw *hw;
1411 u32 msg[E1000_VFMAILBOX_SIZE], type;
1412 int error;
1413
1414 hw = &sc->hw;
1415 memset(msg, 0, sizeof(msg));
1416 if (e1000_read_mbx(hw, msg, nitems(msg), vf->pool, false) != 0)
1417 return (false);
1418 vf->flags &= ~IGB_VF_MBX_PENDING;
1419 vf->mbx_retry_at = 0;
1420 vf->mbx_retry_count = 0;
1421
1422 if (msg[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK)) {
1423 e1000_unlock_mbx(hw, vf->pool);
1424 return (true);
1425 }
1426 if (msg[0] == E1000_VF_RESET) {
1427 igb_iov_reset_msg(sc, vf);
1428 return (true);
1429 }
1430 if (!(vf->flags & IGB_VF_CTS)) {
1431 if (igb_iov_nack_allowed(vf)) {
1432 msg[0] = igb_iov_reply_header(msg[0], false, false);
1433 e1000_write_mbx(hw, msg, 1, vf->pool);
1434 } else
1435 e1000_unlock_mbx(hw, vf->pool);
1436 return (true);
1437 }
1438
1439 type = msg[0] & 0xffff;
1440 switch (type) {
1441 case E1000_VF_SET_MAC_ADDR:
1442 error = igb_iov_set_mac(sc, vf, msg);
1443 break;
1444 case E1000_VF_SET_MULTICAST:
1445 error = igb_iov_set_multicast(sc, vf, msg);
1446 break;
1447 case E1000_VF_SET_VLAN:
1448 if (vf->default_vlan != 0)
1449 error = EPERM;
1450 else if ((msg[1] & ~E1000_VLVF_VLANID_MASK) != 0)
1451 error = EINVAL;
1452 else
1453 error = igb_iov_set_vlan(sc, vf,
1454 msg[1] & E1000_VLVF_VLANID_MASK,
1455 (msg[0] & E1000_VF_SET_VLAN_ADD) != 0);
1456 break;
1457 case E1000_VF_SET_LPE:
1458 error = igb_iov_set_lpe(sc, vf, msg);
1459 break;
1460 case E1000_VF_SET_PROMISC:
1461 error = igb_iov_set_promisc(sc, vf, msg[0]);
1462 break;
1463 default:
1464 error = EOPNOTSUPP;
1465 break;
1466 }
1467
1468 msg[0] = igb_iov_reply_header(msg[0], true, error == 0);
1469 e1000_write_mbx(hw, msg, 1, vf->pool);
1470 return (true);
1471 }
1472
1473 static sbintime_t
igb_iov_service_pending_mbx(struct e1000_softc * sc,struct igb_vf * vf,sbintime_t now)1474 igb_iov_service_pending_mbx(struct e1000_softc *sc, struct igb_vf *vf,
1475 sbintime_t now)
1476 {
1477 sbintime_t delay;
1478
1479 if ((vf->flags & IGB_VF_MBX_PENDING) == 0)
1480 return (0);
1481 if (vf->mbx_retry_at != 0 && now < vf->mbx_retry_at)
1482 return (vf->mbx_retry_at);
1483 if (igb_iov_process_msg(sc, vf))
1484 return (0);
1485
1486 now = getsbinuptime();
1487 if (vf->mbx_retry_count < IGB_IOV_MBX_RETRY_COUNT) {
1488 delay = igb_iov_mbx_retry_delay[vf->mbx_retry_count++];
1489 vf->mbx_retry_at = now + delay;
1490 return (vf->mbx_retry_at);
1491 }
1492
1493 vf->flags &= ~(IGB_VF_CTS | IGB_VF_MBX_PENDING);
1494 vf->flags |= IGB_VF_MBX_GAVE_UP;
1495 vf->mbx_retry_at = 0;
1496 if (ratecheck(&vf->last_mbx_log, &igb_iov_mbx_log_interval))
1497 device_printf(sc->dev,
1498 "mailbox remained busy for VF %u; CTS revoked\n",
1499 vf->pool);
1500 return (0);
1501 }
1502
1503 void
igb_iov_handle_mbx(struct e1000_softc * sc)1504 igb_iov_handle_mbx(struct e1000_softc *sc)
1505 {
1506 struct e1000_hw *hw;
1507 struct igb_vf *vf;
1508 sbintime_t delay, next_retry_at, now, retry_at;
1509 u32 msg;
1510 int i;
1511
1512 if (!sc->iov_hw_active)
1513 return;
1514
1515 hw = &sc->hw;
1516 next_retry_at = 0;
1517 for (i = 0; i < sc->num_vfs; i++) {
1518 vf = &sc->vfs[i];
1519 if (!(vf->flags & IGB_VF_ACTIVE))
1520 continue;
1521 now = getsbinuptime();
1522 if (e1000_check_for_rst(hw, vf->pool) == 0) {
1523 /*
1524 * The old VF is gone. A new owner's reset handshake
1525 * reruns sanitization before enabling its pool.
1526 */
1527 (void)igb_iov_reset_event(sc, vf);
1528 }
1529 if ((vf->flags &
1530 (IGB_VF_MBX_PENDING | IGB_VF_MBX_GAVE_UP)) == 0 &&
1531 e1000_check_for_msg(hw, vf->pool) == 0) {
1532 vf->flags |= IGB_VF_MBX_PENDING;
1533 vf->mbx_retry_at = 0;
1534 vf->mbx_retry_count = 0;
1535 }
1536 retry_at = igb_iov_service_pending_mbx(sc, vf, now);
1537 if (retry_at != 0 &&
1538 (next_retry_at == 0 || retry_at < next_retry_at))
1539 next_retry_at = retry_at;
1540 if (e1000_check_for_ack(hw, vf->pool) == 0 &&
1541 !(vf->flags & IGB_VF_CTS) && igb_iov_nack_allowed(vf)) {
1542 msg = E1000_VT_MSGTYPE_NACK;
1543 e1000_write_mbx(hw, &msg, 1, vf->pool);
1544 }
1545 }
1546 if (next_retry_at != 0) {
1547 delay = next_retry_at - getsbinuptime();
1548 if (delay <= 0)
1549 delay = SBT_1MS;
1550 callout_reset_sbt(&sc->iov_mbx_retry, delay, 0,
1551 igb_iov_mbx_retry_callout, sc, C_PREL(1));
1552 }
1553 }
1554
1555 static bool
igb_iov_notify_vf_mdd_reset(struct e1000_softc * sc,struct igb_vf * vf)1556 igb_iov_notify_vf_mdd_reset(struct e1000_softc *sc, struct igb_vf *vf)
1557 {
1558 u32 msg;
1559
1560 /*
1561 * MDD recovery preserves the VF's admin-vector configuration. Send
1562 * the same no-CTS control message used for PF reset notification so
1563 * the VF discards its state and completes a new reset handshake.
1564 * A failed write is retried from the timer-driven admin pass; the VF's
1565 * transmit watchdog remains the final fallback when traffic is still
1566 * queued and notification never succeeds.
1567 */
1568 msg = E1000_PF_CONTROL_MSG;
1569 if (e1000_write_mbx(&sc->hw, &msg, 1, vf->pool) != 0) {
1570 vf->mdd_notify_at =
1571 getsbinuptime() + igb_iov_mdd_notify_retry;
1572 if (ratecheck(&vf->last_mbx_log,
1573 &igb_iov_mbx_log_interval))
1574 device_printf(sc->dev,
1575 "could not notify VF %u of malicious-driver "
1576 "reset; will retry\n", vf->pool);
1577 return (false);
1578 }
1579 vf->flags &= ~IGB_VF_MDD_NOTIFY_PENDING;
1580 vf->mdd_notify_at = 0;
1581 return (true);
1582 }
1583
1584 void
igb_iov_handle_mdd(struct e1000_softc * sc)1585 igb_iov_handle_mdd(struct e1000_softc *sc)
1586 {
1587 struct igb_vf *vf;
1588 u32 blocked, cleared, handled, lvmmc;
1589 u32 readback, spoofed;
1590 bool mdfb_valid, pending;
1591 int i;
1592
1593 pending = atomic_readandclear_32(&sc->iov_pending) != 0;
1594 lvmmc = pending ?
1595 atomic_readandclear_32(&sc->iov_mdd_cause) : 0;
1596 if (!sc->iov_hw_active) {
1597 atomic_readandclear_32(&sc->iov_spoof_pending);
1598 atomic_readandclear_32(&sc->iov_blocked_pending);
1599 return;
1600 }
1601
1602 blocked = 0;
1603 handled = 0;
1604 mdfb_valid = false;
1605 if (sc->hw.mac.type == e1000_i350) {
1606 u32 mdfb;
1607
1608 spoofed = atomic_readandclear_32(&sc->iov_spoof_pending);
1609 /*
1610 * I350 reports ordinary MAC/VLAN spoofing through the
1611 * interrupt-time LVMMC snapshot rather than WVBR. The
1612 * filter accumulates Last_Q into iov_spoof_pending so events
1613 * from different VFs coalesce safely until this timer-driven
1614 * admin pass.
1615 */
1616 spoofed &= IGB_I350_QUEUE_MASK;
1617 /*
1618 * Sample MDFB on every admin pass so a blocked queue is not
1619 * mislabeled as an ordinary spoof when no MDDET observation
1620 * is pending.
1621 */
1622 mdfb = E1000_READ_REG(&sc->hw, E1000_MDFB);
1623 if (__predict_false(mdfb == 0xffffffff))
1624 mdfb = 0;
1625 else {
1626 mdfb &= IGB_I350_QUEUE_MASK;
1627 mdfb_valid = true;
1628 }
1629 /*
1630 * I350 SDM sections 8.14.10 and 8.14.11: WVBR reports
1631 * spoof and malicious-driver events, while MDFB identifies
1632 * the queues actually blocked for malicious behavior.
1633 */
1634 spoofed &= ~mdfb;
1635 blocked = mdfb;
1636 if (blocked != 0 && lvmmc == 0)
1637 lvmmc = E1000_READ_REG(&sc->hw, E1000_LVMMC);
1638 /*
1639 * A failed diagnostic read does not invalidate the
1640 * blocked-queue bitmap that was read successfully above.
1641 */
1642 if (__predict_false(lvmmc == 0xffffffff))
1643 lvmmc = 0;
1644 /*
1645 * MDFB is authoritative for queues stopped by malicious-driver
1646 * detection. LVMMC reports causes such as VLAN IERR and
1647 * Mal_PF, but its Last_Q field does not establish that a queue
1648 * was blocked. Do not manufacture a blocked bit when MDFB is
1649 * clear.
1650 */
1651 } else {
1652 if (!pending)
1653 return;
1654 /*
1655 * WVBR is read-clear and does not preserve every queue across
1656 * multiple MDDET interrupts. The interrupt filter snapshots and
1657 * accumulates its pool bitmaps before this deferred admin pass.
1658 */
1659 spoofed = atomic_readandclear_32(&sc->iov_spoof_pending);
1660 blocked = atomic_readandclear_32(&sc->iov_blocked_pending);
1661 /* A blocked-queue classification dominates its WVBR low bit. */
1662 spoofed &= ~blocked;
1663 }
1664
1665 for (i = 0; i < sc->num_vfs; i++) {
1666 vf = &sc->vfs[i];
1667 if (!(vf->flags & IGB_VF_ACTIVE))
1668 continue;
1669 if ((vf->flags & IGB_VF_MDD_NOTIFY_PENDING) != 0 &&
1670 getsbinuptime() >= vf->mdd_notify_at)
1671 (void)igb_iov_notify_vf_mdd_reset(sc, vf);
1672 /*
1673 * An invalid MDFB sample must neither report a new edge nor
1674 * masquerade as evidence that an old edge has cleared.
1675 */
1676 if (sc->hw.mac.type == e1000_i350 && mdfb_valid &&
1677 (blocked & (1U << i)) == 0)
1678 vf->flags &= ~IGB_VF_MDD_BLOCKED;
1679 if ((spoofed & (1U << i)) != 0 &&
1680 ratecheck(&vf->last_spoof_log,
1681 &igb_iov_spoof_log_interval))
1682 device_printf(sc->dev,
1683 "spoof event detected from VF %u; packet dropped\n",
1684 vf->pool);
1685 if ((blocked & (1U << i)) == 0)
1686 continue;
1687 if ((vf->flags & IGB_VF_MDD_BLOCKED) != 0)
1688 continue;
1689 vf->flags |= IGB_VF_MDD_BLOCKED;
1690 if (ratecheck(&vf->last_mdd_log, &igb_iov_mdd_log_interval))
1691 device_printf(sc->dev,
1692 "malicious-driver event 0x%08x from VF %u; "
1693 "resetting VF\n", lvmmc, vf->pool);
1694 igb_iov_mdd_reset_event(sc, vf);
1695 vf->flags |= IGB_VF_MDD_NOTIFY_PENDING;
1696 (void)igb_iov_notify_vf_mdd_reset(sc, vf);
1697 handled |= 1U << i;
1698 }
1699 if (sc->hw.mac.type == e1000_i350 && mdfb_valid &&
1700 (blocked & (1U << sc->pool)) == 0)
1701 sc->iov_pf_mdd_blocked = false;
1702 if ((blocked & (1U << sc->pool)) != 0 &&
1703 (sc->hw.mac.type != e1000_i350 ||
1704 !sc->iov_pf_mdd_blocked)) {
1705 if (sc->hw.mac.type == e1000_i350)
1706 sc->iov_pf_mdd_blocked = true;
1707 if (ratecheck(&sc->iov_last_mdd_log,
1708 &igb_iov_mdd_log_interval))
1709 device_printf(sc->dev,
1710 "malicious-driver event 0x%08x from PF queue; "
1711 "resetting PF\n", lvmmc);
1712 iflib_request_reset(sc->ctx);
1713 iflib_admin_intr_deferred(sc->ctx);
1714 handled |= 1U << sc->pool;
1715 }
1716 if (sc->hw.mac.type == e1000_i350 && handled != 0) {
1717 /*
1718 * I350 documentation conflicts: the register summary calls
1719 * MDFB RWS while the detailed field table calls it RO. I350
1720 * silicon clears a blocked bit when software writes it back.
1721 * Write only bits whose recovery was initiated. If a revision
1722 * instead implements MDFB as RO, the edge latch above prevents
1723 * a reset loop and this one transition-time write is harmless.
1724 */
1725 E1000_WRITE_REG(&sc->hw, E1000_MDFB, handled);
1726 E1000_WRITE_FLUSH(&sc->hw);
1727 /*
1728 * Rearm from observed hardware state instead of waiting for
1729 * the next admin pass. The PF context lock prevents a reset
1730 * handshake from re-enabling the VF before this readback. A
1731 * write-to-clear part reports zero; a read-only part retains
1732 * the bit and therefore retains the one-shot edge latch.
1733 */
1734 readback = E1000_READ_REG(&sc->hw, E1000_MDFB);
1735 if (__predict_false(readback == 0xffffffff))
1736 cleared = 0;
1737 else
1738 cleared = handled &
1739 ~(readback & IGB_I350_QUEUE_MASK);
1740 for (i = 0; i < sc->num_vfs; i++)
1741 if ((cleared & (1U << i)) != 0)
1742 sc->vfs[i].flags &= ~IGB_VF_MDD_BLOCKED;
1743 if ((cleared & (1U << sc->pool)) != 0)
1744 sc->iov_pf_mdd_blocked = false;
1745 }
1746 if (sc->hw.mac.type == e1000_i350) {
1747 /*
1748 * I350 can retain EICR.OTHER without delivering the admin MSI-X
1749 * even though its EIMS and legacy IMS bits remain enabled. Kick
1750 * the already-enabled vector on each admin pass so its filter
1751 * consumes any retained ICR/LVMMC cause. A synthetic interrupt
1752 * with no legacy cause is handled entirely by the filter.
1753 */
1754 E1000_WRITE_REG(&sc->hw, E1000_EICS, sc->link_mask);
1755 E1000_WRITE_FLUSH(&sc->hw);
1756 }
1757 }
1758
1759 void
igb_iov_mdd_event(struct e1000_softc * sc)1760 igb_iov_mdd_event(struct e1000_softc *sc)
1761 {
1762 u32 blocked, cause, queues, queue, spoofed, wvbr;
1763
1764 /*
1765 * LVMMC is clear-on-read. Preserve it in the interrupt filter, as
1766 * Intel's igb driver does, rather than deferring the only copy.
1767 */
1768 cause = E1000_READ_REG(&sc->hw, E1000_LVMMC);
1769 if (__predict_false(cause == 0xffffffff))
1770 return;
1771 if (sc->hw.mac.type == e1000_82576) {
1772 /*
1773 * Snapshot WVBR in the interrupt filter. Waiting for the admin
1774 * task loses all but the last of back-to-back VF MDD events on
1775 * 82576. Convert the staggered queue map into pool bits and OR
1776 * each observation into software latches for deferred recovery.
1777 */
1778 wvbr = E1000_READ_REG(&sc->hw, E1000_WVBR);
1779 if (__predict_false(wvbr == 0xffffffff)) {
1780 spoofed = 0;
1781 blocked = 0;
1782 } else {
1783 queues = wvbr & IGB_82576_QUEUE_MASK;
1784 spoofed = (queues & 0xff) |
1785 (queues >> IGB_82576_STAGGERED_QUEUE_SHIFT);
1786 queues = (wvbr >> 16) & IGB_82576_QUEUE_MASK;
1787 blocked = (queues & 0xff) |
1788 (queues >> IGB_82576_STAGGERED_QUEUE_SHIFT);
1789 }
1790 SDT_PROBE4(igb_iov, mdd, sample, wvbr, sc, wvbr, spoofed,
1791 blocked);
1792 /*
1793 * 82576 can report a coalesced block-class event with all affected
1794 * queues in WVBR's low half and no high-half blocked bits. If an
1795 * ordinary spoof shares that snapshot, the register has no per-queue
1796 * cause information. Deliberately fail closed by recovering every
1797 * low-half queue; this can reset a spoof-only sibling, but avoids
1798 * stranding a blocked VF. LVMMC.Last_Q identifies only the final
1799 * event and lost simultaneous blocked VFs on tested silicon.
1800 */
1801 if (blocked == 0 &&
1802 (cause & IGB_82576_LVMMC_BLOCK_MASK) != 0) {
1803 blocked = spoofed;
1804 if (blocked == 0) {
1805 queue = (cause >> 16) & 0xf;
1806 blocked = 1U << (queue & 0x7);
1807 }
1808 }
1809 if (spoofed != 0)
1810 atomic_set_32(&sc->iov_spoof_pending, spoofed);
1811 if (blocked != 0)
1812 atomic_set_32(&sc->iov_blocked_pending, blocked);
1813 }
1814 if (sc->hw.mac.type == e1000_i350 &&
1815 (cause & IGB_I350_LVMMC_MAC_VLAN_SPOOF) != 0) {
1816 queue = (cause >> IGB_I350_LVMMC_LAST_Q_SHIFT) &
1817 IGB_I350_LVMMC_LAST_Q_MASK;
1818 /*
1819 * FreeBSD assigns one queue to each VF pool, so Last_Q is
1820 * also the VF number. Preserve all VFs observed before the
1821 * timer pass, and do not overwrite an unrelated blocked
1822 * queue's diagnostic with this non-blocking spoof event.
1823 */
1824 atomic_set_32(&sc->iov_spoof_pending, 1U << queue);
1825 return;
1826 }
1827 atomic_store_rel_32(&sc->iov_mdd_cause, cause);
1828 atomic_set_32(&sc->iov_pending, 1);
1829 }
1830
1831 void
igb_iov_ping_all_vfs(struct e1000_softc * sc)1832 igb_iov_ping_all_vfs(struct e1000_softc *sc)
1833 {
1834 struct igb_vf *vf;
1835 u32 msg;
1836 int i;
1837
1838 if (!sc->iov_hw_active)
1839 return;
1840
1841 for (i = 0; i < sc->num_vfs; i++) {
1842 vf = &sc->vfs[i];
1843 if (!(vf->flags & IGB_VF_ACTIVE))
1844 continue;
1845 msg = E1000_PF_CONTROL_MSG;
1846 if (vf->flags & IGB_VF_CTS)
1847 msg |= E1000_VT_MSGTYPE_CTS;
1848 e1000_write_mbx(&sc->hw, &msg, 1, vf->pool);
1849 }
1850 }
1851
1852 void
igb_iov_initialize(struct e1000_softc * sc)1853 igb_iov_initialize(struct e1000_softc *sc)
1854 {
1855 struct e1000_hw *hw;
1856 struct igb_vf *vf;
1857 u32 ctrl_ext, dtxctl, mask, rctl, rplolr, vt_ctl;
1858 int i;
1859
1860 if (sc->num_vfs == 0)
1861 return;
1862
1863 hw = &sc->hw;
1864 atomic_readandclear_32(&sc->iov_mdd_cause);
1865 atomic_readandclear_32(&sc->iov_pending);
1866 atomic_readandclear_32(&sc->iov_spoof_pending);
1867 atomic_readandclear_32(&sc->iov_blocked_pending);
1868 /* Plain VMDq keeps every 82576 PF/VF pool on queue zero. */
1869 E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_ENABLE_VMDQ);
1870
1871 vt_ctl = E1000_READ_REG(hw, E1000_VT_CTL);
1872 vt_ctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
1873 E1000_VT_CTL_DISABLE_DEF_POOL);
1874 vt_ctl |= sc->pool << E1000_VT_CTL_DEFAULT_POOL_SHIFT;
1875 vt_ctl |= E1000_VT_CTL_VM_REPL_EN;
1876 E1000_WRITE_REG(hw, E1000_VT_CTL, vt_ctl);
1877
1878 mask = 1U << sc->pool;
1879 E1000_WRITE_REG(hw, E1000_VFRE, mask);
1880 E1000_WRITE_REG(hw, E1000_VFTE, mask);
1881 /* A VF without RX descriptors must not block any other pool. */
1882 E1000_WRITE_REG(hw, E1000_QDE,
1883 hw->mac.type == e1000_i350 ? IGB_I350_QUEUE_MASK : ALL_QUEUES);
1884 e1000_vmdq_set_loopback_pf(hw, true);
1885 dtxctl = E1000_READ_REG(hw, E1000_DTXCTL);
1886 dtxctl |= E1000_DTXCTL_MDP_EN;
1887 if (hw->mac.type == e1000_82576) {
1888 dtxctl |= E1000_DTXCTL_VLAN_ADDED |
1889 E1000_DTXCTL_SPOOF_INT;
1890 rplolr = E1000_READ_REG(hw, E1000_RPLOLR);
1891 rplolr |= E1000_RPLOLR_STRVLAN;
1892 E1000_WRITE_REG(hw, E1000_RPLOLR, rplolr);
1893 } else {
1894 /*
1895 * I350 SDM section 8.12.5 defines this field with inverted
1896 * polarity: setting it keeps an ordinary spoof from disabling
1897 * the VF queue. Enable its notification as well. I350
1898 * hardware reports the VF in LVMMC.Last_Q (WVBR remains zero);
1899 * the moderated admin vector captures that value, while
1900 * timer-driven administration and per-VF ratecheck bound the
1901 * work and console output.
1902 */
1903 dtxctl |= E1000_DTXCTL_SPOOF_INT |
1904 IGB_I350_DTXCTL_ENABLE_SPOOF_QUEUE;
1905 }
1906 E1000_WRITE_REG(hw, E1000_DTXCTL, dtxctl);
1907
1908 igb_iov_map_rar(sc, 0, hw->mac.addr, sc->pool);
1909 igb_iov_configure_pf_vmolr(sc);
1910 igb_iov_set_uta(sc);
1911 for (i = 0; i < sc->num_vfs; i++) {
1912 vf = &sc->vfs[i];
1913 if (!(vf->flags & IGB_VF_ACTIVE))
1914 continue;
1915 /*
1916 * A PF-wide reset is trusted and can require a complete guest
1917 * replay. Guest-controlled RESET and VFLR do not refill this
1918 * allowance.
1919 */
1920 igb_iov_reset_vlan_rate(vf);
1921 igb_iov_clear_mac_filters(sc, vf);
1922 igb_iov_reset_vf_state(sc, vf);
1923 igb_iov_clear_rar(sc, vf->rar_index);
1924 igb_iov_set_anti_spoof(sc, vf);
1925 }
1926 igb_iov_rebuild_mta(sc);
1927 igb_iov_rebuild_vlan(sc);
1928
1929 rctl = E1000_READ_REG(hw, E1000_RCTL);
1930 E1000_WRITE_REG(hw, E1000_RCTL, rctl | E1000_RCTL_VFE);
1931 E1000_WRITE_REG(hw, E1000_MBVFIMR, igb_iov_active_mask(sc));
1932 if (hw->mac.type == e1000_i350)
1933 E1000_WRITE_REG(hw, E1000_DMACR, 0);
1934
1935 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1936 E1000_WRITE_REG(hw, E1000_CTRL_EXT,
1937 ctrl_ext | E1000_CTRL_EXT_PFRSTD);
1938 E1000_WRITE_FLUSH(hw);
1939 /*
1940 * MDDET remains masked until iov_hw_active is published and iflib
1941 * rearms the admin vector. Programming the per-pool policy above can
1942 * leave a setup-time MDDET observation in the read-clear registers.
1943 * If that stale cause is carried across the unmask, a later ordinary
1944 * spoof can update LVMMC without generating a new interrupt edge.
1945 *
1946 * Drain only after all IOV policy is installed and before exposing the
1947 * active state. Mailbox requests are also serviced by the periodic
1948 * admin pass, and ping_all_vfs() below supplies a fresh notification.
1949 */
1950 /*
1951 * Clear the setup-time interrupt latch before its diagnostic state.
1952 * I350 does not reliably generate the next MDDET edge when LVMMC is
1953 * consumed while ICR.MDDET remains latched. This differs deliberately
1954 * from the final arm-time drain, where ICR is read last so a later event
1955 * remains pending for the unmask.
1956 */
1957 if (hw->mac.type == e1000_i350)
1958 (void)E1000_READ_REG(hw, E1000_ICR);
1959 (void)E1000_READ_REG(hw, E1000_LVMMC);
1960 if (hw->mac.type == e1000_82576)
1961 (void)E1000_READ_REG(hw, E1000_WVBR);
1962 if (hw->mac.type != e1000_i350)
1963 (void)E1000_READ_REG(hw, E1000_ICR);
1964 atomic_readandclear_32(&sc->iov_mdd_cause);
1965 atomic_readandclear_32(&sc->iov_pending);
1966 atomic_readandclear_32(&sc->iov_spoof_pending);
1967 atomic_readandclear_32(&sc->iov_blocked_pending);
1968 atomic_store_rel_32(&sc->iov_intr_drain_pending, 1);
1969 sc->iov_hw_active = true;
1970 igb_iov_ping_all_vfs(sc);
1971 }
1972
1973 int
igb_iov_validate(struct e1000_softc * sc,u16 num_vfs)1974 igb_iov_validate(struct e1000_softc *sc, u16 num_vfs)
1975 {
1976 if (!igb_iov_supported(sc))
1977 return (ENXIO);
1978 /* One of the eight hardware pools is reserved for the PF. */
1979 if (num_vfs == 0 || num_vfs > MAX_NUM_VFS)
1980 return (EINVAL);
1981 if (sc->vfs != NULL)
1982 return (EBUSY);
1983 if (sc->intr_type != IFLIB_INTR_MSIX) {
1984 device_printf(sc->dev, "SR-IOV requires MSI-X\n");
1985 return (ENOTSUP);
1986 }
1987 if (sc->tx_num_queues != 1 || sc->rx_num_queues != 1) {
1988 device_printf(sc->dev,
1989 "SR-IOV requires one PF TX and RX queue; set "
1990 "dev.igb.%d.iflib.override_ntxqs=1 and "
1991 "dev.igb.%d.iflib.override_nrxqs=1 before attach\n",
1992 device_get_unit(sc->dev), device_get_unit(sc->dev));
1993 return (EINVAL);
1994 }
1995 return (0);
1996 }
1997
1998 int
igb_if_iov_init(if_ctx_t ctx,u16 num_vfs,const nvlist_t * config)1999 igb_if_iov_init(if_ctx_t ctx, u16 num_vfs, const nvlist_t *config)
2000 {
2001 struct e1000_softc *sc;
2002 int error, i;
2003
2004 sc = iflib_get_softc(ctx);
2005 (void)config;
2006 /*
2007 * This callback may run while the PF is down. Record the software
2008 * layout here; igb_iov_initialize() programs it during interface init.
2009 */
2010 atomic_store_rel_32(&sc->iov_teardown, 0);
2011 error = igb_iov_validate(sc, num_vfs);
2012 if (error != 0)
2013 return (error);
2014
2015 sc->vfs = mallocarray(num_vfs, sizeof(*sc->vfs), M_IGB_IOV,
2016 M_WAITOK | M_ZERO);
2017 sc->num_vf_mac_filters =
2018 sc->hw.mac.rar_entry_count - num_vfs - 1;
2019 sc->vf_mac_filters = mallocarray(sc->num_vf_mac_filters,
2020 sizeof(*sc->vf_mac_filters), M_IGB_IOV, M_WAITOK | M_ZERO);
2021 for (i = 0; i < sc->num_vf_mac_filters; i++)
2022 sc->vf_mac_filters[i].rar_index = i + 1;
2023 sc->pool = num_vfs;
2024 sc->iov_mta_valid = false;
2025 sc->iov_pf_mdd_blocked = false;
2026 sc->tx_queues[0].txr.me = sc->pool;
2027 sc->rx_queues[0].rxr.me = sc->pool;
2028 e1000_init_mbx_params_pf(&sc->hw);
2029 sc->num_vfs = num_vfs;
2030 return (0);
2031 }
2032
2033 void
igb_if_iov_uninit(if_ctx_t ctx)2034 igb_if_iov_uninit(if_ctx_t ctx)
2035 {
2036 struct e1000_softc *sc;
2037 struct e1000_hw *hw;
2038 u32 mask, rah;
2039 int error, i, iov_pos;
2040 u16 iov_ctl;
2041
2042 sc = iflib_get_softc(ctx);
2043 if (sc->vfs == NULL)
2044 return;
2045 hw = &sc->hw;
2046 sc->iov_hw_active = false;
2047 if (sc->iov_mbx_retry_initialized)
2048 callout_drain(&sc->iov_mbx_retry);
2049
2050 E1000_WRITE_REG(hw, E1000_MBVFIMR, 0);
2051 mask = 1U << sc->pool;
2052 E1000_WRITE_REG(hw, E1000_VFRE, mask);
2053 E1000_WRITE_REG(hw, E1000_VFTE, mask);
2054
2055 /*
2056 * pci_iov(4) invokes the driver before it clears VF Enable. Quiesce
2057 * the VFs and clear it here so that 82576's queue-reuse interval is
2058 * measured from the actual IOV-disable event.
2059 */
2060 error = pci_find_extcap(sc->dev, PCIZ_SRIOV, &iov_pos);
2061 if (error == 0) {
2062 iov_ctl = pci_read_config(sc->dev,
2063 iov_pos + PCIR_SRIOV_CTL, 2);
2064 iov_ctl &= ~(PCIM_SRIOV_VF_EN | PCIM_SRIOV_VF_MSE);
2065 pci_write_config(sc->dev, iov_pos + PCIR_SRIOV_CTL,
2066 iov_ctl, 2);
2067 if (hw->mac.type == e1000_82576) {
2068 pause("igbiov", MAX(1, howmany(hz, 10)));
2069 E1000_WRITE_REG(hw, E1000_IOVCTL,
2070 E1000_IOVCTL_REUSE_VFQ);
2071 E1000_WRITE_FLUSH(hw);
2072 pause("igbiov", MAX(1, howmany(hz, 10)));
2073 }
2074 } else
2075 device_printf(sc->dev,
2076 "could not disable PCI SR-IOV before queue reuse: %d\n",
2077 error);
2078
2079 E1000_WRITE_REG(hw, E1000_VT_CTL, 0);
2080 e1000_vmdq_set_loopback_pf(hw, false);
2081 e1000_vmdq_set_anti_spoofing_pf(hw, false, 0);
2082 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
2083 E1000_WRITE_REG(hw, E1000_VLVF(i), 0);
2084 for (i = 0; i < sc->num_vfs; i++)
2085 if (sc->vfs[i].flags & IGB_VF_ACTIVE)
2086 igb_iov_clear_rar(sc, sc->vfs[i].rar_index);
2087 for (i = 0; i < sc->num_vf_mac_filters; i++)
2088 if (sc->vf_mac_filters[i].active)
2089 igb_iov_clear_rar(sc, sc->vf_mac_filters[i].rar_index);
2090 rah = E1000_READ_REG(hw, E1000_RAH(0));
2091 rah &= ~IGB_IOV_RAH_POOLSEL_MASK;
2092 E1000_WRITE_REG(hw, E1000_RAH(0), rah);
2093
2094 free(sc->vfs, M_IGB_IOV);
2095 free(sc->vf_mac_filters, M_IGB_IOV);
2096 sc->vfs = NULL;
2097 sc->vf_mac_filters = NULL;
2098 sc->num_vfs = 0;
2099 sc->num_vf_mac_filters = 0;
2100 sc->pool = 0;
2101 sc->iov_mta_valid = false;
2102 sc->iov_pf_mdd_blocked = false;
2103 sc->iov_pf_vlan_promisc = false;
2104 igb_iov_vfta_shadow_invalidate(sc);
2105 sc->tx_queues[0].txr.me = 0;
2106 sc->rx_queues[0].rxr.me = 0;
2107 atomic_readandclear_32(&sc->iov_mdd_cause);
2108 atomic_readandclear_32(&sc->iov_pending);
2109 atomic_readandclear_32(&sc->iov_spoof_pending);
2110 atomic_readandclear_32(&sc->iov_blocked_pending);
2111 atomic_readandclear_32(&sc->iov_intr_drain_pending);
2112 atomic_store_rel_32(&sc->iov_teardown, 0);
2113 }
2114
2115 static bool
igb_iov_mac_in_use(struct e1000_softc * sc,const u8 * mac,const struct igb_vf * skip)2116 igb_iov_mac_in_use(struct e1000_softc *sc, const u8 *mac,
2117 const struct igb_vf *skip)
2118 {
2119 int i;
2120
2121 if (memcmp(sc->hw.mac.addr, mac, ETHER_ADDR_LEN) == 0)
2122 return (true);
2123 for (i = 0; i < sc->num_vfs; i++)
2124 if (&sc->vfs[i] != skip &&
2125 (sc->vfs[i].flags & IGB_VF_ACTIVE) != 0 &&
2126 memcmp(sc->vfs[i].mac, mac, ETHER_ADDR_LEN) == 0)
2127 return (true);
2128 for (i = 0; i < sc->num_vf_mac_filters; i++)
2129 if (sc->vf_mac_filters[i].active &&
2130 memcmp(sc->vf_mac_filters[i].mac, mac,
2131 ETHER_ADDR_LEN) == 0)
2132 return (true);
2133 return (false);
2134 }
2135
2136 int
igb_if_iov_vf_add(if_ctx_t ctx,u16 vfnum,const nvlist_t * config)2137 igb_if_iov_vf_add(if_ctx_t ctx, u16 vfnum, const nvlist_t *config)
2138 {
2139 struct e1000_softc *sc;
2140 struct igb_vf *vf;
2141 struct ether_addr generated;
2142 const void *mac;
2143 char nameunit[IFNAMSIZ + sizeof("-vf65535")];
2144 size_t mac_size;
2145 uint64_t configured_vlan;
2146 u16 vlan;
2147
2148 sc = iflib_get_softc(ctx);
2149 if (vfnum >= sc->num_vfs)
2150 return (EINVAL);
2151 vf = &sc->vfs[vfnum];
2152 if (vf->flags & IGB_VF_ACTIVE)
2153 return (EBUSY);
2154
2155 configured_vlan = nvlist_get_number(config, "vlan");
2156 if (configured_vlan > VF_VLAN_TRUNK)
2157 return (EINVAL);
2158 vlan = configured_vlan;
2159 if (vlan == 0)
2160 return (ENOTSUP);
2161 if (vlan == VF_VLAN_TRUNK)
2162 vlan = 0;
2163 if (!igb_iov_vlan_present(sc, vlan, false) &&
2164 igb_iov_vlan_unique_count(sc, false) >=
2165 E1000_VLVF_ARRAY_SIZE)
2166 return (ENOSPC);
2167
2168 vf->pool = vfnum;
2169 vf->rar_index = sc->hw.mac.rar_entry_count - (vfnum + 1);
2170 vf->max_frame_size = ETHER_MAX_LEN;
2171 vf->default_vlan = vlan;
2172 igb_iov_reset_vlan_rate(vf);
2173 if (nvlist_exists_binary(config, "mac-addr")) {
2174 mac = nvlist_get_binary(config, "mac-addr", &mac_size);
2175 if (mac_size != ETHER_ADDR_LEN || !igb_iov_mac_valid(mac))
2176 return (EINVAL);
2177 if (igb_iov_mac_in_use(sc, mac, vf))
2178 return (EADDRINUSE);
2179 memcpy(vf->mac, mac, ETHER_ADDR_LEN);
2180 } else {
2181 snprintf(nameunit, sizeof(nameunit), "%s-vf%u",
2182 device_get_nameunit(sc->dev), vfnum);
2183 ether_gen_addr_byname(nameunit, &generated);
2184 memcpy(vf->mac, generated.octet, ETHER_ADDR_LEN);
2185 if (igb_iov_mac_in_use(sc, vf->mac, vf))
2186 return (EADDRINUSE);
2187 }
2188 if (nvlist_get_bool(config, "allow-set-mac"))
2189 vf->flags |= IGB_VF_CAP_MAC;
2190 if (nvlist_get_bool(config, "mac-anti-spoof"))
2191 vf->flags |= IGB_VF_MAC_ANTI_SPOOF;
2192 if (nvlist_get_bool(config, "allow-promisc"))
2193 vf->flags |= IGB_VF_ALLOW_PROMISC;
2194 vf->flags |= IGB_VF_ACTIVE;
2195
2196 igb_iov_reset_vf_state(sc, vf);
2197 igb_iov_set_anti_spoof(sc, vf);
2198 igb_iov_rebuild_vlan(sc);
2199 E1000_WRITE_REG(&sc->hw, E1000_MBVFIMR, igb_iov_active_mask(sc));
2200 return (0);
2201 }
2202
2203 #endif /* PCI_IOV */
2204