xref: /freebsd/sys/dev/e1000/if_igb_iov.c (revision a6999a1c947f3dc8b7491c903231efc2d78538d5)

/*-
 * SPDX-License-Identifier: BSD-3-Clause
 *
 * Copyright (c) 2010-2016, Intel Corporation
 * Copyright (c) 2026 Kevin Bowling <kbowling@FreeBSD.org>
 */

#include "if_em.h"
#include "if_igb_iov.h"

#ifdef PCI_IOV

#include <sys/iov.h>
#include <sys/sdt.h>
#include <sys/time.h>

#define	IGB_IOV_RAH_POOLSEL_SHIFT	18
#define	IGB_IOV_RAH_POOLSEL_MASK	(0xffU << IGB_IOV_RAH_POOLSEL_SHIFT)
#define	IGB_IOV_MAX_MAC_FILTERS		3
#define	IGB_IOV_MAX_MC_HASHES		30
#define	IGB_IOV_MBX_RETRY_COUNT		6
/* Allow two complete 31-VID replays, then sustain eight additions/second. */
#define	IGB_IOV_VLAN_CHANGE_BURST	64
#define	IGB_IOV_VLAN_CHANGE_INTERVAL	(SBT_1S / 8)
/* 82576 Datasheet rev. 2.0, Section 8.14.16: VMOLR[31] must be one. */
#define	IGB_82576_VMOLR_RSV		(1U << 31)
#define	IGB_82576_LVMMC_BLOCK_MASK	0x1c
#define	IGB_82576_NUM_QUEUES		16
#define	IGB_82576_QUEUE_MASK		0xffff
#define	IGB_82576_STAGGERED_QUEUE_SHIFT	8
#define	IGB_82576_VF_QUEUE_STRIDE	8
#define	IGB_82576_VF_QUEUES		2
#define	IGB_I350_DTXCTL_ENABLE_SPOOF_QUEUE	(1U << 2)
#define	IGB_I350_LVMMC_MAC_VLAN_SPOOF	(1U << 25)
#define	IGB_I350_LVMMC_LAST_Q_SHIFT	29
#define	IGB_I350_LVMMC_LAST_Q_MASK	0x7
#define	IGB_I350_NUM_QUEUES		8
#define	IGB_I350_QUEUE_MASK		0xff
#define	IGB_I350_RESET_ACK_TIMEOUT	(100 * SBT_1MS)
#define	IGB_I350_VF_QUEUES		1
#define	IGB_IOV_QUEUE_DISABLE_BUSY_RETRIES	10
#define	IGB_IOV_QUEUE_DISABLE_DELAY_US	10
#define	IGB_IOV_QUEUE_DISABLE_PAUSE	(100 * SBT_1US)
#define	IGB_IOV_QUEUE_DISABLE_RETRIES	20
#define	IGB_IOV_VF_QUEUES_MAX		2

#define	IGB_VF_CTS			(1U << 0)
#define	IGB_VF_CAP_MAC			(1U << 1)
#define	IGB_VF_ACTIVE			(1U << 2)
#define	IGB_VF_MAC_ANTI_SPOOF		(1U << 3)
#define	IGB_VF_ALLOW_PROMISC		(1U << 4)
#define	IGB_VF_UCAST_PROMISC		(1U << 5)
#define	IGB_VF_MCAST_PROMISC		(1U << 6)
#define	IGB_VF_MCAST_OVERFLOW		(1U << 7)
#define	IGB_VF_MCAST_OVERFLOW_WARNED	(1U << 8)
#define	IGB_VF_MDD_BLOCKED		(1U << 9)
#define	IGB_VF_MBX_PENDING		(1U << 10)
/*
 * After bounded PFU retries, suppress future or overlapping VF requests until
 * RST/VFLR starts a new mailbox epoch.  Intel VF drivers assert CTRL.RST
 * before sending their mailbox reset request.
 */
#define	IGB_VF_MBX_GAVE_UP		(1U << 11)
#define	IGB_VF_MDD_NOTIFY_PENDING	(1U << 12)

struct igb_vf {
	u32	flags;
	struct timeval	last_nack;
	struct timeval	last_mbx_log;
	struct timeval	last_spoof_log;
	struct timeval	last_mdd_log;
	struct timeval	last_queue_log;
	sbintime_t	mbx_retry_at;
	sbintime_t	mdd_notify_at;
	sbintime_t	vlan_token_time;
	u16	pool;
	u16	rar_index;
	u16	max_frame_size;
	u16	mc_count;
	u16	vlan_count;
	u16	vlan_tokens;
	u16	default_vlan;
	u8	mbx_retry_count;
	u8	mac[ETHER_ADDR_LEN];
	u16	mc_hashes[IGB_IOV_MAX_MC_HASHES];
	u32	vlans[EM_VFTA_SIZE];
};

struct igb_vf_mac_filter {
	bool	active;
	u16	pool;
	u16	rar_index;
	u8	mac[ETHER_ADDR_LEN];
};

MALLOC_DEFINE(M_IGB_IOV, "igb_iov", "igb SR-IOV allocations");

/*
 * These logical-write probes let hardware tests verify the elision policy.
 * e1000_write_vfta_i350() expands one VFTA call into ten physical writes, so
 * the probes intentionally count calls made by the rebuild rather than MMIO
 * transactions.  The state probe exposes the final software images while the
 * stack arrays are still live.
 */
SDT_PROVIDER_DEFINE(igb_iov);
SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, vfta_clear,
    "struct e1000_softc *", "u_int", "uint32_t");
SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, vlvf_write,
    "struct e1000_softc *", "u_int", "uint32_t");
SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, vfta_set,
    "struct e1000_softc *", "u_int", "uint32_t");
SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, state,
    "struct e1000_softc *", "uint32_t *", "uint32_t *");
SDT_PROBE_DEFINE4(igb_iov, mdd, sample, wvbr,
    "struct e1000_softc *", "uint32_t", "uint32_t", "uint32_t");

static const struct timeval igb_iov_nack_interval = { 2, 0 };
static const struct timeval igb_iov_mbx_log_interval = { 2, 0 };
static const struct timeval igb_iov_spoof_log_interval = { 2, 0 };
static const struct timeval igb_iov_mdd_log_interval = { 2, 0 };
static const sbintime_t igb_iov_mdd_notify_retry = SBT_1S / 2;
static const sbintime_t igb_iov_mbx_retry_delay[IGB_IOV_MBX_RETRY_COUNT] = {
	SBT_1MS,
	2 * SBT_1MS,
	4 * SBT_1MS,
	8 * SBT_1MS,
	16 * SBT_1MS,
	32 * SBT_1MS,
};

static void	igb_iov_clear_mac_filters(struct e1000_softc *,
		    const struct igb_vf *);
static bool	igb_iov_mac_in_use(struct e1000_softc *, const u8 *,
		    const struct igb_vf *);
static bool	igb_iov_vlan_present(struct e1000_softc *, u16, bool);
static int	igb_iov_vlan_unique_count(struct e1000_softc *, bool);

static void
igb_iov_mbx_retry_callout(void *arg)
{
	struct e1000_softc *sc;

	sc = arg;
	/*
	 * Mailbox service is serialized by iflib's context lock.  The
	 * callout only re-enters through the ordinary admin task.
	 */
	iflib_admin_intr_deferred(sc->ctx);
}

static u_int
igb_iov_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int idx)
{
	u8 *mta;

	if (idx == MAX_NUM_MULTICAST_ADDRESSES)
		return (0);
	mta = arg;
	memcpy(&mta[idx * ETHER_ADDR_LEN], LLADDR(sdl), ETHER_ADDR_LEN);
	return (1);
}

static bool
igb_iov_pf_vlan_promisc(struct e1000_softc *sc)
{
	if_t ifp;

	ifp = iflib_get_ifp(sc->ctx);
	return (sc->iov_pf_vlan_promisc ||
	    (if_getflags(ifp) & IFF_PROMISC) != 0);
}

static bool
igb_iov_mac_valid(const u8 *mac)
{
	static const u8 zero[ETHER_ADDR_LEN];

	return (!ETHER_IS_MULTICAST(mac) &&
	    memcmp(mac, zero, ETHER_ADDR_LEN) != 0);
}

static bool
igb_iov_nack_allowed(struct igb_vf *vf)
{
	return (ratecheck(&vf->last_nack, &igb_iov_nack_interval) != 0);
}

static void
igb_iov_reset_vlan_rate(struct igb_vf *vf)
{

	vf->vlan_token_time = getsbinuptime();
	vf->vlan_tokens = IGB_IOV_VLAN_CHANGE_BURST;
}

static bool
igb_iov_vlan_add_allowed(struct igb_vf *vf)
{
	sbintime_t elapsed, now;
	uint64_t refill;

	now = getsbinuptime();
	elapsed = now - vf->vlan_token_time;
	if (elapsed >= IGB_IOV_VLAN_CHANGE_INTERVAL) {
		refill = elapsed / IGB_IOV_VLAN_CHANGE_INTERVAL;
		vf->vlan_tokens = min((uint64_t)IGB_IOV_VLAN_CHANGE_BURST,
		    vf->vlan_tokens + refill);
		vf->vlan_token_time = now;
	}
	if (vf->vlan_tokens == 0)
		return (false);
	vf->vlan_tokens--;
	return (true);
}

static u32
igb_iov_reply_header(u32 request, bool cts, bool ack)
{
	u32 reply, type;

	type = request & 0xffff;
	if (type == E1000_VF_SET_MAC_ADDR &&
	    (request & E1000_VT_MSGINFO_MASK) != 0)
		reply = request;
	else
		reply = type;
	reply &= ~(E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK |
	    E1000_VT_MSGTYPE_CTS);
	if (cts)
		reply |= E1000_VT_MSGTYPE_CTS;
	reply |= ack ? E1000_VT_MSGTYPE_ACK : E1000_VT_MSGTYPE_NACK;
	return (reply);
}

bool
igb_iov_supported(const struct e1000_softc *sc)
{
	switch (sc->hw.mac.type) {
	case e1000_82576:
	case e1000_i350:
		return (true);
	default:
		return (false);
	}
}

bool
igb_iov_enabled(const struct e1000_softc *sc)
{
	return (sc->num_vfs != 0);
}

int
igb_iov_attach(struct e1000_softc *sc)
{
	nvlist_t *pf_schema, *vf_schema;
	int error, iov_pos;

	if (!igb_iov_supported(sc))
		return (0);
	if (pci_find_extcap(sc->dev, PCIZ_SRIOV, &iov_pos) != 0)
		return (0);

	pf_schema = pci_iov_schema_alloc_node();
	vf_schema = pci_iov_schema_alloc_node();
	pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL);
	pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof",
	    IOV_SCHEMA_HASDEFAULT, true);
	pci_iov_schema_add_bool(vf_schema, "allow-set-mac",
	    IOV_SCHEMA_HASDEFAULT, false);
	pci_iov_schema_add_bool(vf_schema, "allow-promisc",
	    IOV_SCHEMA_HASDEFAULT, false);
	pci_iov_schema_add_vlan(vf_schema, "vlan", IOV_SCHEMA_HASDEFAULT,
	    VF_VLAN_TRUNK);

	error = pci_iov_attach(sc->dev, pf_schema, vf_schema);
	if (error != 0)
		device_printf(sc->dev,
		    "failed to attach SR-IOV configuration interface: %d\n",
		    error);
	else {
		callout_init(&sc->iov_mbx_retry, 1);
		sc->iov_mbx_retry_initialized = true;
	}
	return (error);
}

void
igb_iov_detach(struct e1000_softc *sc)
{

	if (!sc->iov_mbx_retry_initialized)
		return;
	callout_drain(&sc->iov_mbx_retry);
	sc->iov_mbx_retry_initialized = false;
}

static u32
igb_iov_active_mask(struct e1000_softc *sc)
{
	u32 mask;
	int i;

	mask = 0;
	for (i = 0; i < sc->num_vfs; i++)
		if (sc->vfs[i].flags & IGB_VF_ACTIVE)
			mask |= 1U << i;
	return (mask);
}

static void
igb_iov_map_rar(struct e1000_softc *sc, u16 rar, const u8 *mac, u16 pool)
{
	struct e1000_hw *hw;
	u32 rah;

	hw = &sc->hw;
	e1000_rar_set(hw, __DECONST(u8 *, mac), rar);
	rah = E1000_READ_REG(hw, E1000_RAH(rar));
	rah &= ~IGB_IOV_RAH_POOLSEL_MASK;
	rah |= 1U << (IGB_IOV_RAH_POOLSEL_SHIFT + pool);
	E1000_WRITE_REG(hw, E1000_RAH(rar), rah);
}

static void
igb_iov_clear_rar(struct e1000_softc *sc, u16 rar)
{
	u8 zero[ETHER_ADDR_LEN] = {};

	e1000_rar_set(&sc->hw, zero, rar);
}

static void
igb_iov_clear_mac_filters(struct e1000_softc *sc, const struct igb_vf *vf)
{
	struct igb_vf_mac_filter *filter;
	int i;

	for (i = 0; i < sc->num_vf_mac_filters; i++) {
		filter = &sc->vf_mac_filters[i];
		if (!filter->active || filter->pool != vf->pool)
			continue;
		igb_iov_clear_rar(sc, filter->rar_index);
		filter->active = false;
		memset(filter->mac, 0, sizeof(filter->mac));
	}
}

static u32
igb_iov_switch_reg(struct e1000_softc *sc)
{
	return (sc->hw.mac.type == e1000_82576 ?
	    E1000_DTXSWC : E1000_TXSWC);
}

static void
igb_iov_set_anti_spoof(struct e1000_softc *sc, struct igb_vf *vf)
{
	struct e1000_hw *hw;
	u32 reg, value;

	hw = &sc->hw;
	reg = igb_iov_switch_reg(sc);
	value = E1000_READ_REG(hw, reg);
	value &= ~((1U << vf->pool) |
	    (1U << (vf->pool + E1000_DTXSWC_VLAN_SPOOF_SHIFT)));
	if (vf->flags & IGB_VF_MAC_ANTI_SPOOF)
		value |= 1U << vf->pool;
	if (vf->flags & IGB_VF_ACTIVE)
		value |= 1U <<
		    (vf->pool + E1000_DTXSWC_VLAN_SPOOF_SHIFT);
	E1000_WRITE_REG(hw, reg, value);
}

static void
igb_iov_set_uta(struct e1000_softc *sc)
{
	struct e1000_hw *hw;
	bool enable;
	int i;

	if (!igb_iov_enabled(sc) || sc->hw.mac.type != e1000_82576)
		return;

	hw = &sc->hw;
	enable = (E1000_READ_REG(hw, E1000_VMOLR(sc->pool)) &
	    E1000_VMOLR_ROPE) != 0;
	for (i = 0; i < sc->num_vfs; i++)
		if ((sc->vfs[i].flags &
		    (IGB_VF_ACTIVE | IGB_VF_UCAST_PROMISC)) ==
		    (IGB_VF_ACTIVE | IGB_VF_UCAST_PROMISC)) {
			enable = true;
			break;
		}

	for (i = 0; i < MAX_MTA_REG; i++)
		E1000_WRITE_REG_ARRAY(hw, E1000_UTA, i,
		    enable ? 0xffffffffU : 0);
}

static void
igb_iov_configure_dvmolr(struct e1000_softc *sc, u16 pool,
    bool strip_vlan, bool hide_vlan, bool vf_pool)
{
	struct e1000_hw *hw;
	u32 dvmolr;

	hw = &sc->hw;
	if (hw->mac.type != e1000_i350)
		return;

	dvmolr = E1000_READ_REG(hw, E1000_DVMOLR(pool));
	dvmolr &= ~(E1000_DVMOLR_HIDVLAN | E1000_DVMOLR_STRVLAN |
	    E1000_DVMOLR_STRCRC);
	if (hide_vlan)
		dvmolr |= E1000_DVMOLR_HIDVLAN;
	if (strip_vlan)
		dvmolr |= E1000_DVMOLR_STRVLAN;
	if (vf_pool || strip_vlan ||
	    (E1000_READ_REG(hw, E1000_RCTL) & E1000_RCTL_SECRC) != 0)
		dvmolr |= E1000_DVMOLR_STRCRC;
	E1000_WRITE_REG(hw, E1000_DVMOLR(pool), dvmolr);
}

static void
igb_iov_configure_vmolr(struct e1000_softc *sc, struct igb_vf *vf)
{
	struct e1000_hw *hw;
	u32 max_frame_size, vmolr, vmvir;

	hw = &sc->hw;
	max_frame_size = vf->max_frame_size;
	if (vf->vlan_count != 0)
		max_frame_size = min(max_frame_size + VLAN_TAG_SIZE,
		    IGB_IOV_MAX_FRAME_SIZE);
	vmolr = E1000_READ_REG(hw, E1000_VMOLR(vf->pool));
	vmolr &= ~(E1000_VMOLR_RLPML_MASK | E1000_VMOLR_RSSE |
	    E1000_VMOLR_VPE | E1000_VMOLR_UPE | E1000_VMOLR_ROMPE |
	    E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_STRVLAN);
	vmolr |= E1000_VMOLR_BAM | E1000_VMOLR_LPE |
	    (max_frame_size & E1000_VMOLR_RLPML_MASK);
	if (vf->default_vlan == 0)
		vmolr |= E1000_VMOLR_AUPE;
	if (vf->mc_count != 0 &&
	    (vf->flags & (IGB_VF_MCAST_PROMISC |
	    IGB_VF_MCAST_OVERFLOW)) == 0)
		vmolr |= E1000_VMOLR_ROMPE;
	if (hw->mac.type == e1000_82576)
		vmolr |= IGB_82576_VMOLR_RSV;

	if (vf->flags & IGB_VF_UCAST_PROMISC) {
		if (hw->mac.type == e1000_82576)
			vmolr |= E1000_VMOLR_ROPE;
		else
			vmolr |= E1000_VMOLR_UPE;
	}
	/*
	 * The mailbox can describe only 30 hashes.  Fall back to receiving all
	 * multicast within the VF's VLAN membership when that list overflows.
	 */
	if ((vf->flags & (IGB_VF_MCAST_PROMISC |
	    IGB_VF_MCAST_OVERFLOW)) != 0)
		vmolr |= E1000_VMOLR_MPME;
	if (hw->mac.type == e1000_82576 && vf->vlan_count != 0)
		vmolr |= E1000_VMOLR_STRVLAN;
	/* A nonzero default VLAN makes this VF an untagged access port. */
	if (vf->default_vlan == 0)
		vmvir = 0;
	else
		vmvir = vf->default_vlan | E1000_VMVIR_VLANA_DEFAULT;

	E1000_WRITE_REG(hw, E1000_VMOLR(vf->pool), vmolr);
	E1000_WRITE_REG(hw, E1000_VMVIR(vf->pool), vmvir);
	igb_iov_configure_dvmolr(sc, vf->pool, vf->vlan_count != 0,
	    vf->default_vlan != 0, true);
}

static void
igb_iov_configure_pf_vmolr(struct e1000_softc *sc)
{
	struct e1000_hw *hw;
	if_t ifp;
	bool strip_vlan;
	u32 max_frame_size;
	u32 old_vmolr, vmolr;

	hw = &sc->hw;
	ifp = iflib_get_ifp(sc->ctx);
	max_frame_size = min(sc->shared->isc_max_frame_size + VLAN_TAG_SIZE,
	    IGB_IOV_MAX_FRAME_SIZE);
	strip_vlan = (E1000_READ_REG(hw, E1000_CTRL) & E1000_CTRL_VME) != 0;
	old_vmolr = E1000_READ_REG(hw, E1000_VMOLR(sc->pool));
	vmolr = E1000_VMOLR_BAM | E1000_VMOLR_AUPE |
	    E1000_VMOLR_LPE |
	    (max_frame_size & E1000_VMOLR_RLPML_MASK);
	if (hw->mac.type == e1000_82576) {
		vmolr |= IGB_82576_VMOLR_RSV;
		if (strip_vlan)
			vmolr |= E1000_VMOLR_STRVLAN;
	} else
		vmolr |= old_vmolr & E1000_VMOLR_VPE;

	if (if_getflags(ifp) & IFF_PROMISC) {
		if (hw->mac.type == e1000_82576)
			vmolr |= E1000_VMOLR_ROPE;
		else
			vmolr |= E1000_VMOLR_UPE | E1000_VMOLR_VPE;
		vmolr |= E1000_VMOLR_MPME;
	} else if ((if_getflags(ifp) & IFF_ALLMULTI) != 0 ||
	    if_llmaddr_count(ifp) >= MAX_NUM_MULTICAST_ADDRESSES)
		vmolr |= E1000_VMOLR_MPME;
	else if (if_llmaddr_count(ifp) != 0)
		vmolr |= E1000_VMOLR_ROMPE;

	E1000_WRITE_REG(hw, E1000_VMOLR(sc->pool), vmolr);
	igb_iov_configure_dvmolr(sc, sc->pool, strip_vlan, false, false);
}

void
igb_iov_update_pf_vmolr(struct e1000_softc *sc)
{
	if (!igb_iov_enabled(sc))
		return;

	igb_iov_configure_pf_vmolr(sc);
	igb_iov_set_uta(sc);
}

u32
igb_iov_intr_mask(const struct e1000_softc *sc)
{
	if (!sc->iov_hw_active)
		return (0);
	return (E1000_IMS_VMMB | E1000_IMS_MDDET);
}

void
igb_iov_intr_drain_stale(struct e1000_softc *sc)
{
	struct e1000_hw *hw;
	u32 icr;

	if (atomic_readandclear_32(&sc->iov_intr_drain_pending) == 0)
		return;
	hw = &sc->hw;
	/*
	 * Consume setup-time diagnostic state at the actual transition from
	 * masked to armed.  Read ICR last so an event arriving after the drain
	 * remains pending and is delivered when the caller enables MDDET.
	 */
	(void)E1000_READ_REG(hw, E1000_LVMMC);
	if (hw->mac.type == e1000_82576)
		(void)E1000_READ_REG(hw, E1000_WVBR);
	icr = E1000_READ_REG(hw, E1000_ICR);
	/*
	 * em_if_init() injects LSC after IOV setup to close the post-reset
	 * link race.  Preserve that cause across this MDDET-specific drain.
	 */
	if (__predict_true(icr != 0xffffffff) &&
	    (icr & E1000_ICR_LSC) != 0)
		E1000_WRITE_REG(hw, E1000_ICS, E1000_ICS_LSC);
}

static void
igb_iov_vfta_shadow_invalidate(struct e1000_softc *sc)
{

	/*
	 * I350 erratum 20 makes VFTA reads unreliable while VMDq loopback or
	 * anti-spoofing is active.  The shadow is therefore authoritative
	 * until a reset or another independent hardware writer invalidates
	 * it.  Readback cannot reliably audit a stale-but-valid shadow on
	 * this part, so keep all shadow mutation in these two helpers.
	 */
	memset(sc->iov_vfta, 0, sizeof(sc->iov_vfta));
	sc->iov_vfta_valid = false;
}

static void
igb_iov_vfta_shadow_store(struct e1000_softc *sc, const u32 *vfta)
{

	memcpy(sc->iov_vfta, vfta, sizeof(sc->iov_vfta));
	sc->iov_vfta_valid = true;
}

static void
igb_iov_notify_vfs_reset(struct e1000_softc *sc)
{
	struct igb_vf *vf;
	struct e1000_hw *hw;
	sbintime_t deadline;
	u32 msg, pending, undelivered;
	int i;

	hw = &sc->hw;
	/*
	 * Process VFLRs first and wait only for VFs that completed their
	 * mailbox handshake.  An unattached VF has nobody who can acknowledge.
	 */
	igb_iov_handle_mbx(sc);
	pending = 0;
	for (i = 0; i < sc->num_vfs; i++) {
		vf = &sc->vfs[i];
		if ((vf->flags & (IGB_VF_ACTIVE | IGB_VF_CTS)) ==
		    (IGB_VF_ACTIVE | IGB_VF_CTS))
			pending |= 1U << i;
	}
	if (pending == 0)
		return;

	/*
	 * I350 SDM section 4.6.11.2.3 requires each VF to acknowledge a
	 * mailbox warning before the PF asserts CTRL.RST.
	 *
	 * The mailbox pass above drained requests and stale acknowledgements.
	 * A VF read of the new notification sets its ACK bit.
	 */
	undelivered = 0;
	for (i = 0; i < sc->num_vfs; i++) {
		if ((pending & (1U << i)) == 0)
			continue;
		msg = E1000_PF_CONTROL_MSG;
		if (e1000_write_mbx(hw, &msg, 1, i) != 0) {
			undelivered |= 1U << i;
			pending &= ~(1U << i);
		}
	}
	if (undelivered != 0)
		device_printf(sc->dev,
		    "could not deliver reset warning to VF mask %#x\n",
		    undelivered);

	deadline = getsbinuptime() + IGB_I350_RESET_ACK_TIMEOUT;
	while (pending != 0 && getsbinuptime() < deadline) {
		for (i = 0; i < sc->num_vfs; i++) {
			if ((pending & (1U << i)) != 0 &&
			    e1000_check_for_ack(hw, i) == 0)
				pending &= ~(1U << i);
		}
		if (pending != 0)
			pause_sbt("igback", SBT_1MS, 0, C_HARDCLOCK);
	}
	if (pending != 0)
		device_printf(sc->dev,
		    "VF reset acknowledgement timed out for mask %#x\n",
		    pending);
}

void
igb_iov_reset_prepare(struct e1000_softc *sc)
{
	struct e1000_hw *hw;
	u32 mask;

	if (sc->iov_hw_active) {
		hw = &sc->hw;
		if (atomic_load_acq_32(&sc->iov_teardown) == 0) {
			if (hw->mac.type == e1000_i350)
				igb_iov_notify_vfs_reset(sc);
			else
				igb_iov_ping_all_vfs(sc);
		}

		/* Stop VF DMA before the PF asserts CTRL.RST. */
		mask = 1U << sc->pool;
		E1000_WRITE_REG(hw, E1000_VFRE, mask);
		E1000_WRITE_REG(hw, E1000_VFTE, mask);
		E1000_WRITE_FLUSH(hw);
	}
	sc->iov_hw_active = false;
	if (sc->iov_mbx_retry_initialized)
		callout_stop(&sc->iov_mbx_retry);
	sc->iov_mta_valid = false;
	igb_iov_vfta_shadow_invalidate(sc);
	atomic_readandclear_32(&sc->iov_mdd_cause);
	atomic_readandclear_32(&sc->iov_pending);
	atomic_readandclear_32(&sc->iov_spoof_pending);
	atomic_readandclear_32(&sc->iov_blocked_pending);
	/*
	 * Normal iflib initialization prepares the reset before
	 * igb_iov_initialize() requests this drain.  Preserve a still-pending
	 * I350 request across a later stop or repeated preparation so the next
	 * interrupt arm consumes it.  Other families retain the ordinary
	 * stop-time cleanup.
	 */
	if (sc->hw.mac.type != e1000_i350)
		atomic_readandclear_32(&sc->iov_intr_drain_pending);
}

void
igb_iov_rebuild_mta(struct e1000_softc *sc)
{
	struct e1000_hw *hw;
	struct igb_vf *vf;
	u32 hash_bit, hash_reg, hash_value;
	u32 mta[MAX_MTA_REG] = {};
	u16 hash;
	bool changed;
	int i, j, mcnt;

	if (!igb_iov_enabled(sc))
		return;

	hw = &sc->hw;
	memset(sc->mta, 0,
	    ETHER_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES);
	mcnt = if_foreach_llmaddr(iflib_get_ifp(sc->ctx),
	    igb_iov_copy_maddr, sc->mta);
	mcnt = min(mcnt, MAX_NUM_MULTICAST_ADDRESSES);
	for (i = 0; i < mcnt; i++) {
		hash_value = e1000_hash_mc_addr(hw,
		    &sc->mta[i * ETHER_ADDR_LEN]);
		hash_reg = (hash_value >> 5) &
		    (hw->mac.mta_reg_count - 1);
		hash_bit = hash_value & 0x1f;
		mta[hash_reg] |= 1U << hash_bit;
	}
	for (i = 0; i < sc->num_vfs; i++) {
		vf = &sc->vfs[i];
		if (!(vf->flags & IGB_VF_ACTIVE))
			continue;
		for (j = 0; j < vf->mc_count; j++) {
			hash = vf->mc_hashes[j] & 0xfff;
			mta[(hash >> 5) & (hw->mac.mta_reg_count - 1)] |=
			    1U << (hash & 0x1f);
		}
	}

	changed = false;
	for (i = hw->mac.mta_reg_count - 1; i >= 0; i--) {
		if (sc->iov_mta_valid && hw->mac.mta_shadow[i] == mta[i])
			continue;
		hw->mac.mta_shadow[i] = mta[i];
		E1000_WRITE_REG_ARRAY(hw, E1000_MTA, i, mta[i]);
		changed = true;
	}
	if (changed)
		E1000_WRITE_FLUSH(hw);
	sc->iov_mta_valid = true;
}

static int
igb_iov_vlvf_add(u32 *vlvf, const u32 *old_vlvf, u16 vid, u16 pool,
    bool preserve_only)
{
	int free_slot, i;

	free_slot = -1;
	for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
		if ((vlvf[i] & E1000_VLVF_VLANID_ENABLE) != 0 &&
		    (vlvf[i] & E1000_VLVF_VLANID_MASK) == vid) {
			vlvf[i] |= 1U << (E1000_VLVF_POOLSEL_SHIFT + pool);
			return (0);
		}
		if (free_slot == -1 &&
		    (vlvf[i] & E1000_VLVF_VLANID_ENABLE) == 0)
			free_slot = i;
	}
	for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
		if ((old_vlvf[i] & E1000_VLVF_VLANID_ENABLE) != 0 &&
		    (old_vlvf[i] & E1000_VLVF_VLANID_MASK) == vid &&
		    (vlvf[i] & E1000_VLVF_VLANID_ENABLE) == 0) {
			free_slot = i;
			break;
		}
	if (preserve_only && (i == E1000_VLVF_ARRAY_SIZE))
		return (ENOENT);
	if (free_slot == -1)
		return (ENOSPC);

	vlvf[free_slot] = E1000_VLVF_VLANID_ENABLE | vid |
	    (1U << (E1000_VLVF_POOLSEL_SHIFT + pool));
	return (0);
}

void
igb_iov_rebuild_vlan(struct e1000_softc *sc)
{
	struct e1000_hw *hw;
	struct igb_vf *vf;
	u32 old_vlvf[E1000_VLVF_ARRAY_SIZE];
	u32 effective_vfta[EM_VFTA_SIZE], vfta[EM_VFTA_SIZE];
	u32 vlvf[E1000_VLVF_ARRAY_SIZE];
	u32 old_vfta, rctl, vmolr;
	bool force_vfta, pf_overflow, pf_vlan_promisc, preserve_pf;
	bool vfta_changed, vlvf_changed;
	int i, vid;

	if (!igb_iov_enabled(sc))
		return;

	hw = &sc->hw;
	rctl = E1000_READ_REG(hw, E1000_RCTL);
	rctl &= ~E1000_RCTL_CFIEN;
	rctl |= E1000_RCTL_VFE;
	E1000_WRITE_REG(hw, E1000_RCTL, rctl);
	memcpy(vfta, sc->shadow_vfta, sizeof(vfta));
	memset(vlvf, 0, sizeof(vlvf));
	for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
		old_vlvf[i] = E1000_READ_REG(hw, E1000_VLVF(i));

	pf_vlan_promisc = igb_iov_pf_vlan_promisc(sc);
	pf_overflow = !pf_vlan_promisc && hw->mac.type == e1000_i350 &&
	    igb_iov_vlan_unique_count(sc, true) > E1000_VLVF_ARRAY_SIZE;
	preserve_pf = !pf_vlan_promisc && !pf_overflow;

	/* First keep every surviving VF mapping in its current slot. */
	for (i = 0; i < sc->num_vfs; i++) {
		vf = &sc->vfs[i];
		if (!(vf->flags & IGB_VF_ACTIVE))
			continue;
		for (vid = 0; vid < 4096; vid++) {
			if ((vf->vlans[vid >> 5] & (1U << (vid & 0x1f))) ==
			    0)
				continue;
			(void)igb_iov_vlvf_add(vlvf, old_vlvf, vid,
			    vf->pool, true);
		}
	}

	/*
	 * Preserve PF mappings unless I350 needs their slots for VFs.
	 * PF-only VLANs on 82576 intentionally have no VLVF mapping and
	 * reach the default PF pool after passing the global VFTA.
	 */
	if (preserve_pf)
		for (vid = 0; vid < 4096; vid++) {
			if ((sc->shadow_vfta[vid >> 5] &
			    (1U << (vid & 0x1f))) == 0)
				continue;
			if (hw->mac.type == e1000_82576 &&
			    !igb_iov_vlan_present(sc, vid, false))
				continue;
			(void)igb_iov_vlvf_add(vlvf, old_vlvf, vid,
			    sc->pool, true);
		}

	/* Allocate new VF mappings before PF mappings. */
	for (i = 0; i < sc->num_vfs; i++) {
		vf = &sc->vfs[i];
		if (!(vf->flags & IGB_VF_ACTIVE))
			continue;
		for (vid = 0; vid < 4096; vid++) {
			if ((vf->vlans[vid >> 5] & (1U << (vid & 0x1f))) ==
			    0)
				continue;
			if (igb_iov_vlvf_add(vlvf, old_vlvf, vid,
			    vf->pool, false) == 0)
				vfta[vid >> 5] |= 1U << (vid & 0x1f);
		}
		igb_iov_configure_vmolr(sc, vf);
	}
	if (!pf_vlan_promisc)
		for (vid = 0; vid < 4096; vid++) {
			if ((sc->shadow_vfta[vid >> 5] &
			    (1U << (vid & 0x1f))) == 0)
				continue;
			/*
			 * With no VLVF match, 82576 sends a globally admitted
			 * VLAN to the default PF pool.  A VLVF entry is needed
			 * only when this VLAN is also assigned to a VF.
			 */
			if (hw->mac.type == e1000_82576 &&
			    !igb_iov_vlan_present(sc, vid, false))
				continue;
			if (igb_iov_vlvf_add(vlvf, old_vlvf, vid,
			    sc->pool, false) != 0)
				pf_overflow = true;
		}

	if (pf_vlan_promisc) {
		memset(vfta, 0xff, sizeof(vfta));
		for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
			if ((vlvf[i] & E1000_VLVF_VLANID_ENABLE) != 0)
				vlvf[i] |= 1U <<
				    (E1000_VLVF_POOLSEL_SHIFT + sc->pool);
	}

	/*
	 * Establish the PF fallback before an overflowing I350 rebuild can
	 * displace one of its old VLVF mappings.
	 */
	vmolr = E1000_READ_REG(hw, E1000_VMOLR(sc->pool));
	vmolr &= ~E1000_VMOLR_VPE;
	if (hw->mac.type == e1000_i350 &&
	    (pf_overflow || pf_vlan_promisc))
		vmolr |= E1000_VMOLR_VPE;
	E1000_WRITE_REG(hw, E1000_VMOLR(sc->pool), vmolr);

	/*
	 * Remove global VFTA membership before removing a VLAN entirely, and
	 * add a VLVF mapping before globally admitting a new VF VLAN.  A
	 * transition to a PF-only VLAN deliberately retains VFTA membership
	 * and falls through to the default PF pool.
	 */
	force_vfta = hw->mac.type == e1000_i350 &&
	    !sc->iov_vfta_valid;
	vfta_changed = false;
	for (i = 0; i < EM_VFTA_SIZE; i++) {
		/*
		 * I350 erratum 20 makes VFTA reads unreliable while VMDq
		 * loopback or anti-spoofing is active.  Its ten-write
		 * workaround is already in e1000_write_vfta_i350().  Force a
		 * complete clear when the authoritative shadow is invalid;
		 * 82576 can safely diff against its live register contents.
		 */
		if (hw->mac.type == e1000_i350)
			old_vfta = force_vfta ? 0 : sc->iov_vfta[i];
		else
			old_vfta =
			    E1000_READ_REG_ARRAY(hw, E1000_VFTA, i);
		effective_vfta[i] = old_vfta & vfta[i];
		if (force_vfta || effective_vfta[i] != old_vfta) {
			SDT_PROBE3(igb_iov, vlan, rebuild, vfta_clear,
			    sc, i, effective_vfta[i]);
			e1000_write_vfta(hw, i, effective_vfta[i]);
			vfta_changed = true;
		}
	}
	if (vfta_changed)
		E1000_WRITE_FLUSH(hw);
	vlvf_changed = false;
	for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
		if (vlvf[i] != old_vlvf[i]) {
			SDT_PROBE3(igb_iov, vlan, rebuild, vlvf_write,
			    sc, i, vlvf[i]);
			E1000_WRITE_REG(hw, E1000_VLVF(i), vlvf[i]);
			vlvf_changed = true;
		}
	if (vlvf_changed)
		E1000_WRITE_FLUSH(hw);
	vfta_changed = false;
	for (i = 0; i < EM_VFTA_SIZE; i++)
		if (vfta[i] != effective_vfta[i]) {
			SDT_PROBE3(igb_iov, vlan, rebuild, vfta_set,
			    sc, i, vfta[i]);
			e1000_write_vfta(hw, i, vfta[i]);
			vfta_changed = true;
		}
	if (vfta_changed)
		E1000_WRITE_FLUSH(hw);
	SDT_PROBE3(igb_iov, vlan, rebuild, state, sc, vfta, vlvf);
	igb_iov_vfta_shadow_store(sc, vfta);
}

static bool
igb_iov_vlan_present(struct e1000_softc *sc, u16 vid, bool include_pf)
{
	int i;

	if (include_pf &&
	    (sc->shadow_vfta[vid >> 5] & (1U << (vid & 0x1f))) != 0)
		return (true);
	for (i = 0; i < sc->num_vfs; i++)
		if ((sc->vfs[i].flags & IGB_VF_ACTIVE) != 0 &&
		    (sc->vfs[i].vlans[vid >> 5] &
		    (1U << (vid & 0x1f))) != 0)
			return (true);
	return (false);
}

static int
igb_iov_vlan_unique_count(struct e1000_softc *sc, bool include_pf)
{
	u32 vlans;
	int count, i, word;

	count = 0;
	for (word = 0; word < EM_VFTA_SIZE; word++) {
		vlans = include_pf ? sc->shadow_vfta[word] : 0;
		for (i = 0; i < sc->num_vfs; i++)
			if ((sc->vfs[i].flags & IGB_VF_ACTIVE) != 0)
				vlans |= sc->vfs[i].vlans[word];
		count += bitcount32(vlans);
	}
	return (count);
}

static int
igb_iov_set_vlan(struct e1000_softc *sc, struct igb_vf *vf, u16 vid,
    bool add)
{
	u32 bit;
	bool present;

	bit = 1U << (vid & 0x1f);
	present = (vf->vlans[vid >> 5] & bit) != 0;
	if (vid == 0) {
		if (!present) {
			vf->vlans[0] |= 1U;
			igb_iov_rebuild_vlan(sc);
		}
		return (0);
	}
	if (add == present)
		return (0);

	/*
	 * Removals always reduce privilege and remain available.  Charge only
	 * additions, which a hostile VF must alternate with removals to force
	 * repeated global VLAN rebuilds.
	 */
	if (add && !igb_iov_vlan_present(sc, vid, false) &&
	    igb_iov_vlan_unique_count(sc, false) >=
	    E1000_VLVF_ARRAY_SIZE)
		return (ENOSPC);
	if (add && !igb_iov_vlan_add_allowed(vf))
		return (EBUSY);

	if (add) {
		vf->vlans[vid >> 5] |= bit;
		vf->vlan_count++;
	} else {
		vf->vlans[vid >> 5] &= ~bit;
		vf->vlan_count--;
	}
	igb_iov_rebuild_vlan(sc);
	return (0);
}

static void
igb_iov_reset_vf_state(struct e1000_softc *sc, struct igb_vf *vf)
{
	bool update_uta;

	update_uta = (vf->flags & IGB_VF_UCAST_PROMISC) != 0;
	vf->flags &= ~(IGB_VF_CTS | IGB_VF_UCAST_PROMISC |
	    IGB_VF_MCAST_PROMISC | IGB_VF_MCAST_OVERFLOW |
	    IGB_VF_MBX_PENDING | IGB_VF_MBX_GAVE_UP |
	    IGB_VF_MDD_NOTIFY_PENDING);
	vf->mbx_retry_at = 0;
	vf->mdd_notify_at = 0;
	vf->mbx_retry_count = 0;
	/*
	 * A reset starts a new mailbox epoch.  Permit one immediate NACK so a
	 * premature non-reset request does not wait for its posted-read
	 * timeout.
	 */
	memset(&vf->last_nack, 0, sizeof(vf->last_nack));
	vf->max_frame_size = ETHER_MAX_LEN;
	vf->mc_count = 0;
	vf->vlan_count = 0;
	memset(vf->mc_hashes, 0, sizeof(vf->mc_hashes));
	memset(vf->vlans, 0, sizeof(vf->vlans));
	/* Preserve the administrative access VLAN across VF and PF resets. */
	if (vf->default_vlan == 0)
		vf->vlans[0] = 1U;
	else {
		vf->vlans[vf->default_vlan >> 5] =
		    1U << (vf->default_vlan & 0x1f);
		vf->vlan_count = 1;
	}
	igb_iov_configure_vmolr(sc, vf);
	if (update_uta)
		igb_iov_set_uta(sc);
}

static bool
igb_iov_vf_vlan_is_default(const struct igb_vf *vf)
{
	u32 expected;
	int i;

	for (i = 0; i < EM_VFTA_SIZE; i++) {
		expected = 0;
		if (i == vf->default_vlan >> 5)
			expected = 1U << (vf->default_vlan & 0x1f);
		if (vf->vlans[i] != expected)
			return (false);
	}
	return (true);
}

static bool
igb_iov_sanitize_vf_queues(struct e1000_softc *sc,
    struct igb_vf *vf)
{
	struct e1000_hw *hw;
	u16 qid[IGB_IOV_VF_QUEUES_MAX];
	u32 rxdctl, txdctl;
	int i, nqueues, retry;

	hw = &sc->hw;
	switch (hw->mac.type) {
	case e1000_82576:
		nqueues = IGB_82576_VF_QUEUES;
		qid[0] = vf->pool;
		qid[1] = vf->pool + IGB_82576_VF_QUEUE_STRIDE;
		break;
	case e1000_i350:
		nqueues = IGB_I350_VF_QUEUES;
		qid[0] = vf->pool;
		break;
	default:
		return (true);
	}

	/*
	 * I350 maps pool n to queue n.  82576 gives VF n physical queues n
	 * and n + 8, so both retained queue configurations must be cleared.
	 */
	for (i = 0; i < nqueues; i++)
		KASSERT(qid[i] < (hw->mac.type == e1000_82576 ?
		    IGB_82576_NUM_QUEUES : IGB_I350_NUM_QUEUES),
		    ("%s: invalid VF queue %u", __func__, qid[i]));

	/*
	 * The 82576 and I350 specification updates, Software Clarification 3,
	 * note that VFLR does not reset the VF queue configuration.  Clear the
	 * PF-programmable state before acknowledging the reset so a new VF
	 * owner cannot inherit it, particularly a descriptor-head write-back
	 * DMA address.  The new VF driver initializes its active ring pointers
	 * during queue setup.
	 *
	 * Disable every queue first, then wait for outstanding DMA activity to
	 * stop before clearing TDWBAL/H and the remaining retained state.
	 * Spin only for the normal fast transition, then sleep so a VF that
	 * keeps asserting QUEUE_ENABLE cannot busy-wait the PF for 10 ms.
	 */
	for (i = 0; i < nqueues; i++) {
		E1000_WRITE_REG(hw, E1000_RXDCTL(qid[i]), 0);
		E1000_WRITE_REG(hw, E1000_TXDCTL(qid[i]), 0);
	}
	E1000_WRITE_FLUSH(hw);
	for (retry = 0; retry < IGB_IOV_QUEUE_DISABLE_RETRIES; retry++) {
		for (i = 0; i < nqueues; i++) {
			rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(qid[i]));
			txdctl = E1000_READ_REG(hw, E1000_TXDCTL(qid[i]));
			if ((rxdctl & E1000_RXDCTL_QUEUE_ENABLE) != 0 ||
			    (txdctl & E1000_TXDCTL_QUEUE_ENABLE) != 0)
				break;
		}
		if (i == nqueues)
			break;
		if (retry + 1 < IGB_IOV_QUEUE_DISABLE_RETRIES) {
			if (retry < IGB_IOV_QUEUE_DISABLE_BUSY_RETRIES)
				DELAY(IGB_IOV_QUEUE_DISABLE_DELAY_US);
			else
				pause_sbt("igbqds",
				    IGB_IOV_QUEUE_DISABLE_PAUSE, 0,
				    C_PREL(1));
		}
	}
	if (retry == IGB_IOV_QUEUE_DISABLE_RETRIES) {
		if (ratecheck(&vf->last_queue_log,
		    &igb_iov_mbx_log_interval))
			device_printf(sc->dev,
			    "could not disable queues for VF %u; "
			    "reset deferred\n", vf->pool);
		return (false);
	}

	for (i = 0; i < nqueues; i++) {
		E1000_WRITE_REG(hw, E1000_SRRCTL(qid[i]), 0);
		E1000_WRITE_REG(hw, E1000_DCA_RXCTRL(qid[i]), 0);
		E1000_WRITE_REG(hw, E1000_TDWBAL(qid[i]), 0);
		E1000_WRITE_REG(hw, E1000_TDWBAH(qid[i]), 0);
		E1000_WRITE_REG(hw, E1000_DCA_TXCTRL(qid[i]), 0);
	}
	E1000_WRITE_REG(hw, E1000_PSRTYPE(vf->pool), 0);
	E1000_WRITE_FLUSH(hw);
	return (true);
}

static bool
igb_iov_reset_event_common(struct e1000_softc *sc, struct igb_vf *vf,
    bool reset_intrs)
{
	struct e1000_hw *hw;
	bool rebuild_mta, rebuild_vlan, sanitized;
	u32 reg;

	hw = &sc->hw;
	rebuild_mta = vf->mc_count != 0;
	rebuild_vlan = !igb_iov_vf_vlan_is_default(vf);
	reg = E1000_READ_REG(hw, E1000_VFTE);
	E1000_WRITE_REG(hw, E1000_VFTE, reg & ~(1U << vf->pool));
	reg = E1000_READ_REG(hw, E1000_VFRE);
	E1000_WRITE_REG(hw, E1000_VFRE, reg & ~(1U << vf->pool));
	if (reset_intrs)
		E1000_WRITE_REG(hw, E1000_VTCTRL(vf->pool),
		    E1000_VTCTRL_RST);
	sanitized = igb_iov_sanitize_vf_queues(sc, vf);
	E1000_WRITE_REG(hw, E1000_VMVIR(vf->pool), 0);
	igb_iov_clear_mac_filters(sc, vf);
	igb_iov_clear_rar(sc, vf->rar_index);
	igb_iov_reset_vf_state(sc, vf);
	if (rebuild_mta)
		igb_iov_rebuild_mta(sc);
	if (rebuild_vlan)
		igb_iov_rebuild_vlan(sc);
	return (sanitized);
}

static bool
igb_iov_reset_event(struct e1000_softc *sc, struct igb_vf *vf)
{
	return (igb_iov_reset_event_common(sc, vf, true));
}

static void
igb_iov_mdd_reset_event(struct e1000_softc *sc, struct igb_vf *vf)
{
	/*
	 * VTCTRL.RST clears the VF's queue-enable and interrupt registers
	 * (I350 section 8.28.1).  It therefore also removes the admin-vector
	 * route needed to deliver the reset notification below.  MDD recovery
	 * explicitly permits toggling VFTE instead (section 7.8.3.8.3).
	 *
	 * Leave the interrupt registers intact, keep VFTE/VFRE disabled until
	 * the VF completes a new reset handshake, and use the no-CTS control
	 * message to make the guest reinitialize.  FreeBSD and DPDK consume
	 * that message directly; Linux ACKs it and the PF's non-CTS ACK path
	 * replies with the NACK that schedules igbvf's reset task.
	 *
	 * Sanitization failure leaves the pool disabled.  The VF reset
	 * handshake retries it and is NACKed while a queue remains active.
	 */
	(void)igb_iov_reset_event_common(sc, vf, false);
}

static void
igb_iov_reset_msg(struct e1000_softc *sc, struct igb_vf *vf)
{
	struct e1000_hw *hw;
	u32 msg[3], reg;

	hw = &sc->hw;
	if (!igb_iov_reset_event(sc, vf)) {
		msg[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_NACK;
		e1000_write_mbx(hw, msg, 1, vf->pool);
		return;
	}
	igb_iov_map_rar(sc, vf->rar_index, vf->mac, vf->pool);
	igb_iov_set_anti_spoof(sc, vf);

	reg = E1000_READ_REG(hw, E1000_VFTE);
	E1000_WRITE_REG(hw, E1000_VFTE, reg | (1U << vf->pool));
	reg = E1000_READ_REG(hw, E1000_VFRE);
	E1000_WRITE_REG(hw, E1000_VFRE, reg | (1U << vf->pool));
	/*
	 * 82576's WVBR blocked bitmap is read-clear, so the reset handshake
	 * completes that event's lifetime.  I350 MDFB might be read-only;
	 * re-arm its edge latch only after a valid MDFB sample reads clear.
	 */
	if (hw->mac.type == e1000_82576)
		vf->flags &= ~IGB_VF_MDD_BLOCKED;
	vf->flags |= IGB_VF_CTS;

	memset(msg, 0, sizeof(msg));
	msg[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
	memcpy(&msg[1], vf->mac, ETHER_ADDR_LEN);
	e1000_write_mbx(hw, msg, 3, vf->pool);
}

static int
igb_iov_set_mac_filter(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg)
{
	struct igb_vf_mac_filter *filter, *free_filter;
	const u8 *mac;
	u32 info;
	int count, i;

	info = msg[0] & E1000_VT_MSGINFO_MASK;
	if (info == E1000_VF_MAC_FILTER_CLR) {
		igb_iov_clear_mac_filters(sc, vf);
		return (0);
	}
	if (info != E1000_VF_MAC_FILTER_ADD)
		return (EINVAL);
	if ((vf->flags & IGB_VF_CAP_MAC) == 0)
		return (EPERM);

	mac = (const u8 *)&msg[1];
	if (!igb_iov_mac_valid(mac))
		return (EINVAL);
	if (memcmp(mac, vf->mac, ETHER_ADDR_LEN) == 0)
		return (0);

	count = 0;
	free_filter = NULL;
	for (i = 0; i < sc->num_vf_mac_filters; i++) {
		filter = &sc->vf_mac_filters[i];
		if (!filter->active) {
			if (free_filter == NULL)
				free_filter = filter;
			continue;
		}
		if (memcmp(filter->mac, mac, ETHER_ADDR_LEN) != 0)
			continue;
		return (filter->pool == vf->pool ? 0 : EADDRINUSE);
	}
	for (i = 0; i < sc->num_vf_mac_filters; i++)
		if (sc->vf_mac_filters[i].active &&
		    sc->vf_mac_filters[i].pool == vf->pool)
			count++;
	if (igb_iov_mac_in_use(sc, mac, vf))
		return (EADDRINUSE);
	if (count >= IGB_IOV_MAX_MAC_FILTERS)
		return (ENOSPC);
	if (free_filter == NULL)
		return (ENOSPC);

	free_filter->active = true;
	free_filter->pool = vf->pool;
	memcpy(free_filter->mac, mac, ETHER_ADDR_LEN);
	igb_iov_map_rar(sc, free_filter->rar_index, free_filter->mac, vf->pool);
	return (0);
}

static int
igb_iov_set_mac(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg)
{
	u8 *mac;

	if ((msg[0] & E1000_VT_MSGINFO_MASK) != 0)
		return (igb_iov_set_mac_filter(sc, vf, msg));

	mac = (u8 *)&msg[1];
	if (!igb_iov_mac_valid(mac))
		return (EINVAL);
	if (memcmp(mac, vf->mac, ETHER_ADDR_LEN) != 0 &&
	    !(vf->flags & IGB_VF_CAP_MAC))
		return (EPERM);
	if (memcmp(mac, vf->mac, ETHER_ADDR_LEN) != 0 &&
	    igb_iov_mac_in_use(sc, mac, vf))
		return (EADDRINUSE);

	memcpy(vf->mac, mac, ETHER_ADDR_LEN);
	igb_iov_map_rar(sc, vf->rar_index, vf->mac, vf->pool);
	return (0);
}

static int
igb_iov_set_multicast(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg)
{
	u16 hashes[IGB_IOV_MAX_MC_HASHES] = {};
	bool overflow;
	int count, i;

	count = (msg[0] & E1000_VF_SET_MULTICAST_COUNT_MASK) >>
	    E1000_VT_MSGINFO_SHIFT;
	overflow = count > IGB_IOV_MAX_MC_HASHES ||
	    (msg[0] & E1000_VF_SET_MULTICAST_OVERFLOW) != 0;
	count = min(count, IGB_IOV_MAX_MC_HASHES);
	for (i = 0; i < count; i++)
		hashes[i] =
		    (msg[1 + i / 2] >> ((i & 1) * 16)) & 0xffff;
	if (vf->mc_count == count &&
	    ((vf->flags & IGB_VF_MCAST_OVERFLOW) != 0) == overflow &&
	    memcmp(vf->mc_hashes, hashes, sizeof(hashes)) == 0)
		return (0);
	memcpy(vf->mc_hashes, hashes, sizeof(vf->mc_hashes));
	vf->mc_count = count;
	if (overflow)
		vf->flags |= IGB_VF_MCAST_OVERFLOW;
	else
		vf->flags &= ~IGB_VF_MCAST_OVERFLOW;
	if (overflow &&
	    (vf->flags & IGB_VF_MCAST_OVERFLOW_WARNED) == 0) {
		vf->flags |= IGB_VF_MCAST_OVERFLOW_WARNED;
		device_printf(sc->dev,
		    "VF %u multicast list exceeds 30 entries; "
		    "enabling all-multicast reception\n", vf->pool);
	}
	igb_iov_configure_vmolr(sc, vf);
	igb_iov_rebuild_mta(sc);
	return (0);
}

static int
igb_iov_set_lpe(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg)
{
	u32 size;

	size = msg[1];
	if (size < ETHER_MIN_LEN)
		return (EINVAL);
	vf->max_frame_size = min(size, IGB_IOV_MAX_FRAME_SIZE);
	igb_iov_configure_vmolr(sc, vf);
	return (0);
}

static int
igb_iov_set_promisc(struct e1000_softc *sc, struct igb_vf *vf, u32 msg)
{
	u32 mode;

	mode = msg & E1000_VT_MSGINFO_MASK;
	if (mode & ~(E1000_VF_SET_PROMISC_UNICAST |
	    E1000_VF_SET_PROMISC_MULTICAST))
		return (EINVAL);
	if (mode != 0 && !(vf->flags & IGB_VF_ALLOW_PROMISC))
		return (EPERM);

	vf->flags &= ~(IGB_VF_UCAST_PROMISC | IGB_VF_MCAST_PROMISC);
	if (mode & E1000_VF_SET_PROMISC_UNICAST)
		vf->flags |= IGB_VF_UCAST_PROMISC;
	if (mode & E1000_VF_SET_PROMISC_MULTICAST)
		vf->flags |= IGB_VF_MCAST_PROMISC;
	igb_iov_configure_vmolr(sc, vf);
	igb_iov_set_uta(sc);
	return (0);
}

static bool
igb_iov_process_msg(struct e1000_softc *sc, struct igb_vf *vf)
{
	struct e1000_hw *hw;
	u32 msg[E1000_VFMAILBOX_SIZE], type;
	int error;

	hw = &sc->hw;
	memset(msg, 0, sizeof(msg));
	if (e1000_read_mbx(hw, msg, nitems(msg), vf->pool, false) != 0)
		return (false);
	vf->flags &= ~IGB_VF_MBX_PENDING;
	vf->mbx_retry_at = 0;
	vf->mbx_retry_count = 0;

	if (msg[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK)) {
		e1000_unlock_mbx(hw, vf->pool);
		return (true);
	}
	if (msg[0] == E1000_VF_RESET) {
		igb_iov_reset_msg(sc, vf);
		return (true);
	}
	if (!(vf->flags & IGB_VF_CTS)) {
		if (igb_iov_nack_allowed(vf)) {
			msg[0] = igb_iov_reply_header(msg[0], false, false);
			e1000_write_mbx(hw, msg, 1, vf->pool);
		} else
			e1000_unlock_mbx(hw, vf->pool);
		return (true);
	}

	type = msg[0] & 0xffff;
	switch (type) {
	case E1000_VF_SET_MAC_ADDR:
		error = igb_iov_set_mac(sc, vf, msg);
		break;
	case E1000_VF_SET_MULTICAST:
		error = igb_iov_set_multicast(sc, vf, msg);
		break;
	case E1000_VF_SET_VLAN:
		if (vf->default_vlan != 0)
			error = EPERM;
		else if ((msg[1] & ~E1000_VLVF_VLANID_MASK) != 0)
			error = EINVAL;
		else
			error = igb_iov_set_vlan(sc, vf,
			    msg[1] & E1000_VLVF_VLANID_MASK,
			    (msg[0] & E1000_VF_SET_VLAN_ADD) != 0);
		break;
	case E1000_VF_SET_LPE:
		error = igb_iov_set_lpe(sc, vf, msg);
		break;
	case E1000_VF_SET_PROMISC:
		error = igb_iov_set_promisc(sc, vf, msg[0]);
		break;
	default:
		error = EOPNOTSUPP;
		break;
	}

	msg[0] = igb_iov_reply_header(msg[0], true, error == 0);
	e1000_write_mbx(hw, msg, 1, vf->pool);
	return (true);
}

static sbintime_t
igb_iov_service_pending_mbx(struct e1000_softc *sc, struct igb_vf *vf,
    sbintime_t now)
{
	sbintime_t delay;

	if ((vf->flags & IGB_VF_MBX_PENDING) == 0)
		return (0);
	if (vf->mbx_retry_at != 0 && now < vf->mbx_retry_at)
		return (vf->mbx_retry_at);
	if (igb_iov_process_msg(sc, vf))
		return (0);

	now = getsbinuptime();
	if (vf->mbx_retry_count < IGB_IOV_MBX_RETRY_COUNT) {
		delay = igb_iov_mbx_retry_delay[vf->mbx_retry_count++];
		vf->mbx_retry_at = now + delay;
		return (vf->mbx_retry_at);
	}

	vf->flags &= ~(IGB_VF_CTS | IGB_VF_MBX_PENDING);
	vf->flags |= IGB_VF_MBX_GAVE_UP;
	vf->mbx_retry_at = 0;
	if (ratecheck(&vf->last_mbx_log, &igb_iov_mbx_log_interval))
		device_printf(sc->dev,
		    "mailbox remained busy for VF %u; CTS revoked\n",
		    vf->pool);
	return (0);
}

void
igb_iov_handle_mbx(struct e1000_softc *sc)
{
	struct e1000_hw *hw;
	struct igb_vf *vf;
	sbintime_t delay, next_retry_at, now, retry_at;
	u32 msg;
	int i;

	if (!sc->iov_hw_active)
		return;

	hw = &sc->hw;
	next_retry_at = 0;
	for (i = 0; i < sc->num_vfs; i++) {
		vf = &sc->vfs[i];
		if (!(vf->flags & IGB_VF_ACTIVE))
			continue;
		now = getsbinuptime();
		if (e1000_check_for_rst(hw, vf->pool) == 0) {
			/*
			 * The old VF is gone.  A new owner's reset handshake
			 * reruns sanitization before enabling its pool.
			 */
			(void)igb_iov_reset_event(sc, vf);
		}
		if ((vf->flags &
		    (IGB_VF_MBX_PENDING | IGB_VF_MBX_GAVE_UP)) == 0 &&
		    e1000_check_for_msg(hw, vf->pool) == 0) {
			vf->flags |= IGB_VF_MBX_PENDING;
			vf->mbx_retry_at = 0;
			vf->mbx_retry_count = 0;
		}
		retry_at = igb_iov_service_pending_mbx(sc, vf, now);
		if (retry_at != 0 &&
		    (next_retry_at == 0 || retry_at < next_retry_at))
			next_retry_at = retry_at;
		if (e1000_check_for_ack(hw, vf->pool) == 0 &&
		    !(vf->flags & IGB_VF_CTS) && igb_iov_nack_allowed(vf)) {
			msg = E1000_VT_MSGTYPE_NACK;
			e1000_write_mbx(hw, &msg, 1, vf->pool);
		}
	}
	if (next_retry_at != 0) {
		delay = next_retry_at - getsbinuptime();
		if (delay <= 0)
			delay = SBT_1MS;
		callout_reset_sbt(&sc->iov_mbx_retry, delay, 0,
		    igb_iov_mbx_retry_callout, sc, C_PREL(1));
	}
}

static bool
igb_iov_notify_vf_mdd_reset(struct e1000_softc *sc, struct igb_vf *vf)
{
	u32 msg;

	/*
	 * MDD recovery preserves the VF's admin-vector configuration.  Send
	 * the same no-CTS control message used for PF reset notification so
	 * the VF discards its state and completes a new reset handshake.
	 * A failed write is retried from the timer-driven admin pass; the VF's
	 * transmit watchdog remains the final fallback when traffic is still
	 * queued and notification never succeeds.
	 */
	msg = E1000_PF_CONTROL_MSG;
	if (e1000_write_mbx(&sc->hw, &msg, 1, vf->pool) != 0) {
		vf->mdd_notify_at =
		    getsbinuptime() + igb_iov_mdd_notify_retry;
		if (ratecheck(&vf->last_mbx_log,
		    &igb_iov_mbx_log_interval))
			device_printf(sc->dev,
			    "could not notify VF %u of malicious-driver "
			    "reset; will retry\n", vf->pool);
		return (false);
	}
	vf->flags &= ~IGB_VF_MDD_NOTIFY_PENDING;
	vf->mdd_notify_at = 0;
	return (true);
}

void
igb_iov_handle_mdd(struct e1000_softc *sc)
{
	struct igb_vf *vf;
	u32 blocked, cleared, handled, lvmmc;
	u32 readback, spoofed;
	bool mdfb_valid, pending;
	int i;

	pending = atomic_readandclear_32(&sc->iov_pending) != 0;
	lvmmc = pending ?
	    atomic_readandclear_32(&sc->iov_mdd_cause) : 0;
	if (!sc->iov_hw_active) {
		atomic_readandclear_32(&sc->iov_spoof_pending);
		atomic_readandclear_32(&sc->iov_blocked_pending);
		return;
	}

	blocked = 0;
	handled = 0;
	mdfb_valid = false;
	if (sc->hw.mac.type == e1000_i350) {
		u32 mdfb;

		spoofed = atomic_readandclear_32(&sc->iov_spoof_pending);
		/*
		 * I350 reports ordinary MAC/VLAN spoofing through the
		 * interrupt-time LVMMC snapshot rather than WVBR.  The
		 * filter accumulates Last_Q into iov_spoof_pending so events
		 * from different VFs coalesce safely until this timer-driven
		 * admin pass.
		 */
		spoofed &= IGB_I350_QUEUE_MASK;
		/*
		 * Sample MDFB on every admin pass so a blocked queue is not
		 * mislabeled as an ordinary spoof when no MDDET observation
		 * is pending.
		 */
		mdfb = E1000_READ_REG(&sc->hw, E1000_MDFB);
		if (__predict_false(mdfb == 0xffffffff))
			mdfb = 0;
		else {
			mdfb &= IGB_I350_QUEUE_MASK;
			mdfb_valid = true;
		}
		/*
		 * I350 SDM sections 8.14.10 and 8.14.11: WVBR reports
		 * spoof and malicious-driver events, while MDFB identifies
		 * the queues actually blocked for malicious behavior.
		 */
		spoofed &= ~mdfb;
		blocked = mdfb;
		if (blocked != 0 && lvmmc == 0)
			lvmmc = E1000_READ_REG(&sc->hw, E1000_LVMMC);
		/*
		 * A failed diagnostic read does not invalidate the
		 * blocked-queue bitmap that was read successfully above.
		 */
		if (__predict_false(lvmmc == 0xffffffff))
			lvmmc = 0;
		/*
		 * MDFB is authoritative for queues stopped by malicious-driver
		 * detection.  LVMMC reports causes such as VLAN IERR and
		 * Mal_PF, but its Last_Q field does not establish that a queue
		 * was blocked.  Do not manufacture a blocked bit when MDFB is
		 * clear.
		 */
	} else {
		if (!pending)
			return;
		/*
		 * WVBR is read-clear and does not preserve every queue across
		 * multiple MDDET interrupts.  The interrupt filter snapshots and
		 * accumulates its pool bitmaps before this deferred admin pass.
		 */
		spoofed = atomic_readandclear_32(&sc->iov_spoof_pending);
		blocked = atomic_readandclear_32(&sc->iov_blocked_pending);
		/* A blocked-queue classification dominates its WVBR low bit. */
		spoofed &= ~blocked;
	}

	for (i = 0; i < sc->num_vfs; i++) {
		vf = &sc->vfs[i];
		if (!(vf->flags & IGB_VF_ACTIVE))
			continue;
		if ((vf->flags & IGB_VF_MDD_NOTIFY_PENDING) != 0 &&
		    getsbinuptime() >= vf->mdd_notify_at)
			(void)igb_iov_notify_vf_mdd_reset(sc, vf);
		/*
		 * An invalid MDFB sample must neither report a new edge nor
		 * masquerade as evidence that an old edge has cleared.
		 */
		if (sc->hw.mac.type == e1000_i350 && mdfb_valid &&
		    (blocked & (1U << i)) == 0)
			vf->flags &= ~IGB_VF_MDD_BLOCKED;
		if ((spoofed & (1U << i)) != 0 &&
		    ratecheck(&vf->last_spoof_log,
		    &igb_iov_spoof_log_interval))
			device_printf(sc->dev,
			    "spoof event detected from VF %u; packet dropped\n",
			    vf->pool);
		if ((blocked & (1U << i)) == 0)
			continue;
		if ((vf->flags & IGB_VF_MDD_BLOCKED) != 0)
			continue;
		vf->flags |= IGB_VF_MDD_BLOCKED;
		if (ratecheck(&vf->last_mdd_log, &igb_iov_mdd_log_interval))
			device_printf(sc->dev,
			    "malicious-driver event 0x%08x from VF %u; "
			    "resetting VF\n", lvmmc, vf->pool);
		igb_iov_mdd_reset_event(sc, vf);
		vf->flags |= IGB_VF_MDD_NOTIFY_PENDING;
		(void)igb_iov_notify_vf_mdd_reset(sc, vf);
		handled |= 1U << i;
	}
	if (sc->hw.mac.type == e1000_i350 && mdfb_valid &&
	    (blocked & (1U << sc->pool)) == 0)
		sc->iov_pf_mdd_blocked = false;
	if ((blocked & (1U << sc->pool)) != 0 &&
	    (sc->hw.mac.type != e1000_i350 ||
	    !sc->iov_pf_mdd_blocked)) {
		if (sc->hw.mac.type == e1000_i350)
			sc->iov_pf_mdd_blocked = true;
		if (ratecheck(&sc->iov_last_mdd_log,
		    &igb_iov_mdd_log_interval))
			device_printf(sc->dev,
			    "malicious-driver event 0x%08x from PF queue; "
			    "resetting PF\n", lvmmc);
		iflib_request_reset(sc->ctx);
		iflib_admin_intr_deferred(sc->ctx);
		handled |= 1U << sc->pool;
	}
	if (sc->hw.mac.type == e1000_i350 && handled != 0) {
		/*
		 * I350 documentation conflicts: the register summary calls
		 * MDFB RWS while the detailed field table calls it RO.  I350
		 * silicon clears a blocked bit when software writes it back.
		 * Write only bits whose recovery was initiated.  If a revision
		 * instead implements MDFB as RO, the edge latch above prevents
		 * a reset loop and this one transition-time write is harmless.
		 */
		E1000_WRITE_REG(&sc->hw, E1000_MDFB, handled);
		E1000_WRITE_FLUSH(&sc->hw);
		/*
		 * Rearm from observed hardware state instead of waiting for
		 * the next admin pass.  The PF context lock prevents a reset
		 * handshake from re-enabling the VF before this readback.  A
		 * write-to-clear part reports zero; a read-only part retains
		 * the bit and therefore retains the one-shot edge latch.
		 */
		readback = E1000_READ_REG(&sc->hw, E1000_MDFB);
		if (__predict_false(readback == 0xffffffff))
			cleared = 0;
		else
			cleared = handled &
			    ~(readback & IGB_I350_QUEUE_MASK);
		for (i = 0; i < sc->num_vfs; i++)
			if ((cleared & (1U << i)) != 0)
				sc->vfs[i].flags &= ~IGB_VF_MDD_BLOCKED;
		if ((cleared & (1U << sc->pool)) != 0)
			sc->iov_pf_mdd_blocked = false;
	}
	if (sc->hw.mac.type == e1000_i350) {
		/*
		 * I350 can retain EICR.OTHER without delivering the admin MSI-X
		 * even though its EIMS and legacy IMS bits remain enabled.  Kick
		 * the already-enabled vector on each admin pass so its filter
		 * consumes any retained ICR/LVMMC cause.  A synthetic interrupt
		 * with no legacy cause is handled entirely by the filter.
		 */
		E1000_WRITE_REG(&sc->hw, E1000_EICS, sc->link_mask);
		E1000_WRITE_FLUSH(&sc->hw);
	}
}

void
igb_iov_mdd_event(struct e1000_softc *sc)
{
	u32 blocked, cause, queues, queue, spoofed, wvbr;

	/*
	 * LVMMC is clear-on-read.  Preserve it in the interrupt filter, as
	 * Intel's igb driver does, rather than deferring the only copy.
	 */
	cause = E1000_READ_REG(&sc->hw, E1000_LVMMC);
	if (__predict_false(cause == 0xffffffff))
		return;
	if (sc->hw.mac.type == e1000_82576) {
		/*
		 * Snapshot WVBR in the interrupt filter.  Waiting for the admin
		 * task loses all but the last of back-to-back VF MDD events on
		 * 82576.  Convert the staggered queue map into pool bits and OR
		 * each observation into software latches for deferred recovery.
		 */
		wvbr = E1000_READ_REG(&sc->hw, E1000_WVBR);
		if (__predict_false(wvbr == 0xffffffff)) {
			spoofed = 0;
			blocked = 0;
		} else {
			queues = wvbr & IGB_82576_QUEUE_MASK;
			spoofed = (queues & 0xff) |
			    (queues >> IGB_82576_STAGGERED_QUEUE_SHIFT);
			queues = (wvbr >> 16) & IGB_82576_QUEUE_MASK;
			blocked = (queues & 0xff) |
			    (queues >> IGB_82576_STAGGERED_QUEUE_SHIFT);
		}
		SDT_PROBE4(igb_iov, mdd, sample, wvbr, sc, wvbr, spoofed,
		    blocked);
		/*
		 * 82576 can report a coalesced block-class event with all affected
		 * queues in WVBR's low half and no high-half blocked bits.  If an
		 * ordinary spoof shares that snapshot, the register has no per-queue
		 * cause information.  Deliberately fail closed by recovering every
		 * low-half queue; this can reset a spoof-only sibling, but avoids
		 * stranding a blocked VF.  LVMMC.Last_Q identifies only the final
		 * event and lost simultaneous blocked VFs on tested silicon.
		 */
		if (blocked == 0 &&
		    (cause & IGB_82576_LVMMC_BLOCK_MASK) != 0) {
			blocked = spoofed;
			if (blocked == 0) {
				queue = (cause >> 16) & 0xf;
				blocked = 1U << (queue & 0x7);
			}
		}
		if (spoofed != 0)
			atomic_set_32(&sc->iov_spoof_pending, spoofed);
		if (blocked != 0)
			atomic_set_32(&sc->iov_blocked_pending, blocked);
	}
	if (sc->hw.mac.type == e1000_i350 &&
	    (cause & IGB_I350_LVMMC_MAC_VLAN_SPOOF) != 0) {
		queue = (cause >> IGB_I350_LVMMC_LAST_Q_SHIFT) &
		    IGB_I350_LVMMC_LAST_Q_MASK;
		/*
		 * FreeBSD assigns one queue to each VF pool, so Last_Q is
		 * also the VF number.  Preserve all VFs observed before the
		 * timer pass, and do not overwrite an unrelated blocked
		 * queue's diagnostic with this non-blocking spoof event.
		 */
		atomic_set_32(&sc->iov_spoof_pending, 1U << queue);
		return;
	}
	atomic_store_rel_32(&sc->iov_mdd_cause, cause);
	atomic_set_32(&sc->iov_pending, 1);
}

void
igb_iov_ping_all_vfs(struct e1000_softc *sc)
{
	struct igb_vf *vf;
	u32 msg;
	int i;

	if (!sc->iov_hw_active)
		return;

	for (i = 0; i < sc->num_vfs; i++) {
		vf = &sc->vfs[i];
		if (!(vf->flags & IGB_VF_ACTIVE))
			continue;
		msg = E1000_PF_CONTROL_MSG;
		if (vf->flags & IGB_VF_CTS)
			msg |= E1000_VT_MSGTYPE_CTS;
		e1000_write_mbx(&sc->hw, &msg, 1, vf->pool);
	}
}

void
igb_iov_initialize(struct e1000_softc *sc)
{
	struct e1000_hw *hw;
	struct igb_vf *vf;
	u32 ctrl_ext, dtxctl, mask, rctl, rplolr, vt_ctl;
	int i;

	if (sc->num_vfs == 0)
		return;

	hw = &sc->hw;
	atomic_readandclear_32(&sc->iov_mdd_cause);
	atomic_readandclear_32(&sc->iov_pending);
	atomic_readandclear_32(&sc->iov_spoof_pending);
	atomic_readandclear_32(&sc->iov_blocked_pending);
	/* Plain VMDq keeps every 82576 PF/VF pool on queue zero. */
	E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_ENABLE_VMDQ);

	vt_ctl = E1000_READ_REG(hw, E1000_VT_CTL);
	vt_ctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
	    E1000_VT_CTL_DISABLE_DEF_POOL);
	vt_ctl |= sc->pool << E1000_VT_CTL_DEFAULT_POOL_SHIFT;
	vt_ctl |= E1000_VT_CTL_VM_REPL_EN;
	E1000_WRITE_REG(hw, E1000_VT_CTL, vt_ctl);

	mask = 1U << sc->pool;
	E1000_WRITE_REG(hw, E1000_VFRE, mask);
	E1000_WRITE_REG(hw, E1000_VFTE, mask);
	/* A VF without RX descriptors must not block any other pool. */
	E1000_WRITE_REG(hw, E1000_QDE,
	    hw->mac.type == e1000_i350 ? IGB_I350_QUEUE_MASK : ALL_QUEUES);
	e1000_vmdq_set_loopback_pf(hw, true);
	dtxctl = E1000_READ_REG(hw, E1000_DTXCTL);
	dtxctl |= E1000_DTXCTL_MDP_EN;
	if (hw->mac.type == e1000_82576) {
		dtxctl |= E1000_DTXCTL_VLAN_ADDED |
		    E1000_DTXCTL_SPOOF_INT;
		rplolr = E1000_READ_REG(hw, E1000_RPLOLR);
		rplolr |= E1000_RPLOLR_STRVLAN;
		E1000_WRITE_REG(hw, E1000_RPLOLR, rplolr);
	} else {
		/*
		 * I350 SDM section 8.12.5 defines this field with inverted
		 * polarity: setting it keeps an ordinary spoof from disabling
		 * the VF queue.  Enable its notification as well.  I350
		 * hardware reports the VF in LVMMC.Last_Q (WVBR remains zero);
		 * the moderated admin vector captures that value, while
		 * timer-driven administration and per-VF ratecheck bound the
		 * work and console output.
		 */
		dtxctl |= E1000_DTXCTL_SPOOF_INT |
		    IGB_I350_DTXCTL_ENABLE_SPOOF_QUEUE;
	}
	E1000_WRITE_REG(hw, E1000_DTXCTL, dtxctl);

	igb_iov_map_rar(sc, 0, hw->mac.addr, sc->pool);
	igb_iov_configure_pf_vmolr(sc);
	igb_iov_set_uta(sc);
	for (i = 0; i < sc->num_vfs; i++) {
		vf = &sc->vfs[i];
		if (!(vf->flags & IGB_VF_ACTIVE))
			continue;
		/*
		 * A PF-wide reset is trusted and can require a complete guest
		 * replay.  Guest-controlled RESET and VFLR do not refill this
		 * allowance.
		 */
		igb_iov_reset_vlan_rate(vf);
		igb_iov_clear_mac_filters(sc, vf);
		igb_iov_reset_vf_state(sc, vf);
		igb_iov_clear_rar(sc, vf->rar_index);
		igb_iov_set_anti_spoof(sc, vf);
	}
	igb_iov_rebuild_mta(sc);
	igb_iov_rebuild_vlan(sc);

	rctl = E1000_READ_REG(hw, E1000_RCTL);
	E1000_WRITE_REG(hw, E1000_RCTL, rctl | E1000_RCTL_VFE);
	E1000_WRITE_REG(hw, E1000_MBVFIMR, igb_iov_active_mask(sc));
	if (hw->mac.type == e1000_i350)
		E1000_WRITE_REG(hw, E1000_DMACR, 0);

	ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
	E1000_WRITE_REG(hw, E1000_CTRL_EXT,
	    ctrl_ext | E1000_CTRL_EXT_PFRSTD);
	E1000_WRITE_FLUSH(hw);
	/*
	 * MDDET remains masked until iov_hw_active is published and iflib
	 * rearms the admin vector.  Programming the per-pool policy above can
	 * leave a setup-time MDDET observation in the read-clear registers.
	 * If that stale cause is carried across the unmask, a later ordinary
	 * spoof can update LVMMC without generating a new interrupt edge.
	 *
	 * Drain only after all IOV policy is installed and before exposing the
	 * active state.  Mailbox requests are also serviced by the periodic
	 * admin pass, and ping_all_vfs() below supplies a fresh notification.
	 */
	/*
	 * Clear the setup-time interrupt latch before its diagnostic state.
	 * I350 does not reliably generate the next MDDET edge when LVMMC is
	 * consumed while ICR.MDDET remains latched.  This differs deliberately
	 * from the final arm-time drain, where ICR is read last so a later event
	 * remains pending for the unmask.
	 */
	if (hw->mac.type == e1000_i350)
		(void)E1000_READ_REG(hw, E1000_ICR);
	(void)E1000_READ_REG(hw, E1000_LVMMC);
	if (hw->mac.type == e1000_82576)
		(void)E1000_READ_REG(hw, E1000_WVBR);
	if (hw->mac.type != e1000_i350)
		(void)E1000_READ_REG(hw, E1000_ICR);
	atomic_readandclear_32(&sc->iov_mdd_cause);
	atomic_readandclear_32(&sc->iov_pending);
	atomic_readandclear_32(&sc->iov_spoof_pending);
	atomic_readandclear_32(&sc->iov_blocked_pending);
	atomic_store_rel_32(&sc->iov_intr_drain_pending, 1);
	sc->iov_hw_active = true;
	igb_iov_ping_all_vfs(sc);
}

int
igb_iov_validate(struct e1000_softc *sc, u16 num_vfs)
{
	if (!igb_iov_supported(sc))
		return (ENXIO);
	/* One of the eight hardware pools is reserved for the PF. */
	if (num_vfs == 0 || num_vfs > MAX_NUM_VFS)
		return (EINVAL);
	if (sc->vfs != NULL)
		return (EBUSY);
	if (sc->intr_type != IFLIB_INTR_MSIX) {
		device_printf(sc->dev, "SR-IOV requires MSI-X\n");
		return (ENOTSUP);
	}
	if (sc->tx_num_queues != 1 || sc->rx_num_queues != 1) {
		device_printf(sc->dev,
		    "SR-IOV requires one PF TX and RX queue; set "
		    "dev.igb.%d.iflib.override_ntxqs=1 and "
		    "dev.igb.%d.iflib.override_nrxqs=1 before attach\n",
		    device_get_unit(sc->dev), device_get_unit(sc->dev));
		return (EINVAL);
	}
	return (0);
}

int
igb_if_iov_init(if_ctx_t ctx, u16 num_vfs, const nvlist_t *config)
{
	struct e1000_softc *sc;
	int error, i;

	sc = iflib_get_softc(ctx);
	(void)config;
	/*
	 * This callback may run while the PF is down.  Record the software
	 * layout here; igb_iov_initialize() programs it during interface init.
	 */
	atomic_store_rel_32(&sc->iov_teardown, 0);
	error = igb_iov_validate(sc, num_vfs);
	if (error != 0)
		return (error);

	sc->vfs = mallocarray(num_vfs, sizeof(*sc->vfs), M_IGB_IOV,
	    M_WAITOK | M_ZERO);
	sc->num_vf_mac_filters =
	    sc->hw.mac.rar_entry_count - num_vfs - 1;
	sc->vf_mac_filters = mallocarray(sc->num_vf_mac_filters,
	    sizeof(*sc->vf_mac_filters), M_IGB_IOV, M_WAITOK | M_ZERO);
	for (i = 0; i < sc->num_vf_mac_filters; i++)
		sc->vf_mac_filters[i].rar_index = i + 1;
	sc->pool = num_vfs;
	sc->iov_mta_valid = false;
	sc->iov_pf_mdd_blocked = false;
	sc->tx_queues[0].txr.me = sc->pool;
	sc->rx_queues[0].rxr.me = sc->pool;
	e1000_init_mbx_params_pf(&sc->hw);
	sc->num_vfs = num_vfs;
	return (0);
}

void
igb_if_iov_uninit(if_ctx_t ctx)
{
	struct e1000_softc *sc;
	struct e1000_hw *hw;
	u32 mask, rah;
	int error, i, iov_pos;
	u16 iov_ctl;

	sc = iflib_get_softc(ctx);
	if (sc->vfs == NULL)
		return;
	hw = &sc->hw;
	sc->iov_hw_active = false;
	if (sc->iov_mbx_retry_initialized)
		callout_drain(&sc->iov_mbx_retry);

	E1000_WRITE_REG(hw, E1000_MBVFIMR, 0);
	mask = 1U << sc->pool;
	E1000_WRITE_REG(hw, E1000_VFRE, mask);
	E1000_WRITE_REG(hw, E1000_VFTE, mask);

	/*
	 * pci_iov(4) invokes the driver before it clears VF Enable.  Quiesce
	 * the VFs and clear it here so that 82576's queue-reuse interval is
	 * measured from the actual IOV-disable event.
	 */
	error = pci_find_extcap(sc->dev, PCIZ_SRIOV, &iov_pos);
	if (error == 0) {
		iov_ctl = pci_read_config(sc->dev,
		    iov_pos + PCIR_SRIOV_CTL, 2);
		iov_ctl &= ~(PCIM_SRIOV_VF_EN | PCIM_SRIOV_VF_MSE);
		pci_write_config(sc->dev, iov_pos + PCIR_SRIOV_CTL,
		    iov_ctl, 2);
		if (hw->mac.type == e1000_82576) {
			pause("igbiov", MAX(1, howmany(hz, 10)));
			E1000_WRITE_REG(hw, E1000_IOVCTL,
			    E1000_IOVCTL_REUSE_VFQ);
			E1000_WRITE_FLUSH(hw);
			pause("igbiov", MAX(1, howmany(hz, 10)));
		}
	} else
		device_printf(sc->dev,
		    "could not disable PCI SR-IOV before queue reuse: %d\n",
		    error);

	E1000_WRITE_REG(hw, E1000_VT_CTL, 0);
	e1000_vmdq_set_loopback_pf(hw, false);
	e1000_vmdq_set_anti_spoofing_pf(hw, false, 0);
	for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
		E1000_WRITE_REG(hw, E1000_VLVF(i), 0);
	for (i = 0; i < sc->num_vfs; i++)
		if (sc->vfs[i].flags & IGB_VF_ACTIVE)
			igb_iov_clear_rar(sc, sc->vfs[i].rar_index);
	for (i = 0; i < sc->num_vf_mac_filters; i++)
		if (sc->vf_mac_filters[i].active)
			igb_iov_clear_rar(sc, sc->vf_mac_filters[i].rar_index);
	rah = E1000_READ_REG(hw, E1000_RAH(0));
	rah &= ~IGB_IOV_RAH_POOLSEL_MASK;
	E1000_WRITE_REG(hw, E1000_RAH(0), rah);

	free(sc->vfs, M_IGB_IOV);
	free(sc->vf_mac_filters, M_IGB_IOV);
	sc->vfs = NULL;
	sc->vf_mac_filters = NULL;
	sc->num_vfs = 0;
	sc->num_vf_mac_filters = 0;
	sc->pool = 0;
	sc->iov_mta_valid = false;
	sc->iov_pf_mdd_blocked = false;
	sc->iov_pf_vlan_promisc = false;
	igb_iov_vfta_shadow_invalidate(sc);
	sc->tx_queues[0].txr.me = 0;
	sc->rx_queues[0].rxr.me = 0;
	atomic_readandclear_32(&sc->iov_mdd_cause);
	atomic_readandclear_32(&sc->iov_pending);
	atomic_readandclear_32(&sc->iov_spoof_pending);
	atomic_readandclear_32(&sc->iov_blocked_pending);
	atomic_readandclear_32(&sc->iov_intr_drain_pending);
	atomic_store_rel_32(&sc->iov_teardown, 0);
}

static bool
igb_iov_mac_in_use(struct e1000_softc *sc, const u8 *mac,
    const struct igb_vf *skip)
{
	int i;

	if (memcmp(sc->hw.mac.addr, mac, ETHER_ADDR_LEN) == 0)
		return (true);
	for (i = 0; i < sc->num_vfs; i++)
		if (&sc->vfs[i] != skip &&
		    (sc->vfs[i].flags & IGB_VF_ACTIVE) != 0 &&
		    memcmp(sc->vfs[i].mac, mac, ETHER_ADDR_LEN) == 0)
			return (true);
	for (i = 0; i < sc->num_vf_mac_filters; i++)
		if (sc->vf_mac_filters[i].active &&
		    memcmp(sc->vf_mac_filters[i].mac, mac,
		    ETHER_ADDR_LEN) == 0)
			return (true);
	return (false);
}

int
igb_if_iov_vf_add(if_ctx_t ctx, u16 vfnum, const nvlist_t *config)
{
	struct e1000_softc *sc;
	struct igb_vf *vf;
	struct ether_addr generated;
	const void *mac;
	char nameunit[IFNAMSIZ + sizeof("-vf65535")];
	size_t mac_size;
	uint64_t configured_vlan;
	u16 vlan;

	sc = iflib_get_softc(ctx);
	if (vfnum >= sc->num_vfs)
		return (EINVAL);
	vf = &sc->vfs[vfnum];
	if (vf->flags & IGB_VF_ACTIVE)
		return (EBUSY);

	configured_vlan = nvlist_get_number(config, "vlan");
	if (configured_vlan > VF_VLAN_TRUNK)
		return (EINVAL);
	vlan = configured_vlan;
	if (vlan == 0)
		return (ENOTSUP);
	if (vlan == VF_VLAN_TRUNK)
		vlan = 0;
	if (!igb_iov_vlan_present(sc, vlan, false) &&
	    igb_iov_vlan_unique_count(sc, false) >=
	    E1000_VLVF_ARRAY_SIZE)
		return (ENOSPC);

	vf->pool = vfnum;
	vf->rar_index = sc->hw.mac.rar_entry_count - (vfnum + 1);
	vf->max_frame_size = ETHER_MAX_LEN;
	vf->default_vlan = vlan;
	igb_iov_reset_vlan_rate(vf);
	if (nvlist_exists_binary(config, "mac-addr")) {
		mac = nvlist_get_binary(config, "mac-addr", &mac_size);
		if (mac_size != ETHER_ADDR_LEN || !igb_iov_mac_valid(mac))
			return (EINVAL);
		if (igb_iov_mac_in_use(sc, mac, vf))
			return (EADDRINUSE);
		memcpy(vf->mac, mac, ETHER_ADDR_LEN);
	} else {
		snprintf(nameunit, sizeof(nameunit), "%s-vf%u",
		    device_get_nameunit(sc->dev), vfnum);
		ether_gen_addr_byname(nameunit, &generated);
		memcpy(vf->mac, generated.octet, ETHER_ADDR_LEN);
		if (igb_iov_mac_in_use(sc, vf->mac, vf))
			return (EADDRINUSE);
	}
	if (nvlist_get_bool(config, "allow-set-mac"))
		vf->flags |= IGB_VF_CAP_MAC;
	if (nvlist_get_bool(config, "mac-anti-spoof"))
		vf->flags |= IGB_VF_MAC_ANTI_SPOOF;
	if (nvlist_get_bool(config, "allow-promisc"))
		vf->flags |= IGB_VF_ALLOW_PROMISC;
	vf->flags |= IGB_VF_ACTIVE;

	igb_iov_reset_vf_state(sc, vf);
	igb_iov_set_anti_spoof(sc, vf);
	igb_iov_rebuild_vlan(sc);
	E1000_WRITE_REG(&sc->hw, E1000_MBVFIMR, igb_iov_active_mask(sc));
	return (0);
}

#endif /* PCI_IOV */