xref: /linux/drivers/net/dsa/netc/netc_main.c (revision 05b5ee610fbb8ca4ce9dc21299442aa827b38008)

// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
/*
 * NXP NETC switch driver
 * Copyright 2025-2026 NXP
 */

#include <linux/clk.h>
#include <linux/etherdevice.h>
#include <linux/fsl/enetc_mdio.h>
#include <linux/if_bridge.h>
#include <linux/if_vlan.h>
#include <linux/of_mdio.h>

#include "netc_switch.h"

static struct netc_fdb_entry *
netc_lookup_fdb_entry(struct netc_switch *priv,
		      const unsigned char *addr,
		      u16 vid)
{
	struct netc_fdb_entry *entry;

	hlist_for_each_entry(entry, &priv->fdb_list, node)
		if (ether_addr_equal(entry->keye.mac_addr, addr) &&
		    le16_to_cpu(entry->keye.fid) == vid)
			return entry;

	return NULL;
}

static void netc_destroy_fdb_list(struct netc_switch *priv)
{
	struct netc_fdb_entry *entry;
	struct hlist_node *tmp;

	hlist_for_each_entry_safe(entry, tmp, &priv->fdb_list, node)
		netc_del_fdb_entry(entry);
}

static struct netc_vlan_entry *
netc_lookup_vlan_entry(struct netc_switch *priv, u16 vid)
{
	struct netc_vlan_entry *entry;

	hlist_for_each_entry(entry, &priv->vlan_list, node)
		if (entry->vid == vid)
			return entry;

	return NULL;
}

static void netc_destroy_vlan_list(struct netc_switch *priv)
{
	struct netc_vlan_entry *entry;
	struct hlist_node *tmp;

	hlist_for_each_entry_safe(entry, tmp, &priv->vlan_list, node)
		netc_del_vlan_entry(entry);
}

static enum dsa_tag_protocol
netc_get_tag_protocol(struct dsa_switch *ds, int port,
		      enum dsa_tag_protocol mprot)
{
	return DSA_TAG_PROTO_NETC;
}

static void netc_port_rmw(struct netc_port *np, u32 reg,
			  u32 mask, u32 val)
{
	u32 old, new;

	WARN_ON((mask | val) != mask);

	old = netc_port_rd(np, reg);
	new = (old & ~mask) | val;
	if (new == old)
		return;

	netc_port_wr(np, reg, new);
}

static void netc_mac_port_wr(struct netc_port *np, u32 reg, u32 val)
{
	if (is_netc_pseudo_port(np))
		return;

	netc_port_wr(np, reg, val);
	if (np->caps.pmac)
		netc_port_wr(np, reg + NETC_PMAC_OFFSET, val);
}

/* netc_mac_port_rmw() is used to synchronize the configurations of eMAC
 * and pMAC to maintain consistency. This function should not be used if
 * differentiated settings are required.
 */
static void netc_mac_port_rmw(struct netc_port *np, u32 reg,
			      u32 mask, u32 val)
{
	u32 old, new;

	if (is_netc_pseudo_port(np))
		return;

	WARN_ON((mask | val) != mask);

	old = netc_port_rd(np, reg);
	new = (old & ~mask) | val;
	if (new == old)
		return;

	netc_port_wr(np, reg, new);
	if (np->caps.pmac)
		netc_port_wr(np, reg + NETC_PMAC_OFFSET, new);
}

static void netc_port_get_capability(struct netc_port *np)
{
	u32 val;

	val = netc_port_rd(np, NETC_PMCAPR);
	if (val & PMCAPR_HD)
		np->caps.half_duplex = true;

	if (FIELD_GET(PMCAPR_FP, val) == FP_SUPPORT)
		np->caps.pmac = true;

	val = netc_port_rd(np, NETC_PCAPR);
	if (val & PCAPR_LINK_TYPE)
		np->caps.pseudo_link = true;
}

static int netc_port_get_info_from_dt(struct netc_port *np,
				      struct device_node *node,
				      struct device *dev)
{
	if (of_find_property(node, "clock-names", NULL)) {
		np->ref_clk = devm_get_clk_from_child(dev, node, "ref");
		if (IS_ERR(np->ref_clk)) {
			dev_err(dev, "Port %d cannot get reference clock\n",
				np->dp->index);
			return PTR_ERR(np->ref_clk);
		}
	}

	return 0;
}

static int netc_port_create_emdio_bus(struct netc_port *np,
				      struct device_node *node)
{
	struct netc_switch *priv = np->switch_priv;
	struct enetc_mdio_priv *mdio_priv;
	struct device *dev = priv->dev;
	struct enetc_hw *hw;
	struct mii_bus *bus;
	int err;

	hw = enetc_hw_alloc(dev, np->iobase);
	if (IS_ERR(hw))
		return dev_err_probe(dev, PTR_ERR(hw),
				     "Failed to allocate enetc_hw\n");

	bus = devm_mdiobus_alloc_size(dev, sizeof(*mdio_priv));
	if (!bus)
		return -ENOMEM;

	bus->name = "NXP NETC switch external MDIO Bus";
	bus->read = enetc_mdio_read_c22;
	bus->write = enetc_mdio_write_c22;
	bus->read_c45 = enetc_mdio_read_c45;
	bus->write_c45 = enetc_mdio_write_c45;
	bus->parent = dev;
	mdio_priv = bus->priv;
	mdio_priv->hw = hw;
	mdio_priv->mdio_base = NETC_EMDIO_BASE;
	snprintf(bus->id, MII_BUS_ID_SIZE, "%s-p%d-emdio",
		 dev_name(dev), np->dp->index);

	err = devm_of_mdiobus_register(dev, bus, node);
	if (err)
		return dev_err_probe(dev, err,
				     "Cannot register EMDIO bus\n");

	np->emdio = bus;

	return 0;
}

static int netc_port_create_mdio_bus(struct netc_port *np,
				     struct device_node *node)
{
	struct device_node *mdio_node;
	int err;

	mdio_node = of_get_child_by_name(node, "mdio");
	if (mdio_node) {
		err = netc_port_create_emdio_bus(np, mdio_node);
		of_node_put(mdio_node);
		if (err)
			return err;
	}

	return 0;
}

static int netc_init_switch_id(struct netc_switch *priv)
{
	struct netc_switch_regs *regs = &priv->regs;
	struct dsa_switch *ds = priv->ds;

	/* The value of 0 is reserved for the VEPA switch and cannot
	 * be used. So 'dsa,member' is a required property for NETC
	 * switch, the member is used to specify the switch ID, which
	 * cannot be zero. This way, the hardware switch ID and the
	 * software switch ID are consistent.
	 */
	if (ds->index > FIELD_MAX(SWCR_SWID) || !ds->index) {
		dev_err(priv->dev, "Switch index %d out of range\n",
			ds->index);
		return -ERANGE;
	}

	netc_base_wr(regs, NETC_SWCR, ds->index);

	return 0;
}

static void netc_get_switch_capabilities(struct netc_switch *priv)
{
	struct netc_switch_regs *regs = &priv->regs;
	u32 val;

	val = netc_base_rd(regs, NETC_HTMCAPR);
	priv->htmcapr_num_words = FIELD_GET(HTMCAPR_NUM_WORDS, val);

	val = netc_base_rd(regs, NETC_BPCAPR);
	priv->num_bp = FIELD_GET(BPCAPR_NUM_BP, val);
}

static int netc_init_all_ports(struct netc_switch *priv)
{
	struct device *dev = priv->dev;
	struct netc_port *np;
	struct dsa_port *dp;
	int ett_offset = 0;
	int err;

	priv->ports = devm_kcalloc(dev, priv->info->num_ports,
				   sizeof(struct netc_port *),
				   GFP_KERNEL);
	if (!priv->ports)
		return -ENOMEM;

	/* Some DSA interfaces may set the port even it is disabled, such
	 * as .port_disable(), .port_stp_state_set() and so on. To avoid
	 * crash caused by accessing NULL port pointer, each port is
	 * allocated its own memory. Otherwise, we need to check whether
	 * the port pointer is NULL in these interfaces. The latter is
	 * difficult for us to cover.
	 */
	for (int i = 0; i < priv->info->num_ports; i++) {
		np = devm_kzalloc(dev, sizeof(*np), GFP_KERNEL);
		if (!np)
			return -ENOMEM;

		np->switch_priv = priv;
		np->iobase = priv->regs.port + PORT_IOBASE(i);
		netc_port_get_capability(np);
		priv->ports[i] = np;
	}

	dsa_switch_for_each_available_port(dp, priv->ds) {
		np = priv->ports[dp->index];
		np->dp = dp;
		np->ett_offset = ett_offset++;
		priv->port_bitmap |= BIT(dp->index);

		err = netc_port_get_info_from_dt(np, dp->dn, dev);
		if (err)
			return err;

		if (dsa_port_is_user(dp)) {
			err = netc_port_create_mdio_bus(np, dp->dn);
			if (err) {
				dev_err(dev, "Failed to create MDIO bus\n");
				return err;
			}
		}
	}

	return 0;
}

static void netc_init_ntmp_tbl_versions(struct netc_switch *priv)
{
	struct ntmp_user *ntmp = &priv->ntmp;

	/* All tables default to version 0 */
	memset(&ntmp->tbl, 0, sizeof(ntmp->tbl));
}

static int netc_init_all_cbdrs(struct netc_switch *priv)
{
	struct netc_switch_regs *regs = &priv->regs;
	struct ntmp_user *ntmp = &priv->ntmp;
	int i, err;

	ntmp->cbdr_num = NETC_CBDR_NUM;
	ntmp->dev = priv->dev;
	ntmp->ring = devm_kcalloc(ntmp->dev, ntmp->cbdr_num,
				  sizeof(struct netc_cbdr),
				  GFP_KERNEL);
	if (!ntmp->ring)
		return -ENOMEM;

	for (i = 0; i < ntmp->cbdr_num; i++) {
		struct netc_cbdr *cbdr = &ntmp->ring[i];
		struct netc_cbdr_regs cbdr_regs;

		cbdr_regs.pir = regs->base + NETC_CBDRPIR(i);
		cbdr_regs.cir = regs->base + NETC_CBDRCIR(i);
		cbdr_regs.mr = regs->base + NETC_CBDRMR(i);
		cbdr_regs.bar0 = regs->base + NETC_CBDRBAR0(i);
		cbdr_regs.bar1 = regs->base + NETC_CBDRBAR1(i);
		cbdr_regs.lenr = regs->base + NETC_CBDRLENR(i);

		err = ntmp_init_cbdr(cbdr, ntmp->dev, &cbdr_regs);
		if (err)
			goto free_cbdrs;
	}

	return 0;

free_cbdrs:
	for (i--; i >= 0; i--)
		ntmp_free_cbdr(&ntmp->ring[i]);

	return err;
}

static void netc_remove_all_cbdrs(struct netc_switch *priv)
{
	struct ntmp_user *ntmp = &priv->ntmp;

	for (int i = 0; i < NETC_CBDR_NUM; i++)
		ntmp_free_cbdr(&ntmp->ring[i]);
}

static u32 netc_num_available_ports(struct netc_switch *priv)
{
	struct dsa_port *dp;
	u32 num_ports = 0;

	dsa_switch_for_each_available_port(dp, priv->ds)
		num_ports++;

	return num_ports;
}

static int netc_init_ntmp_bitmap_sizes(struct netc_switch *priv)
{
	u32 num_ports = netc_num_available_ports(priv);
	struct netc_switch_regs *regs = &priv->regs;
	struct ntmp_user *ntmp = &priv->ntmp;
	u32 val;

	if (!num_ports)
		return -EINVAL;

	val = netc_base_rd(regs, NETC_ETTCAPR);
	ntmp->ett_bitmap_size = NETC_GET_NUM_ENTRIES(val) / num_ports;
	if (!ntmp->ett_bitmap_size)
		return -EINVAL;

	val = netc_base_rd(regs, NETC_ECTCAPR);
	ntmp->ect_bitmap_size = NETC_GET_NUM_ENTRIES(val) / num_ports;
	if (!ntmp->ect_bitmap_size)
		return -EINVAL;

	return 0;
}

static int netc_init_ntmp_bitmaps(struct netc_switch *priv)
{
	struct ntmp_user *ntmp = &priv->ntmp;

	ntmp->ett_gid_bitmap = bitmap_zalloc(ntmp->ett_bitmap_size,
					     GFP_KERNEL);
	if (!ntmp->ett_gid_bitmap)
		return -ENOMEM;

	ntmp->ect_gid_bitmap = bitmap_zalloc(ntmp->ect_bitmap_size,
					     GFP_KERNEL);
	if (!ntmp->ect_gid_bitmap)
		goto free_ett_gid_bitmap;

	return 0;

free_ett_gid_bitmap:
	bitmap_free(ntmp->ett_gid_bitmap);
	ntmp->ett_gid_bitmap = NULL;

	return -ENOMEM;
}

static void netc_free_ntmp_bitmaps(struct netc_switch *priv)
{
	struct ntmp_user *ntmp = &priv->ntmp;

	bitmap_free(ntmp->ect_gid_bitmap);
	ntmp->ect_gid_bitmap = NULL;

	bitmap_free(ntmp->ett_gid_bitmap);
	ntmp->ett_gid_bitmap = NULL;
}

static int netc_init_ntmp_user(struct netc_switch *priv)
{
	int err;

	netc_init_ntmp_tbl_versions(priv);

	err = netc_init_ntmp_bitmap_sizes(priv);
	if (err)
		return err;

	err = netc_init_ntmp_bitmaps(priv);
	if (err)
		return err;

	err = netc_init_all_cbdrs(priv);
	if (err)
		goto free_ntmp_bitmaps;

	return 0;

free_ntmp_bitmaps:
	netc_free_ntmp_bitmaps(priv);

	return err;
}

static void netc_free_ntmp_user(struct netc_switch *priv)
{
	netc_remove_all_cbdrs(priv);
	netc_free_ntmp_bitmaps(priv);
}

static void netc_clean_fdbt_ageing_entries(struct work_struct *work)
{
	struct delayed_work *dwork = to_delayed_work(work);
	struct netc_switch *priv;

	priv = container_of(dwork, struct netc_switch, fdbt_ageing_work);

	/* Update the activity element in FDB table */
	mutex_lock(&priv->fdbt_lock);
	ntmp_fdbt_update_activity_element(&priv->ntmp);
	/* Delete the ageing entries after the activity element is updated */
	ntmp_fdbt_delete_ageing_entries(&priv->ntmp, NETC_FDBT_AGEING_THRESH);
	mutex_unlock(&priv->fdbt_lock);

	if (atomic_read(&priv->br_cnt))
		schedule_delayed_work(&priv->fdbt_ageing_work,
				      READ_ONCE(priv->fdbt_ageing_delay));
}

static void netc_switch_dos_default_config(struct netc_switch *priv)
{
	struct netc_switch_regs *regs = &priv->regs;
	u32 val;

	val = DOSL2CR_SAMEADDR | DOSL2CR_MSAMCC;
	netc_base_wr(regs, NETC_DOSL2CR, val);

	val = DOSL3CR_SAMEADDR | DOSL3CR_IPSAMCC;
	netc_base_wr(regs, NETC_DOSL3CR, val);
}

static void netc_switch_vfht_default_config(struct netc_switch *priv)
{
	struct netc_switch_regs *regs = &priv->regs;
	u32 val;

	val = netc_base_rd(regs, NETC_VFHTDECR2);

	/* If no match is found in the VLAN Filter table, then VFHTDECR2[MLO]
	 * will take effect. VFHTDECR2[MLO] is set to "Software MAC learning
	 * secure" by default. Notice BPCR[MLO] will override VFHTDECR2[MLO]
	 * if its value is not zero.
	 */
	val = u32_replace_bits(val, MLO_SW_SEC, VFHTDECR2_MLO);
	val = u32_replace_bits(val, MFO_NO_MATCH_DISCARD, VFHTDECR2_MFO);
	netc_base_wr(regs, NETC_VFHTDECR2, val);
}

static void netc_port_set_max_frame_size(struct netc_port *np,
					 u32 max_frame_size)
{
	netc_mac_port_wr(np, NETC_PM_MAXFRM(0),
			 max_frame_size & PM_MAXFRAM);
}

static void netc_switch_fixed_config(struct netc_switch *priv)
{
	netc_switch_dos_default_config(priv);
	netc_switch_vfht_default_config(priv);
}

static void netc_port_set_tc_max_sdu(struct netc_port *np,
				     int tc, u32 max_sdu)
{
	u32 val = FIELD_PREP(PTCTMSDUR_MAXSDU, max_sdu) |
		  FIELD_PREP(PTCTMSDUR_SDU_TYPE, SDU_TYPE_MPDU);

	netc_port_wr(np, NETC_PTCTMSDUR(tc), val);
}

static void netc_port_set_all_tc_msdu(struct netc_port *np)
{
	for (int tc = 0; tc < NETC_TC_NUM; tc++)
		netc_port_set_tc_max_sdu(np, tc, NETC_MAX_FRAME_LEN);
}

static void netc_port_set_mlo(struct netc_port *np, enum netc_mlo mlo)
{
	netc_port_rmw(np, NETC_BPCR, BPCR_MLO, FIELD_PREP(BPCR_MLO, mlo));
}

static void netc_port_set_pvid(struct netc_port *np, u16 pvid)
{
	netc_port_rmw(np, NETC_BPDVR, BPDVR_VID, pvid);
}

static void netc_port_set_vlan_aware(struct netc_port *np, bool aware)
{
	netc_port_rmw(np, NETC_BPDVR, BPDVR_RXVAM,
		      aware ? 0 : BPDVR_RXVAM);
}

static void netc_port_fixed_config(struct netc_port *np)
{
	/* Default IPV and DR setting */
	netc_port_rmw(np, NETC_PQOSMR, PQOSMR_VS | PQOSMR_VE,
		      PQOSMR_VS | PQOSMR_VE);

	/* Enable L2 and L3 DOS */
	netc_port_rmw(np, NETC_PCR, PCR_L2DOSE | PCR_L3DOSE,
		      PCR_L2DOSE | PCR_L3DOSE);

	/* Set the quanta value of TX PAUSE frame */
	netc_mac_port_wr(np, NETC_PM_PAUSE_QUANTA(0), NETC_PAUSE_QUANTA);

	/* When a quanta timer counts down and reaches this value,
	 * the MAC sends a refresh PAUSE frame with the programmed
	 * full quanta value if a pause condition still exists.
	 */
	netc_mac_port_wr(np, NETC_PM_PAUSE_THRESH(0), NETC_PAUSE_THRESH);
}

static void netc_port_default_config(struct netc_port *np)
{
	netc_port_fixed_config(np);

	/* Default VLAN unaware */
	netc_port_set_vlan_aware(np, false);

	if (dsa_port_is_cpu(np->dp))
		/* For CPU port, source port pruning is disabled */
		netc_port_rmw(np, NETC_BPCR, BPCR_SRCPRND, BPCR_SRCPRND);
	else
		netc_port_set_mlo(np, MLO_DISABLE);

	netc_port_set_max_frame_size(np, NETC_MAX_FRAME_LEN);
	netc_port_set_all_tc_msdu(np);
}

static u32 netc_available_port_bitmap(struct netc_switch *priv)
{
	struct dsa_port *dp;
	u32 bitmap = 0;

	dsa_switch_for_each_available_port(dp, priv->ds)
		bitmap |= BIT(dp->index);

	return bitmap;
}

static int netc_add_standalone_vlan_entry(struct netc_switch *priv)
{
	u32 bitmap_stg = VFT_STG_ID(0) | netc_available_port_bitmap(priv);
	struct vft_cfge_data *cfge;
	u16 cfg;
	int err;

	cfge = kzalloc_obj(*cfge);
	if (!cfge)
		return -ENOMEM;

	cfge->bitmap_stg = cpu_to_le32(bitmap_stg);
	cfge->et_eid = cpu_to_le32(NTMP_NULL_ENTRY_ID);
	cfge->fid = cpu_to_le16(NETC_STANDALONE_PVID);

	/* For standalone ports, MAC learning needs to be disabled, so frames
	 * from other user ports will not be forwarded to the standalone ports,
	 * because there are no FDB entries on the standalone ports. Also, the
	 * frames received by the standalone ports cannot be flooded to other
	 * ports, so MAC forwarding option needs to be set to
	 * MFO_NO_MATCH_DISCARD, so the frames will be discarded rather than
	 * flooding to other ports.
	 */
	cfg = FIELD_PREP(VFT_MLO, MLO_DISABLE) |
	      FIELD_PREP(VFT_MFO, MFO_NO_MATCH_DISCARD);
	cfge->cfg = cpu_to_le16(cfg);

	err = ntmp_vft_add_entry(&priv->ntmp, NETC_STANDALONE_PVID, cfge);
	if (err)
		dev_err(priv->dev,
			"Failed to add standalone VLAN entry\n");

	kfree(cfge);

	return err;
}

static int netc_port_add_fdb_entry(struct netc_port *np,
				   const unsigned char *addr, u16 vid)
{
	struct netc_switch *priv = np->switch_priv;
	struct netc_fdb_entry *entry;
	struct fdbt_keye_data *keye;
	struct fdbt_cfge_data *cfge;
	int port = np->dp->index;
	u32 cfg = 0;
	int err;

	entry = kzalloc_obj(*entry);
	if (!entry)
		return -ENOMEM;

	keye = &entry->keye;
	cfge = &entry->cfge;
	ether_addr_copy(keye->mac_addr, addr);
	keye->fid = cpu_to_le16(vid);

	cfge->port_bitmap = cpu_to_le32(BIT(port));
	cfge->cfg = cpu_to_le32(cfg);
	cfge->et_eid = cpu_to_le32(NTMP_NULL_ENTRY_ID);

	err = ntmp_fdbt_add_entry(&priv->ntmp, &entry->entry_id, keye, cfge);
	if (err) {
		kfree(entry);

		return err;
	}

	netc_add_fdb_entry(priv, entry);

	return 0;
}

static int netc_port_set_fdb_entry(struct netc_port *np,
				   const unsigned char *addr, u16 vid)
{
	struct netc_switch *priv = np->switch_priv;
	struct netc_fdb_entry *entry;
	struct fdbt_cfge_data *cfge;
	int port = np->dp->index;
	__le32 old_port_bitmap;
	int err = 0;

	mutex_lock(&priv->fdbt_lock);

	entry = netc_lookup_fdb_entry(priv, addr, vid);
	if (!entry) {
		err = netc_port_add_fdb_entry(np, addr, vid);
		if (err)
			dev_err(priv->dev,
				"Failed to add FDB entry on port %d\n",
				port);

		goto unlock_fdbt;
	}

	cfge = &entry->cfge;
	/* If the entry already exists on the port, return 0 directly */
	if (unlikely(cfge->port_bitmap & cpu_to_le32(BIT(port))))
		goto unlock_fdbt;

	/* If the entry already exists, but not on this port, we need to
	 * update the port bitmap. In general, it should only be valid
	 * for multicast or broadcast address.
	 */
	old_port_bitmap = cfge->port_bitmap;
	if (is_multicast_ether_addr(addr))
		cfge->port_bitmap |= cpu_to_le32(BIT(port));
	else
		cfge->port_bitmap = cpu_to_le32(BIT(port));

	err = ntmp_fdbt_update_entry(&priv->ntmp, entry->entry_id, cfge);
	if (err) {
		cfge->port_bitmap = old_port_bitmap;
		dev_err(priv->dev, "Failed to set FDB entry on port %d\n",
			port);
	}

unlock_fdbt:
	mutex_unlock(&priv->fdbt_lock);

	return err;
}

static int netc_port_del_fdb_entry(struct netc_port *np,
				   const unsigned char *addr, u16 vid)
{
	struct netc_switch *priv = np->switch_priv;
	struct ntmp_user *ntmp = &priv->ntmp;
	struct netc_fdb_entry *entry;
	struct fdbt_cfge_data *cfge;
	int port = np->dp->index;
	int err = 0;

	mutex_lock(&priv->fdbt_lock);

	entry = netc_lookup_fdb_entry(priv, addr, vid);
	if (unlikely(!entry))
		/* The hardware-learned dynamic FDB entries cannot be deleted
		 * through .port_fdb_del() interface.
		 * For NTF_MASTER path: Since hardware-learned dynamic FDB
		 * entries are never synchronized back to the bridge software
		 * database. br_fdb_delete() -> br_fdb_find() cannot find the
		 * FDB entry, so .port_fdb_del() will not be called.
		 * For NTF_SELF path: dsa_user_netdev_ops does not implement
		 * ndo_fdb_del(), so rtnl_fdb_del() falls back to
		 * ndo_dflt_fdb_del(), which only supports NUD_PERMANENT static
		 * entries and rejects all others with -EINVAL.
		 */
		goto unlock_fdbt;

	cfge = &entry->cfge;
	if (unlikely(!(cfge->port_bitmap & cpu_to_le32(BIT(port)))))
		goto unlock_fdbt;

	if (cfge->port_bitmap != cpu_to_le32(BIT(port))) {
		/* If the entry also exists on other ports, we need to
		 * update the entry in the FDB table.
		 */
		cfge->port_bitmap &= cpu_to_le32(~BIT(port));
		err = ntmp_fdbt_update_entry(ntmp, entry->entry_id, cfge);
		if (err) {
			cfge->port_bitmap |= cpu_to_le32(BIT(port));
			goto unlock_fdbt;
		}
	} else {
		/* If the entry only exists on this port, just delete
		 * it from the FDB table.
		 */
		err = ntmp_fdbt_delete_entry(ntmp, entry->entry_id);
		if (err)
			goto unlock_fdbt;

		netc_del_fdb_entry(entry);
	}

unlock_fdbt:
	mutex_unlock(&priv->fdbt_lock);

	return err;
}

static int netc_add_standalone_fdb_bcast_entry(struct netc_switch *priv)
{
	const u8 bcast[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
	struct dsa_port *dp, *cpu_dp = NULL;

	dsa_switch_for_each_cpu_port(dp, priv->ds) {
		/* The switch has only one CPU port, so only need to find
		 * the first CPU port to break out of the loop.
		 */
		cpu_dp = dp;
		break;
	}

	if (!cpu_dp)
		return -ENODEV;

	/* If the user port acts as a standalone port, then its PVID is 0,
	 * MLO is set to "disable MAC learning" and MFO is set to "discard
	 * frames if no matching entry found in FDB table". Therefore, we
	 * need to add a broadcast FDB entry on the CPU port so that the
	 * broadcast frames received on the user port can be forwarded to
	 * the CPU port.
	 */
	return netc_port_set_fdb_entry(NETC_PORT(priv->ds, cpu_dp->index),
				       bcast, NETC_STANDALONE_PVID);
}

static void netc_port_set_pbpmcr(struct netc_port *np, u64 mapping)
{
	u32 pbpmcr0 = lower_32_bits(mapping);
	u32 pbpmcr1 = upper_32_bits(mapping);

	netc_port_wr(np, NETC_PBPMCR0, pbpmcr0);
	netc_port_wr(np, NETC_PBPMCR1, pbpmcr1);
}

static void netc_ipv_to_buffer_pool_mapping(struct netc_switch *priv)
{
	int bp_per_port = priv->num_bp / priv->info->num_ports;
	int q = NETC_IPV_NUM / bp_per_port;
	int r = NETC_IPV_NUM % bp_per_port;
	int num = q + r;

	/* IPV-to-buffer-pool mapping per port:
	 * Each port is allocated 'bp_per_port' buffer pools and supports 8
	 * IPVs, where a higher IPV indicates a higher frame priority. Each
	 * IPV can be mapped to only one buffer pool, from hardware design
	 * perspective, bp_per_port will not be greater than 8. So 'q' will
	 * not be 0.
	 *
	 * The mapping rule is as follows:
	 * - The first 'num' IPVs share the port's first buffer pool (index
	 * 'base_id').
	 * - After that, every 'q' IPVs share one buffer pool, with pool
	 * indices increasing sequentially.
	 */
	for (int i = 0; i < priv->info->num_ports; i++) {
		u32 base_id = i * bp_per_port;
		u32 bp_id = base_id;
		u64 mapping = 0;

		for (int ipv = 0; ipv < NETC_IPV_NUM; ipv++) {
			/* Update the buffer pool index */
			if (ipv >= num)
				bp_id = base_id + ((ipv - num) / q) + 1;

			mapping |= (u64)bp_id << (ipv * 8);
		}

		netc_port_set_pbpmcr(priv->ports[i], mapping);
	}
}

static int netc_switch_bpt_default_config(struct netc_switch *priv)
{
	if (priv->num_bp < priv->info->num_ports)
		return -EINVAL;

	priv->bpt_list = devm_kcalloc(priv->dev, priv->num_bp,
				      sizeof(struct bpt_cfge_data),
				      GFP_KERNEL);
	if (!priv->bpt_list)
		return -ENOMEM;

	/* Initialize the maximum threshold of each buffer pool entry */
	for (int i = 0; i < priv->num_bp; i++) {
		struct bpt_cfge_data *cfge = &priv->bpt_list[i];
		int err;

		cfge->max_thresh = cpu_to_le16(NETC_BP_THRESH);
		err = ntmp_bpt_update_entry(&priv->ntmp, i, cfge);
		if (err)
			return err;
	}

	netc_ipv_to_buffer_pool_mapping(priv);

	return 0;
}

static int netc_setup(struct dsa_switch *ds)
{
	struct netc_switch *priv = ds->priv;
	struct dsa_port *dp;
	int err;

	err = netc_init_switch_id(priv);
	if (err)
		return err;

	netc_get_switch_capabilities(priv);

	err = netc_init_all_ports(priv);
	if (err)
		return err;

	err = netc_init_ntmp_user(priv);
	if (err)
		return err;

	INIT_HLIST_HEAD(&priv->fdb_list);
	mutex_init(&priv->fdbt_lock);
	priv->fdbt_ageing_delay = NETC_FDBT_AGEING_DELAY;
	atomic_set(&priv->br_cnt, 0);
	INIT_DELAYED_WORK(&priv->fdbt_ageing_work,
			  netc_clean_fdbt_ageing_entries);
	INIT_HLIST_HEAD(&priv->vlan_list);
	mutex_init(&priv->vft_lock);

	netc_switch_fixed_config(priv);

	/* default setting for ports */
	dsa_switch_for_each_available_port(dp, ds)
		netc_port_default_config(priv->ports[dp->index]);

	err = netc_switch_bpt_default_config(priv);
	if (err)
		goto free_lock_and_ntmp_user;

	err = netc_add_standalone_vlan_entry(priv);
	if (err)
		goto free_lock_and_ntmp_user;

	err = netc_add_standalone_fdb_bcast_entry(priv);
	if (err)
		goto free_lock_and_ntmp_user;

	return 0;

free_lock_and_ntmp_user:
	/* No need to clear the hardware state, netc_setup() is only called
	 * when the driver is bound, and FLR will be performed to reset the
	 * hardware state.
	 */
	mutex_destroy(&priv->fdbt_lock);
	mutex_destroy(&priv->vft_lock);
	netc_free_ntmp_user(priv);

	return err;
}

static void netc_destroy_all_lists(struct netc_switch *priv)
{
	netc_destroy_fdb_list(priv);
	mutex_destroy(&priv->fdbt_lock);
	netc_destroy_vlan_list(priv);
	mutex_destroy(&priv->vft_lock);
}

static void netc_free_host_flood_rules(struct netc_switch *priv)
{
	struct dsa_port *dp;

	dsa_switch_for_each_user_port(dp, priv->ds) {
		struct netc_port *np = priv->ports[dp->index];

		/* No need to clear the hardware IPFT entry. Because PCIe
		 * FLR will be performed when the switch is re-registered,
		 * it will reset hardware state. So only need to free the
		 * memory to avoid memory leak.
		 */
		kfree(np->host_flood);
		np->host_flood = NULL;
	}
}

static void netc_teardown(struct dsa_switch *ds)
{
	struct netc_switch *priv = ds->priv;

	disable_delayed_work_sync(&priv->fdbt_ageing_work);
	netc_destroy_all_lists(priv);
	netc_free_host_flood_rules(priv);
	netc_free_ntmp_user(priv);
}

static bool netc_port_is_emdio_consumer(struct device_node *node)
{
	struct device_node *mdio_node;

	/* If the port node has phy-handle property and it does
	 * not contain a mdio child node, then the port is the
	 * EMDIO consumer.
	 */
	mdio_node = of_get_child_by_name(node, "mdio");
	if (!mdio_node)
		return true;

	of_node_put(mdio_node);

	return false;
}

/* Currently, phylink_of_phy_connect() is called by dsa_user_create(),
 * so if the switch uses the external MDIO controller (like the EMDIO
 * function) to manage the external PHYs. The MDIO bus may not be
 * created when phylink_of_phy_connect() is called, so it will return
 * an error and cause the switch driver to fail to probe.
 * This workaround can be removed when DSA phylink_of_phy_connect()
 * calls are moved from probe() to ndo_open().
 */
static int netc_switch_check_emdio_is_ready(struct device *dev)
{
	struct device_node *ports, *phy_node;
	struct phy_device *phydev;
	int err = 0;

	ports = of_get_child_by_name(dev->of_node, "ethernet-ports");
	if (!ports) {
		dev_err(dev, "Cannot find the ethernet-ports node\n");
		return -EINVAL;
	}

	for_each_available_child_of_node_scoped(ports, child) {
		/* If the node does not have phy-handle property, then the
		 * port does not connect to a PHY, so the port is not the
		 * EMDIO consumer.
		 */
		phy_node = of_parse_phandle(child, "phy-handle", 0);
		if (!phy_node)
			continue;

		/* Note that from the hardware perspective, the switch ports
		 * do not support sharing the MDIO bus defined under one port.
		 * Each port can only access its own external PHY through its
		 * port MDIO bus.
		 */
		if (!netc_port_is_emdio_consumer(child)) {
			of_node_put(phy_node);
			continue;
		}

		phydev = of_phy_find_device(phy_node);
		of_node_put(phy_node);
		if (!phydev) {
			err = -EPROBE_DEFER;
			goto out;
		}

		put_device(&phydev->mdio.dev);
	}

out:
	of_node_put(ports);

	return err;
}

static int netc_switch_pci_init(struct pci_dev *pdev)
{
	struct device *dev = &pdev->dev;
	struct netc_switch_regs *regs;
	struct netc_switch *priv;
	void __iomem *base;
	int err;

	pcie_flr(pdev);
	err = pcim_enable_device(pdev);
	if (err)
		return dev_err_probe(dev, err, "Failed to enable device\n");

	err = pcim_request_all_regions(pdev, KBUILD_MODNAME);
	if (err)
		return dev_err_probe(dev, err, "Failed to request regions\n");

	/* The command BD rings and NTMP tables need DMA. No need to check
	 * the return value, because it never returns fail when the mask is
	 * DMA_BIT_MASK(64), see dma-api-howto.rst.
	 */
	dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));

	if (pci_resource_len(pdev, NETC_REGS_BAR) < NETC_REGS_SIZE) {
		return dev_err_probe(dev, -EINVAL,
				     "Invalid register space size\n");
	}

	base = pcim_iomap(pdev, NETC_REGS_BAR, 0);
	if (!base)
		return dev_err_probe(dev, -ENXIO, "pcim_iomap() failed\n");

	pci_set_master(pdev);

	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
	if (!priv)
		return -ENOMEM;

	priv->pdev = pdev;
	priv->dev = dev;

	regs = &priv->regs;
	regs->base = base;
	regs->port = regs->base + NETC_REGS_PORT_BASE;
	regs->global = regs->base + NETC_REGS_GLOBAL_BASE;
	pci_set_drvdata(pdev, priv);

	return 0;
}

static void netc_switch_get_ip_revision(struct netc_switch *priv)
{
	struct netc_switch_regs *regs = &priv->regs;
	u32 val = netc_glb_rd(regs, NETC_IPBRR0);

	priv->revision = FIELD_GET(IPBRR0_IP_REV, val);
}

static void netc_init_ett_cfge(struct ett_cfge_data *cfge,
			       bool untagged, u32 ect_eid)
{
	u32 vuda_sqta = FMTEID_VUDA_SQTA;
	u16 efm_cfg = 0;

	if (ect_eid != NTMP_NULL_ENTRY_ID) {
		/* Increase egress frame counter */
		efm_cfg |= FIELD_PREP(ETT_ECA, ETT_ECA_INC);
		cfge->ec_eid = cpu_to_le32(ect_eid);
	}

	/* If egress rule is VLAN untagged */
	if (untagged) {
		/* delete outer VLAN tag */
		vuda_sqta |= FIELD_PREP(FMTEID_VUDA, FMTEID_VUDA_DEL_OTAG);
		/* length change: twos-complement notation */
		efm_cfg |= FIELD_PREP(ETT_EFM_LEN_CHANGE,
				      ETT_FRM_LEN_DEL_VLAN);
	}

	cfge->efm_eid = cpu_to_le32(vuda_sqta);
	cfge->efm_cfg = cpu_to_le16(efm_cfg);
}

static int netc_add_ett_entry(struct netc_switch *priv, bool untagged,
			      u32 ett_eid, u32 ect_eid)
{
	struct ntmp_user *ntmp = &priv->ntmp;
	struct ett_cfge_data cfge = {};

	netc_init_ett_cfge(&cfge, untagged, ect_eid);

	return ntmp_ett_add_entry(ntmp, ett_eid, &cfge);
}

static int netc_update_ett_entry(struct netc_switch *priv, bool untagged,
				 u32 ett_eid, u32 ect_eid)
{
	struct ntmp_user *ntmp = &priv->ntmp;
	struct ett_cfge_data cfge = {};

	netc_init_ett_cfge(&cfge, untagged, ect_eid);

	return ntmp_ett_update_entry(ntmp, ett_eid, &cfge);
}

static int netc_add_ett_group_entries(struct netc_switch *priv,
				      u32 untagged_port_bitmap,
				      u32 ett_base_eid,
				      u32 ect_base_eid)
{
	struct netc_port **ports = priv->ports;
	u32 ett_eid, ect_eid;
	bool untagged;
	int i, err;

	for (i = 0; i < priv->info->num_ports; i++) {
		if (!ports[i]->dp)
			continue;

		untagged = !!(untagged_port_bitmap & BIT(i));
		ett_eid = ett_base_eid + ports[i]->ett_offset;
		ect_eid = NTMP_NULL_ENTRY_ID;
		if (ect_base_eid != NTMP_NULL_ENTRY_ID)
			ect_eid = ect_base_eid + ports[i]->ett_offset;

		err = netc_add_ett_entry(priv, untagged, ett_eid, ect_eid);
		if (err)
			goto clear_ett_entries;
	}

	return 0;

clear_ett_entries:
	while (--i >= 0) {
		if (!ports[i]->dp)
			continue;

		ett_eid = ett_base_eid + ports[i]->ett_offset;
		ntmp_ett_delete_entry(&priv->ntmp, ett_eid);
	}

	return err;
}

static int netc_add_vlan_egress_rule(struct netc_switch *priv,
				     struct netc_vlan_entry *entry)
{
	u32 num_ports = netc_num_available_ports(priv);
	struct ntmp_user *ntmp = &priv->ntmp;
	u32 ect_eid = NTMP_NULL_ENTRY_ID;
	u32 ett_eid, ett_gid, ect_gid;
	int err;

	/* Step 1: Find available egress counter table entries and update
	 * these entries.
	 */
	ect_gid = ntmp_lookup_free_eid(ntmp->ect_gid_bitmap,
				       ntmp->ect_bitmap_size);
	if (ect_gid == NTMP_NULL_ENTRY_ID) {
		dev_info(priv->dev,
			 "No egress counter table entries available\n");
	} else {
		ect_eid = ect_gid * num_ports;
		for (int i = 0; i < num_ports; i++)
			/* There is no need to check the return value, the only
			 * issue is that the entry's counter might be inaccurate,
			 * but it will not affect the functionality, it is only
			 * for future debugging.
			 */
			ntmp_ect_update_entry(ntmp, ect_eid + i);
	}

	/* Step 2: Find available egress treatment table entries and add
	 * these entries.
	 */
	ett_gid = ntmp_lookup_free_eid(ntmp->ett_gid_bitmap,
				       ntmp->ett_bitmap_size);
	if (ett_gid == NTMP_NULL_ENTRY_ID) {
		dev_err(priv->dev,
			"No egress treatment table entries available\n");
		err = -ENOSPC;
		goto clear_ect_gid;
	}

	ett_eid = ett_gid * num_ports;
	err = netc_add_ett_group_entries(priv, entry->untagged_port_bitmap,
					 ett_eid, ect_eid);
	if (err)
		goto clear_ett_gid;

	entry->cfge.et_eid = cpu_to_le32(ett_eid);
	entry->ect_gid = ect_gid;

	return 0;

clear_ett_gid:
	ntmp_clear_eid_bitmap(ntmp->ett_gid_bitmap, ett_gid);

clear_ect_gid:
	if (ect_gid != NTMP_NULL_ENTRY_ID)
		ntmp_clear_eid_bitmap(ntmp->ect_gid_bitmap, ect_gid);

	return err;
}

static void netc_delete_vlan_egress_rule(struct netc_switch *priv,
					 struct netc_vlan_entry *entry)
{
	u32 num_ports = netc_num_available_ports(priv);
	struct ntmp_user *ntmp = &priv->ntmp;
	u32 ett_eid, ett_gid;

	ett_eid = le32_to_cpu(entry->cfge.et_eid);
	if (ett_eid == NTMP_NULL_ENTRY_ID)
		return;

	ett_gid = ett_eid / num_ports;
	ntmp_clear_eid_bitmap(ntmp->ett_gid_bitmap, ett_gid);
	for (int i = 0; i < num_ports; i++)
		ntmp_ett_delete_entry(ntmp, ett_eid + i);

	if (entry->ect_gid == NTMP_NULL_ENTRY_ID)
		return;

	ntmp_clear_eid_bitmap(ntmp->ect_gid_bitmap, entry->ect_gid);
}

static int netc_port_update_vlan_egress_rule(struct netc_port *np,
					     struct netc_vlan_entry *entry)
{
	bool untagged = !!(entry->untagged_port_bitmap & BIT(np->dp->index));
	u32 num_ports = netc_num_available_ports(np->switch_priv);
	u32 ett_eid = le32_to_cpu(entry->cfge.et_eid);
	struct netc_switch *priv = np->switch_priv;
	u32 ect_eid = NTMP_NULL_ENTRY_ID;
	int err;

	if (ett_eid == NTMP_NULL_ENTRY_ID)
		return 0;

	if (entry->ect_gid != NTMP_NULL_ENTRY_ID)
		/* Each ETT entry maps to an ECT entry if ect_gid is not NULL
		 * entry ID. The offset of the ECT entry corresponding to the
		 * port in the group is equal to ett_offset.
		 */
		ect_eid = entry->ect_gid * num_ports + np->ett_offset;

	ett_eid += np->ett_offset;
	err = netc_update_ett_entry(priv, untagged, ett_eid, ect_eid);
	if (err) {
		dev_err(priv->dev,
			"Failed to update VLAN %u egress rule on port %d\n",
			entry->vid, np->dp->index);
		return err;
	}

	if (ect_eid != NTMP_NULL_ENTRY_ID)
		ntmp_ect_update_entry(&priv->ntmp, ect_eid);

	return 0;
}

static int netc_port_add_vlan_entry(struct netc_port *np, u16 vid,
				    bool untagged)
{
	struct netc_switch *priv = np->switch_priv;
	struct netc_vlan_entry *entry;
	struct vft_cfge_data *cfge;
	u32 index = np->dp->index;
	u32 bitmap_stg;
	int err;
	u16 cfg;

	entry = kzalloc_obj(*entry);
	if (!entry)
		return -ENOMEM;

	entry->vid = vid;
	entry->ect_gid = NTMP_NULL_ENTRY_ID;

	bitmap_stg = BIT(index) | VFT_STG_ID(0);
	/* If the VID is a VLAN-unaware PVID, the CPU port needs to be
	 * a member of this VLAN.
	 */
	if (dsa_port_is_user(np->dp) &&
	    vid >= NETC_VLAN_UNAWARE_PVID(priv->ds->max_num_bridges)) {
		struct dsa_port *cpu_dp = np->dp->cpu_dp;

		bitmap_stg |= BIT(cpu_dp->index);
	}

	cfg = FIELD_PREP(VFT_MLO, MLO_HW) |
	      FIELD_PREP(VFT_MFO, MFO_NO_MATCH_FLOOD);

	cfge = &entry->cfge;
	cfge->et_eid = cpu_to_le32(NTMP_NULL_ENTRY_ID);
	cfge->bitmap_stg = cpu_to_le32(bitmap_stg);
	cfge->fid = cpu_to_le16(vid);
	cfge->cfg = cpu_to_le16(cfg);
	cfge->eta_port_bitmap = cpu_to_le32(priv->port_bitmap);

	if (untagged)
		entry->untagged_port_bitmap = BIT(index);

	err = netc_add_vlan_egress_rule(priv, entry);
	if (err)
		goto free_vlan_entry;

	err = ntmp_vft_add_entry(&priv->ntmp, vid, cfge);
	if (err) {
		dev_err(priv->dev,
			"Failed to add VLAN %u entry on port %d\n",
			vid, index);
		goto delete_vlan_egress_rule;
	}

	netc_add_vlan_entry(priv, entry);

	return 0;

delete_vlan_egress_rule:
	netc_delete_vlan_egress_rule(priv, entry);
free_vlan_entry:
	kfree(entry);

	return err;
}

static bool netc_port_vlan_egress_rule_changed(struct netc_switch *priv,
					       struct netc_vlan_entry *entry,
					       int port, bool untagged)
{
	bool old_untagged = !!(entry->untagged_port_bitmap & BIT(port));

	/* VLAN-unaware VIDs have no egress rules, so return 'false' */
	if (entry->vid >= NETC_VLAN_UNAWARE_PVID(priv->ds->max_num_bridges))
		return false;

	return old_untagged != untagged;
}

static int netc_port_set_vlan_entry(struct netc_port *np, u16 vid,
				    bool untagged)
{
	struct netc_switch *priv = np->switch_priv;
	struct netc_vlan_entry *entry;
	struct vft_cfge_data *cfge;
	int port = np->dp->index;
	bool changed;
	int err = 0;

	mutex_lock(&priv->vft_lock);

	entry = netc_lookup_vlan_entry(priv, vid);
	if (!entry) {
		err = netc_port_add_vlan_entry(np, vid, untagged);
		goto unlock_vft;
	}

	/* Check whether the egress VLAN rule is changed */
	changed = netc_port_vlan_egress_rule_changed(priv, entry, port,
						     untagged);
	if (changed) {
		entry->untagged_port_bitmap ^= BIT(port);
		err = netc_port_update_vlan_egress_rule(np, entry);
		if (err) {
			entry->untagged_port_bitmap ^= BIT(port);
			goto unlock_vft;
		}
	}

	cfge = &entry->cfge;
	if (cfge->bitmap_stg & cpu_to_le32(BIT(port)))
		goto unlock_vft;

	cfge->bitmap_stg |= cpu_to_le32(BIT(port));
	err = ntmp_vft_update_entry(&priv->ntmp, vid, cfge);
	if (err) {
		dev_err(priv->dev,
			"Failed to update VLAN %u entry on port %d\n",
			vid, port);

		goto restore_bitmap_stg;
	}

	mutex_unlock(&priv->vft_lock);

	return 0;

restore_bitmap_stg:
	cfge->bitmap_stg &= cpu_to_le32(~BIT(port));
	if (changed) {
		entry->untagged_port_bitmap ^= BIT(port);
		/* Recover the corresponding ETT entry. It doesn't matter
		 * if it fails because the bit corresponding to the port
		 * in the port bitmap of the VFT entry is not set. so the
		 * frame will not match that ETT entry.
		 */
		if (netc_port_update_vlan_egress_rule(np, entry))
			entry->untagged_port_bitmap ^= BIT(port);
	}
unlock_vft:
	mutex_unlock(&priv->vft_lock);

	return err;
}

static int netc_port_del_vlan_entry(struct netc_port *np, u16 vid)
{
	struct netc_switch *priv = np->switch_priv;
	struct netc_vlan_entry *entry;
	struct vft_cfge_data *cfge;
	int port = np->dp->index;
	u32 vlan_port_bitmap;
	int err = 0;

	mutex_lock(&priv->vft_lock);

	entry = netc_lookup_vlan_entry(priv, vid);
	if (!entry)
		goto unlock_vft;

	cfge = &entry->cfge;
	vlan_port_bitmap = FIELD_GET(VFT_PORT_MEMBERSHIP,
				     le32_to_cpu(cfge->bitmap_stg));
	/* If the VID is a VLAN-unaware PVID, we need to clear the CPU
	 * port bit of vlan_port_bitmap, so that the VLAN entry can be
	 * deleted if no user ports use this VLAN.
	 */
	if (dsa_port_is_user(np->dp) &&
	    vid >= NETC_VLAN_UNAWARE_PVID(priv->ds->max_num_bridges)) {
		struct dsa_port *cpu_dp = np->dp->cpu_dp;

		vlan_port_bitmap &= ~BIT(cpu_dp->index);
	}

	/* If the VLAN only belongs to the current port */
	if (vlan_port_bitmap == BIT(port)) {
		err = ntmp_vft_delete_entry(&priv->ntmp, vid);
		if (err)
			goto unlock_vft;

		netc_delete_vlan_egress_rule(priv, entry);
		netc_del_vlan_entry(entry);

		goto unlock_vft;
	}

	if (!(vlan_port_bitmap & BIT(port)))
		goto unlock_vft;

	cfge->bitmap_stg &= cpu_to_le32(~BIT(port));
	err = ntmp_vft_update_entry(&priv->ntmp, vid, cfge);
	if (err) {
		cfge->bitmap_stg |= cpu_to_le32(BIT(port));
		goto unlock_vft;
	}

unlock_vft:
	mutex_unlock(&priv->vft_lock);

	return err;
}

static int netc_port_enable(struct dsa_switch *ds, int port,
			    struct phy_device *phy)
{
	struct netc_port *np = NETC_PORT(ds, port);
	int err;

	if (np->enable)
		return 0;

	err = clk_prepare_enable(np->ref_clk);
	if (err) {
		dev_err(ds->dev,
			"Failed to enable enet_ref_clk of port %d\n", port);
		return err;
	}

	np->enable = true;

	return 0;
}

static void netc_port_disable(struct dsa_switch *ds, int port)
{
	struct netc_port *np = NETC_PORT(ds, port);

	/* When .port_disable() is called, .port_enable() may not have been
	 * called. In this case, both the prepare_count and enable_count of
	 * clock are 0. Calling clk_disable_unprepare() at this time will
	 * cause warnings.
	 */
	if (!np->enable)
		return;

	clk_disable_unprepare(np->ref_clk);
	np->enable = false;
}

static void netc_port_stp_state_set(struct dsa_switch *ds,
				    int port, u8 state)
{
	struct netc_port *np = NETC_PORT(ds, port);
	u32 val;

	switch (state) {
	case BR_STATE_DISABLED:
	case BR_STATE_LISTENING:
	case BR_STATE_BLOCKING:
		val = NETC_STG_STATE_DISABLED;
		break;
	case BR_STATE_LEARNING:
		val = NETC_STG_STATE_LEARNING;
		break;
	case BR_STATE_FORWARDING:
		val = NETC_STG_STATE_FORWARDING;
		break;
	default:
		return;
	}

	netc_port_wr(np, NETC_BPSTGSR, val);
}

static int netc_port_change_mtu(struct dsa_switch *ds,
				int port, int mtu)
{
	u32 max_frame_size = mtu + VLAN_ETH_HLEN + ETH_FCS_LEN;

	netc_port_set_max_frame_size(NETC_PORT(ds, port), max_frame_size);

	return 0;
}

static int netc_port_max_mtu(struct dsa_switch *ds, int port)
{
	return NETC_MAX_FRAME_LEN - VLAN_ETH_HLEN - ETH_FCS_LEN;
}

static struct net_device *netc_classify_db(struct dsa_db db)
{
	switch (db.type) {
	case DSA_DB_PORT:
		return NULL;
	case DSA_DB_BRIDGE:
		return db.bridge.dev;
	default:
		return ERR_PTR(-EOPNOTSUPP);
	}
}

static u16 netc_vlan_unaware_pvid(struct dsa_bridge *bridge)
{
	u32 br_num;

	if (!bridge)
		return NETC_STANDALONE_PVID;

	br_num = bridge->num;

	/* The br_num is supposed to be 1 ~ ds->max_num_bridges, see
	 * dsa_bridge_num_get(). Since max_num_bridges is non-zero,
	 * so dsa_port_bridge_create() will return an error if
	 * dsa_bridge_num_get() returns 0.
	 */
	if (WARN_ON(!br_num))
		return NETC_STANDALONE_PVID;

	return NETC_VLAN_UNAWARE_PVID(br_num);
}

static int netc_port_fdb_add(struct dsa_switch *ds, int port,
			     const unsigned char *addr, u16 vid,
			     struct dsa_db db)
{
	struct net_device *br_ndev = netc_classify_db(db);
	struct netc_port *np = NETC_PORT(ds, port);

	if (IS_ERR(br_ndev))
		return PTR_ERR(br_ndev);

	if (!vid)
		vid = netc_vlan_unaware_pvid(br_ndev ? &db.bridge : NULL);

	return netc_port_set_fdb_entry(np, addr, vid);
}

static int netc_port_fdb_del(struct dsa_switch *ds, int port,
			     const unsigned char *addr, u16 vid,
			     struct dsa_db db)
{
	struct net_device *br_ndev = netc_classify_db(db);
	struct netc_port *np = NETC_PORT(ds, port);

	if (IS_ERR(br_ndev))
		return PTR_ERR(br_ndev);

	if (!vid)
		vid = netc_vlan_unaware_pvid(br_ndev ? &db.bridge : NULL);

	return netc_port_del_fdb_entry(np, addr, vid);
}

static int netc_port_fdb_dump(struct dsa_switch *ds, int port,
			      dsa_fdb_dump_cb_t *cb, void *data)
{
	struct netc_switch *priv = ds->priv;
	u32 resume_eid = NTMP_NULL_ENTRY_ID;
	struct fdbt_entry_data *entry;
	struct fdbt_keye_data *keye;
	struct fdbt_cfge_data *cfge;
	u32 cfg, cnt = 0;
	bool is_static;
	int err;
	u16 vid;

	entry = kmalloc_obj(*entry);
	if (!entry)
		return -ENOMEM;

	keye = &entry->keye;
	cfge = &entry->cfge;
	mutex_lock(&priv->fdbt_lock);

	do {
		memset(entry, 0, sizeof(*entry));
		err = ntmp_fdbt_search_port_entry(&priv->ntmp, port,
						  &resume_eid, entry);
		if (err || entry->entry_id == NTMP_NULL_ENTRY_ID)
			break;

		cfg = le32_to_cpu(cfge->cfg);
		is_static = (cfg & FDBT_DYNAMIC) ? false : true;
		vid = le16_to_cpu(keye->fid);
		if (vid >= NETC_VLAN_UNAWARE_PVID(ds->max_num_bridges))
			vid = 0;

		err = cb(keye->mac_addr, vid, is_static, data);
		if (err)
			break;

		/* To prevent hardware malfunctions from causing an
		 * infinite loop.
		 */
		if (++cnt >= priv->htmcapr_num_words)
			break;
	} while (resume_eid != NTMP_NULL_ENTRY_ID);

	mutex_unlock(&priv->fdbt_lock);
	kfree(entry);

	return err;
}

static int netc_port_mdb_add(struct dsa_switch *ds, int port,
			     const struct switchdev_obj_port_mdb *mdb,
			     struct dsa_db db)
{
	return netc_port_fdb_add(ds, port, mdb->addr, mdb->vid, db);
}

static int netc_port_mdb_del(struct dsa_switch *ds, int port,
			     const struct switchdev_obj_port_mdb *mdb,
			     struct dsa_db db)
{
	return netc_port_fdb_del(ds, port, mdb->addr, mdb->vid, db);
}

static int netc_port_add_host_flood_rule(struct netc_port *np,
					 bool uc, bool mc)
{
	const u8 dmac_mask[ETH_ALEN] = {0x1, 0, 0, 0, 0, 0};
	struct netc_switch *priv = np->switch_priv;
	struct ipft_entry_data *host_flood;
	struct ipft_keye_data *keye;
	struct ipft_cfge_data *cfge;
	u16 src_port;
	u32 cfg;
	int err;

	if (!uc && !mc) {
		/* Disable ingress port filter table lookup */
		netc_port_wr(np, NETC_PIPFCR, 0);
		np->uc = false;
		np->mc = false;

		return 0;
	}

	host_flood = kzalloc_obj(*host_flood);
	if (!host_flood)
		return -ENOMEM;

	keye = &host_flood->keye;
	cfge = &host_flood->cfge;

	src_port = FIELD_PREP(IPFT_SRC_PORT, np->dp->index);
	src_port |= IPFT_SRC_PORT_MASK;
	keye->src_port = cpu_to_le16(src_port);

	/* If either only unicast or only multicast need to be flooded
	 * to the host, we always set the mask that tests the first MAC
	 * DA octet. The value should be 0 for the first bit (if unicast
	 * has to be flooded) or 1 (if multicast). If both unicast and
	 * multicast have to be flooded, we leave the key mask empty, so
	 * it matches everything.
	 */
	if (uc && !mc)
		ether_addr_copy(keye->dmac_mask, dmac_mask);

	if (!uc && mc) {
		ether_addr_copy(keye->dmac, dmac_mask);
		ether_addr_copy(keye->dmac_mask, dmac_mask);
	}

	cfg = FIELD_PREP(IPFT_FLTFA, IPFT_FLTFA_REDIRECT);
	cfg |= FIELD_PREP(IPFT_HR, NETC_HR_HOST_FLOOD);
	cfge->cfg = cpu_to_le32(cfg);

	err = ntmp_ipft_add_entry(&priv->ntmp, host_flood);
	if (err) {
		kfree(host_flood);
		return err;
	}

	np->uc = uc;
	np->mc = mc;
	np->host_flood = host_flood;
	/* Enable ingress port filter table lookup */
	netc_port_wr(np, NETC_PIPFCR, PIPFCR_EN);

	return 0;
}

static void netc_port_remove_host_flood(struct netc_port *np,
					struct ipft_entry_data *host_flood)
{
	struct netc_switch *priv = np->switch_priv;
	bool disable_host_flood = false;

	if (!host_flood)
		return;

	if (np->host_flood == host_flood)
		disable_host_flood = true;

	ntmp_ipft_delete_entry(&priv->ntmp, host_flood->entry_id);
	kfree(host_flood);

	if (disable_host_flood) {
		np->host_flood = NULL;
		np->uc = false;
		np->mc = false;
		netc_port_wr(np, NETC_PIPFCR, 0);
	}
}

static void netc_port_set_host_flood(struct dsa_switch *ds, int port,
				     bool uc, bool mc)
{
	struct netc_port *np = NETC_PORT(ds, port);
	struct ipft_entry_data *old_host_flood;

	/* Do not add host flood rule to ingress port filter table when
	 * the port has joined a bridge. Otherwise, the ingress frames
	 * will bypass FDB table lookup and MAC learning, so the frames
	 * will be redirected directly to the CPU port.
	 */
	if (dsa_port_bridge_dev_get(np->dp)) {
		netc_port_remove_host_flood(np, np->host_flood);

		return;
	}

	if (np->uc == uc && np->mc == mc)
		return;

	/* IPFT does not support in-place updates to the KEYE element,
	 * we need to add a new entry and then delete the old one. So
	 * save the old entry first.
	 */
	old_host_flood = np->host_flood;
	np->host_flood = NULL;

	if (netc_port_add_host_flood_rule(np, uc, mc)) {
		np->host_flood = old_host_flood;
		dev_err(ds->dev, "Failed to add host flood rule on port %d\n",
			port);
		return;
	}

	/* Remove the old host flood entry */
	netc_port_remove_host_flood(np, old_host_flood);
}

static int netc_single_vlan_aware_bridge(struct dsa_switch *ds,
					 struct netlink_ext_ack *extack)
{
	struct net_device *br_ndev = NULL;
	struct dsa_port *dp;

	dsa_switch_for_each_available_port(dp, ds) {
		struct net_device *port_br = dsa_port_bridge_dev_get(dp);

		if (!port_br || !br_vlan_enabled(port_br))
			continue;

		if (!br_ndev) {
			br_ndev = port_br;
			continue;
		}

		if (br_ndev == port_br)
			continue;

		NL_SET_ERR_MSG_MOD(extack,
				   "Only one VLAN-aware bridge is supported");

		return -EBUSY;
	}

	return 0;
}

static int netc_port_vlan_filtering(struct dsa_switch *ds,
				    int port, bool vlan_aware,
				    struct netlink_ext_ack *extack)
{
	struct netc_port *np = NETC_PORT(ds, port);
	u16 pvid;
	int err;

	/* Before calling port_vlan_filtering(), br_vlan_filter_toggle() has
	 * already updated the BROPT_VLAN_ENABLED bit of br->options. So the
	 * VLAN filtering status of the switch ports can be checked by the
	 * br_vlan_enabled() function.
	 */
	err = netc_single_vlan_aware_bridge(ds, extack);
	if (err)
		return err;

	pvid = netc_vlan_unaware_pvid(np->dp->bridge);
	if (pvid == NETC_STANDALONE_PVID) {
		vlan_aware = false;
		goto bpdvr_config;
	}

	if (vlan_aware) {
		/* The FDB entries associated with unaware_pvid do not need
		 * to be deleted, so that when switching from VLAN-aware to
		 * VLAN-unaware mode, these FDB entries do not need to be
		 * re-added.
		 */
		err = netc_port_del_vlan_entry(np, pvid);
		if (err)
			return err;

		pvid = np->pvid;
	} else {
		err = netc_port_set_vlan_entry(np, pvid, false);
		if (err)
			return err;
	}

bpdvr_config:
	netc_port_set_vlan_aware(np, vlan_aware);
	netc_port_set_pvid(np, pvid);

	return 0;
}

static int netc_port_vlan_add(struct dsa_switch *ds, int port,
			      const struct switchdev_obj_port_vlan *vlan,
			      struct netlink_ext_ack *extack)
{
	struct netc_port *np = NETC_PORT(ds, port);
	struct dsa_port *dp = np->dp;
	bool untagged;
	int err;

	/* The 8021q layer may attempt to change NETC_STANDALONE_PVID
	 * (VID 0), so we need to ignore it.
	 */
	if (vlan->vid == NETC_STANDALONE_PVID)
		return 0;

	if (vlan->vid >= NETC_VLAN_UNAWARE_PVID(ds->max_num_bridges)) {
		NL_SET_ERR_MSG_FMT_MOD(extack,
				       "VID %d~4095 reserved for VLAN-unaware bridge",
				       NETC_VLAN_UNAWARE_PVID(ds->max_num_bridges));
		return -EINVAL;
	}

	untagged = !!(vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED);
	err = netc_port_set_vlan_entry(np, vlan->vid, untagged);
	if (err)
		return err;

	if (vlan->flags & BRIDGE_VLAN_INFO_PVID) {
		np->pvid = vlan->vid;
		if (dsa_port_is_vlan_filtering(dp))
			netc_port_set_pvid(np, vlan->vid);

		return 0;
	}

	if (np->pvid != vlan->vid)
		return 0;

	/* Delete PVID */
	np->pvid = NETC_STANDALONE_PVID;
	if (dsa_port_is_vlan_filtering(dp))
		netc_port_set_pvid(np, NETC_STANDALONE_PVID);

	return 0;
}

static int netc_port_vlan_del(struct dsa_switch *ds, int port,
			      const struct switchdev_obj_port_vlan *vlan)
{
	struct netc_port *np = NETC_PORT(ds, port);
	int err;

	if (vlan->vid == NETC_STANDALONE_PVID)
		return 0;

	if (vlan->vid >= NETC_VLAN_UNAWARE_PVID(ds->max_num_bridges))
		return -EINVAL;

	err = netc_port_del_vlan_entry(np, vlan->vid);
	if (err)
		return err;

	if (np->pvid == vlan->vid) {
		np->pvid = NETC_STANDALONE_PVID;

		/* Set the port PVID to NETC_STANDALONE_PVID if the VLAN-aware
		 * bridge port has no PVID. The untagged frames will not be
		 * forwarded to other user ports, as NETC_STANDALONE_PVID VLAN
		 * entry has disabled MAC learning and flooding, and other user
		 * ports do not have FDB entries with NETC_STANDALONE_PVID.
		 */
		if (dsa_port_is_vlan_filtering(np->dp))
			netc_port_set_pvid(np, NETC_STANDALONE_PVID);
	}

	return 0;
}

static int netc_port_bridge_join(struct dsa_switch *ds, int port,
				 struct dsa_bridge bridge,
				 bool *tx_fwd_offload,
				 struct netlink_ext_ack *extack)
{
	struct netc_port *np = NETC_PORT(ds, port);
	struct netc_switch *priv = ds->priv;
	u16 vlan_unaware_pvid;
	int err;

	if (!bridge.num) {
		NL_SET_ERR_MSG_MOD(extack, "Bridge number 0 is unsupported");
		return -EINVAL;
	}

	err = netc_single_vlan_aware_bridge(ds, extack);
	if (err)
		return err;

	netc_port_set_mlo(np, MLO_NOT_OVERRIDE);

	if (br_vlan_enabled(bridge.dev))
		goto out;

	vlan_unaware_pvid = NETC_VLAN_UNAWARE_PVID(bridge.num);
	err = netc_port_set_vlan_entry(np, vlan_unaware_pvid, false);
	if (err)
		goto disable_mlo;

	netc_port_set_pvid(np, vlan_unaware_pvid);

out:
	netc_port_remove_host_flood(np, np->host_flood);

	if (atomic_inc_return(&priv->br_cnt) == 1)
		schedule_delayed_work(&priv->fdbt_ageing_work,
				      READ_ONCE(priv->fdbt_ageing_delay));

	return 0;

disable_mlo:
	netc_port_set_mlo(np, MLO_DISABLE);

	return err;
}

static void netc_port_remove_dynamic_entries(struct netc_port *np)
{
	struct netc_switch *priv = np->switch_priv;

	/* Return if the port is not available */
	if (!np->dp)
		return;

	mutex_lock(&priv->fdbt_lock);
	ntmp_fdbt_delete_port_dynamic_entries(&priv->ntmp, np->dp->index);
	mutex_unlock(&priv->fdbt_lock);
}

static void netc_port_bridge_leave(struct dsa_switch *ds, int port,
				   struct dsa_bridge bridge)
{
	struct netc_port *np = NETC_PORT(ds, port);
	struct net_device *ndev = np->dp->user;
	struct netc_switch *priv = ds->priv;
	u16 vlan_unaware_pvid;
	bool mc, uc;

	netc_port_set_mlo(np, MLO_DISABLE);
	netc_port_set_pvid(np, NETC_STANDALONE_PVID);
	np->pvid = NETC_STANDALONE_PVID;

	if (atomic_dec_and_test(&priv->br_cnt))
		cancel_delayed_work_sync(&priv->fdbt_ageing_work);

	netc_port_remove_dynamic_entries(np);
	uc = ndev->flags & IFF_PROMISC;
	mc = ndev->flags & (IFF_PROMISC | IFF_ALLMULTI);

	if (netc_port_add_host_flood_rule(np, uc, mc))
		dev_warn(ds->dev,
			 "Failed to restore host flood rule on port %d\n",
			 port);

	/* When a port leaves a VLAN-aware bridge, dsa_port_bridge_leave()
	 * follows the sequence below:
	 *
	 * 1. dsa_port_bridge_destroy() is called to set dp->bridge to NULL.
	 * 2. dsa_broadcast() is called, which eventually invokes
	 *    ds->ops->port_bridge_leave()
	 * 3. dsa_port_switchdev_unsync_attrs() is called, which triggers
	 *    dsa_port_reset_vlan_filtering() and ultimately calls
	 *    ds->ops->port_vlan_filtering() to transition the port from
	 *    VLAN-aware mode to VLAN-unaware mode.
	 *
	 * At step 3, since dp->bridge has already been set to NULL in step 1,
	 * netc_port_vlan_filtering() will detect this and skip the creation
	 * of an unaware PVID entry in the VLAN filter table. Therefore, it is
	 * safe to return directly here.
	 */
	if (br_vlan_enabled(bridge.dev))
		return;

	vlan_unaware_pvid = NETC_VLAN_UNAWARE_PVID(bridge.num);
	/* There is no need to check the return value even if it fails.
	 * Because the PVID has been set to NETC_STANDALONE_PVID, the
	 * frames will not match this VLAN entry.
	 */
	netc_port_del_vlan_entry(np, vlan_unaware_pvid);
}

static int netc_set_ageing_time(struct dsa_switch *ds, unsigned int msecs)
{
	struct netc_switch *priv = ds->priv;
	unsigned long delay_jiffies;

	/* The dynamic FDB entry is deleted when its activity counter reaches
	 * NETC_FDBT_AGEING_THRESH (100). Each delayed_work tick increments
	 * the counter by 1 if the entry is inactive.
	 *
	 * Therefore:
	 *   msecs (ms)    = NETC_FDBT_AGEING_THRESH * delay_ms (ms)
	 *   delay_ms      = msecs / NETC_FDBT_AGEING_THRESH
	 *   delay_jiffies = (delay_ms / 1000) * HZ
	 *                 = (msecs * HZ) / (1000 * NETC_FDBT_AGEING_THRESH)
	 *
	 * Use DIV_ROUND_CLOSEST_ULL to perform a single nearest-jiffy
	 * rounding, avoiding the two-step rounding error of the intermediate
	 * delay_ms approach.
	 *   Maximum error = +/-0.5 jiffy * 100 = +/-50000/HZ ms.
	 */
	delay_jiffies = DIV_ROUND_CLOSEST_ULL((u64)msecs * HZ,
					      1000 * NETC_FDBT_AGEING_THRESH);
	WRITE_ONCE(priv->fdbt_ageing_delay, delay_jiffies);

	if (atomic_read(&priv->br_cnt))
		mod_delayed_work(system_percpu_wq, &priv->fdbt_ageing_work,
				 READ_ONCE(priv->fdbt_ageing_delay));

	return 0;
}

static void netc_port_fast_age(struct dsa_switch *ds, int port)
{
	struct netc_port *np = NETC_PORT(ds, port);

	netc_port_remove_dynamic_entries(np);
}

static void netc_phylink_get_caps(struct dsa_switch *ds, int port,
				  struct phylink_config *config)
{
	struct netc_switch *priv = ds->priv;

	priv->info->phylink_get_caps(port, config);
}

static void netc_port_set_mac_mode(struct netc_port *np,
				   unsigned int mode,
				   phy_interface_t phy_mode)
{
	u32 mask = PM_IF_MODE_IFMODE | PM_IF_MODE_REVMII;
	u32 val = 0;

	switch (phy_mode) {
	case PHY_INTERFACE_MODE_RGMII:
	case PHY_INTERFACE_MODE_RGMII_ID:
	case PHY_INTERFACE_MODE_RGMII_RXID:
	case PHY_INTERFACE_MODE_RGMII_TXID:
		val |= IFMODE_RGMII;
		break;
	case PHY_INTERFACE_MODE_RMII:
		val |= IFMODE_RMII;
		break;
	case PHY_INTERFACE_MODE_REVMII:
		val |= PM_IF_MODE_REVMII;
		fallthrough;
	case PHY_INTERFACE_MODE_MII:
		val |= IFMODE_MII;
		break;
	case PHY_INTERFACE_MODE_SGMII:
	case PHY_INTERFACE_MODE_2500BASEX:
		val |= IFMODE_SGMII;
		break;
	default:
		break;
	}

	netc_mac_port_rmw(np, NETC_PM_IF_MODE(0), mask, val);
}

static void netc_mac_config(struct phylink_config *config, unsigned int mode,
			    const struct phylink_link_state *state)
{
	struct dsa_port *dp = dsa_phylink_to_port(config);

	netc_port_set_mac_mode(NETC_PORT(dp->ds, dp->index), mode,
			       state->interface);
}

static void netc_port_set_speed(struct netc_port *np, int speed)
{
	netc_port_rmw(np, NETC_PCR, PCR_PSPEED, PSPEED_SET_VAL(speed));
}

static void netc_port_set_rgmii_mac(struct netc_port *np,
				    int speed, int duplex)
{
	u32 mask, val;

	mask = PM_IF_MODE_SSP | PM_IF_MODE_HD | PM_IF_MODE_M10;

	switch (speed) {
	default:
	case SPEED_1000:
		val = FIELD_PREP(PM_IF_MODE_SSP, SSP_1G);
		break;
	case SPEED_100:
		val = FIELD_PREP(PM_IF_MODE_SSP, SSP_100M);
		break;
	case SPEED_10:
		val = FIELD_PREP(PM_IF_MODE_SSP, SSP_10M);
		break;
	}

	if (duplex != DUPLEX_FULL)
		val |= PM_IF_MODE_HD;

	netc_mac_port_rmw(np, NETC_PM_IF_MODE(0), mask, val);
}

static void netc_port_set_rmii_mii_mac(struct netc_port *np,
				       int speed, int duplex)
{
	u32 mask, val = 0;

	mask = PM_IF_MODE_SSP | PM_IF_MODE_HD | PM_IF_MODE_M10;

	if (speed == SPEED_10)
		val |= PM_IF_MODE_M10;

	if (duplex != DUPLEX_FULL)
		val |= PM_IF_MODE_HD;

	netc_mac_port_rmw(np, NETC_PM_IF_MODE(0), mask, val);
}

static void netc_port_set_tx_pause(struct netc_port *np, bool tx_pause)
{
	struct netc_switch *priv = np->switch_priv;
	int port = np->dp->index;
	int i, j, num_bp;

	num_bp = priv->num_bp / priv->info->num_ports;
	for (i = 0, j = port * num_bp; i < num_bp; i++, j++) {
		struct bpt_cfge_data *cfge = &priv->bpt_list[j];
		struct bpt_cfge_data old_cfge = *cfge;

		if (tx_pause) {
			cfge->fc_on_thresh = cpu_to_le16(NETC_FC_THRESH_ON);
			cfge->fc_off_thresh = cpu_to_le16(NETC_FC_THRESH_OFF);
			cfge->fccfg_sbpen = FIELD_PREP(BPT_FC_CFG,
						       BPT_FC_CFG_EN_BPFC);
			cfge->fc_ports = cpu_to_le32(BIT(port));
		} else {
			cfge->fc_on_thresh = cpu_to_le16(0);
			cfge->fc_off_thresh = cpu_to_le16(0);
			cfge->fccfg_sbpen = 0;
			cfge->fc_ports = cpu_to_le32(0);
		}

		if (ntmp_bpt_update_entry(&priv->ntmp, j, cfge)) {
			*cfge = old_cfge;
			dev_warn(priv->dev,
				 "Failed to %s TX pause of buffer pool %d (swp%d)\n",
				 tx_pause ? "enable" : "disable", j, port);
		}
	}
}

static void netc_port_set_rx_pause(struct netc_port *np, bool rx_pause)
{
	netc_mac_port_rmw(np, NETC_PM_CMD_CFG(0), PM_CMD_CFG_PAUSE_IGN,
			  rx_pause ? 0 : PM_CMD_CFG_PAUSE_IGN);
}

static void netc_port_mac_rx_enable(struct netc_port *np)
{
	netc_port_rmw(np, NETC_POR, POR_RXDIS, 0);
	netc_mac_port_rmw(np, NETC_PM_CMD_CFG(0), PM_CMD_CFG_RX_EN,
			  PM_CMD_CFG_RX_EN);
}

static void netc_port_wait_rx_empty(struct netc_port *np, int mac)
{
	u32 val;

	/* PM_IEVENT_RX_EMPTY is a read-only bit, it is automatically set by
	 * hardware if RX FIFO is empty and no RX packet receive in process.
	 * And it is automatically cleared if RX FIFO is not empty or RX
	 * packet receive in process.
	 */
	if (read_poll_timeout(netc_port_rd, val, val & PM_IEVENT_RX_EMPTY,
			      100, 10000, false, np, NETC_PM_IEVENT(mac)))
		dev_warn(np->switch_priv->dev,
			 "swp%d MAC%d: RX is not idle\n", np->dp->index, mac);
}

static void netc_port_mac_rx_graceful_stop(struct netc_port *np)
{
	u32 val;

	if (is_netc_pseudo_port(np))
		goto rx_disable;

	if (np->caps.pmac) {
		netc_port_rmw(np, NETC_PM_CMD_CFG(1), PM_CMD_CFG_RX_EN, 0);
		netc_port_wait_rx_empty(np, 1);
	}

	netc_port_rmw(np, NETC_PM_CMD_CFG(0), PM_CMD_CFG_RX_EN, 0);
	netc_port_wait_rx_empty(np, 0);

	if (read_poll_timeout(netc_port_rd, val, !(val & PSR_RX_BUSY),
			      100, 10000, false, np, NETC_PSR))
		dev_warn(np->switch_priv->dev, "swp%d RX is busy\n",
			 np->dp->index);

rx_disable:
	netc_port_rmw(np, NETC_POR, POR_RXDIS, POR_RXDIS);
}

static void netc_port_mac_tx_enable(struct netc_port *np)
{
	netc_mac_port_rmw(np, NETC_PM_CMD_CFG(0), PM_CMD_CFG_TX_EN,
			  PM_CMD_CFG_TX_EN);
	netc_port_rmw(np, NETC_POR, POR_TXDIS, 0);
}

static void netc_port_wait_tx_empty(struct netc_port *np, int mac)
{
	u32 val;

	/* PM_IEVENT_TX_EMPTY is a read-only bit, it is automatically set by
	 * hardware if TX FIFO is empty. And it is automatically cleared if
	 * TX FIFO is not empty.
	 */
	if (read_poll_timeout(netc_port_rd, val, val & PM_IEVENT_TX_EMPTY,
			      100, 10000, false, np, NETC_PM_IEVENT(mac)))
		dev_warn(np->switch_priv->dev,
			 "swp%d MAC%d: TX FIFO is not empty\n",
			 np->dp->index, mac);
}

static void netc_port_mac_tx_graceful_stop(struct netc_port *np)
{
	netc_port_rmw(np, NETC_POR, POR_TXDIS, POR_TXDIS);

	if (is_netc_pseudo_port(np))
		return;

	netc_port_wait_tx_empty(np, 0);
	if (np->caps.pmac)
		netc_port_wait_tx_empty(np, 1);

	netc_mac_port_rmw(np, NETC_PM_CMD_CFG(0), PM_CMD_CFG_TX_EN, 0);
}

static void netc_mac_link_up(struct phylink_config *config,
			     struct phy_device *phy, unsigned int mode,
			     phy_interface_t interface, int speed,
			     int duplex, bool tx_pause, bool rx_pause)
{
	struct dsa_port *dp = dsa_phylink_to_port(config);
	struct netc_port *np;

	np = NETC_PORT(dp->ds, dp->index);
	netc_port_set_speed(np, speed);

	if (phy_interface_mode_is_rgmii(interface))
		netc_port_set_rgmii_mac(np, speed, duplex);

	if (interface == PHY_INTERFACE_MODE_RMII ||
	    interface == PHY_INTERFACE_MODE_REVMII ||
	    interface == PHY_INTERFACE_MODE_MII)
		netc_port_set_rmii_mii_mac(np, speed, duplex);

	netc_port_set_tx_pause(np, tx_pause);
	netc_port_set_rx_pause(np, rx_pause);
	netc_port_mac_tx_enable(np);
	netc_port_mac_rx_enable(np);
}

static void netc_mac_link_down(struct phylink_config *config,
			       unsigned int mode,
			       phy_interface_t interface)
{
	struct dsa_port *dp = dsa_phylink_to_port(config);
	struct netc_port *np;

	np = NETC_PORT(dp->ds, dp->index);
	netc_port_mac_rx_graceful_stop(np);
	netc_port_mac_tx_graceful_stop(np);
	netc_port_remove_dynamic_entries(np);
}

static const struct phylink_mac_ops netc_phylink_mac_ops = {
	.mac_config		= netc_mac_config,
	.mac_link_up		= netc_mac_link_up,
	.mac_link_down		= netc_mac_link_down,
};

static const struct dsa_switch_ops netc_switch_ops = {
	.get_tag_protocol		= netc_get_tag_protocol,
	.setup				= netc_setup,
	.teardown			= netc_teardown,
	.phylink_get_caps		= netc_phylink_get_caps,
	.port_enable			= netc_port_enable,
	.port_disable			= netc_port_disable,
	.port_stp_state_set		= netc_port_stp_state_set,
	.port_change_mtu		= netc_port_change_mtu,
	.port_max_mtu			= netc_port_max_mtu,
	.port_fdb_add			= netc_port_fdb_add,
	.port_fdb_del			= netc_port_fdb_del,
	.port_fdb_dump			= netc_port_fdb_dump,
	.port_mdb_add			= netc_port_mdb_add,
	.port_mdb_del			= netc_port_mdb_del,
	.port_set_host_flood		= netc_port_set_host_flood,
	.port_vlan_filtering		= netc_port_vlan_filtering,
	.port_vlan_add			= netc_port_vlan_add,
	.port_vlan_del			= netc_port_vlan_del,
	.port_bridge_join		= netc_port_bridge_join,
	.port_bridge_leave		= netc_port_bridge_leave,
	.set_ageing_time		= netc_set_ageing_time,
	.port_fast_age			= netc_port_fast_age,
	.get_pause_stats		= netc_port_get_pause_stats,
	.get_rmon_stats			= netc_port_get_rmon_stats,
	.get_eth_ctrl_stats		= netc_port_get_eth_ctrl_stats,
	.get_eth_mac_stats		= netc_port_get_eth_mac_stats,
	.get_sset_count			= netc_port_get_sset_count,
	.get_strings			= netc_port_get_strings,
	.get_ethtool_stats		= netc_port_get_ethtool_stats,
};

static int netc_switch_probe(struct pci_dev *pdev,
			     const struct pci_device_id *id)
{
	struct device_node *node = dev_of_node(&pdev->dev);
	struct device *dev = &pdev->dev;
	struct netc_switch *priv;
	struct dsa_switch *ds;
	int err;

	if (!node)
		return dev_err_probe(dev, -ENODEV,
				     "No DT bindings, skipping\n");

	err = netc_switch_check_emdio_is_ready(dev);
	if (err)
		return err;

	err = netc_switch_pci_init(pdev);
	if (err)
		return err;

	priv = pci_get_drvdata(pdev);
	netc_switch_get_ip_revision(priv);

	err = netc_switch_platform_probe(priv);
	if (err)
		return err;

	ds = devm_kzalloc(dev, sizeof(*ds), GFP_KERNEL);
	if (!ds)
		return -ENOMEM;

	ds->dev = dev;
	ds->num_ports = priv->info->num_ports;
	ds->num_tx_queues = NETC_TC_NUM;
	ds->ops = &netc_switch_ops;
	ds->phylink_mac_ops = &netc_phylink_mac_ops;
	ds->fdb_isolation = true;
	ds->max_num_bridges = priv->info->num_ports - 1;
	ds->ageing_time_min = 1000;
	ds->ageing_time_max = U32_MAX;
	ds->priv = priv;
	priv->ds = ds;

	err = dsa_register_switch(ds);
	if (err)
		return dev_err_probe(dev, err,
				     "Failed to register DSA switch\n");

	return 0;
}

static void netc_switch_remove(struct pci_dev *pdev)
{
	struct netc_switch *priv = pci_get_drvdata(pdev);

	if (!priv)
		return;

	dsa_unregister_switch(priv->ds);
}

static void netc_switch_shutdown(struct pci_dev *pdev)
{
	struct netc_switch *priv = pci_get_drvdata(pdev);

	if (!priv)
		return;

	dsa_switch_shutdown(priv->ds);
	pci_set_drvdata(pdev, NULL);
}

static const struct pci_device_id netc_switch_ids[] = {
	{ PCI_DEVICE(NETC_SWITCH_VENDOR_ID, NETC_SWITCH_DEVICE_ID) },
	{ }
};
MODULE_DEVICE_TABLE(pci, netc_switch_ids);

static struct pci_driver netc_switch_driver = {
	.name		= KBUILD_MODNAME,
	.id_table	= netc_switch_ids,
	.probe		= netc_switch_probe,
	.remove		= netc_switch_remove,
	.shutdown	= netc_switch_shutdown,
};
module_pci_driver(netc_switch_driver);

MODULE_DESCRIPTION("NXP NETC Switch driver");
MODULE_LICENSE("Dual BSD/GPL");