xref: /linux/net/ipv4/udp_tunnel_nic.c (revision 8be4d31cb8aaeea27bde4b7ddb26e28a89062ebf)

// SPDX-License-Identifier: GPL-2.0-only
// Copyright (c) 2020 Facebook Inc.

#include <linux/ethtool_netlink.h>
#include <linux/netdevice.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/workqueue.h>
#include <net/udp_tunnel.h>
#include <net/vxlan.h>

enum udp_tunnel_nic_table_entry_flags {
	UDP_TUNNEL_NIC_ENTRY_ADD	= BIT(0),
	UDP_TUNNEL_NIC_ENTRY_DEL	= BIT(1),
	UDP_TUNNEL_NIC_ENTRY_OP_FAIL	= BIT(2),
	UDP_TUNNEL_NIC_ENTRY_FROZEN	= BIT(3),
};

struct udp_tunnel_nic_table_entry {
	__be16 port;
	u8 type;
	u8 flags;
	u16 use_cnt;
#define UDP_TUNNEL_NIC_USE_CNT_MAX	U16_MAX
	u8 hw_priv;
};

/**
 * struct udp_tunnel_nic - UDP tunnel port offload state
 * @work:	async work for talking to hardware from process context
 * @dev:	netdev pointer
 * @lock:	protects all fields
 * @need_sync:	at least one port start changed
 * @need_replay: space was freed, we need a replay of all ports
 * @work_pending: @work is currently scheduled
 * @n_tables:	number of tables under @entries
 * @missed:	bitmap of tables which overflown
 * @entries:	table of tables of ports currently offloaded
 */
struct udp_tunnel_nic {
	struct work_struct work;

	struct net_device *dev;

	struct mutex lock;

	u8 need_sync:1;
	u8 need_replay:1;
	u8 work_pending:1;

	unsigned int n_tables;
	unsigned long missed;
	struct udp_tunnel_nic_table_entry *entries[] __counted_by(n_tables);
};

/* We ensure all work structs are done using driver state, but not the code.
 * We need a workqueue we can flush before module gets removed.
 */
static struct workqueue_struct *udp_tunnel_nic_workqueue;

static const char *udp_tunnel_nic_tunnel_type_name(unsigned int type)
{
	switch (type) {
	case UDP_TUNNEL_TYPE_VXLAN:
		return "vxlan";
	case UDP_TUNNEL_TYPE_GENEVE:
		return "geneve";
	case UDP_TUNNEL_TYPE_VXLAN_GPE:
		return "vxlan-gpe";
	default:
		return "unknown";
	}
}

static bool
udp_tunnel_nic_entry_is_free(struct udp_tunnel_nic_table_entry *entry)
{
	return entry->use_cnt == 0 && !entry->flags;
}

static bool
udp_tunnel_nic_entry_is_present(struct udp_tunnel_nic_table_entry *entry)
{
	return entry->use_cnt && !(entry->flags & ~UDP_TUNNEL_NIC_ENTRY_FROZEN);
}

static bool
udp_tunnel_nic_entry_is_frozen(struct udp_tunnel_nic_table_entry *entry)
{
	return entry->flags & UDP_TUNNEL_NIC_ENTRY_FROZEN;
}

static void
udp_tunnel_nic_entry_freeze_used(struct udp_tunnel_nic_table_entry *entry)
{
	if (!udp_tunnel_nic_entry_is_free(entry))
		entry->flags |= UDP_TUNNEL_NIC_ENTRY_FROZEN;
}

static void
udp_tunnel_nic_entry_unfreeze(struct udp_tunnel_nic_table_entry *entry)
{
	entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_FROZEN;
}

static bool
udp_tunnel_nic_entry_is_queued(struct udp_tunnel_nic_table_entry *entry)
{
	return entry->flags & (UDP_TUNNEL_NIC_ENTRY_ADD |
			       UDP_TUNNEL_NIC_ENTRY_DEL);
}

static void
udp_tunnel_nic_entry_queue(struct udp_tunnel_nic *utn,
			   struct udp_tunnel_nic_table_entry *entry,
			   unsigned int flag)
{
	entry->flags |= flag;
	utn->need_sync = 1;
}

static void
udp_tunnel_nic_ti_from_entry(struct udp_tunnel_nic_table_entry *entry,
			     struct udp_tunnel_info *ti)
{
	memset(ti, 0, sizeof(*ti));
	ti->port = entry->port;
	ti->type = entry->type;
	ti->hw_priv = entry->hw_priv;
}

static bool
udp_tunnel_nic_is_empty(struct net_device *dev, struct udp_tunnel_nic *utn)
{
	const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
	unsigned int i, j;

	for (i = 0; i < utn->n_tables; i++)
		for (j = 0; j < info->tables[i].n_entries; j++)
			if (!udp_tunnel_nic_entry_is_free(&utn->entries[i][j]))
				return false;
	return true;
}

static bool
udp_tunnel_nic_should_replay(struct net_device *dev, struct udp_tunnel_nic *utn)
{
	const struct udp_tunnel_nic_table_info *table;
	unsigned int i, j;

	if (!utn->missed)
		return false;

	for (i = 0; i < utn->n_tables; i++) {
		table = &dev->udp_tunnel_nic_info->tables[i];
		if (!test_bit(i, &utn->missed))
			continue;

		for (j = 0; j < table->n_entries; j++)
			if (udp_tunnel_nic_entry_is_free(&utn->entries[i][j]))
				return true;
	}

	return false;
}

static void
__udp_tunnel_nic_get_port(struct net_device *dev, unsigned int table,
			  unsigned int idx, struct udp_tunnel_info *ti)
{
	struct udp_tunnel_nic_table_entry *entry;
	struct udp_tunnel_nic *utn;

	utn = dev->udp_tunnel_nic;
	entry = &utn->entries[table][idx];

	if (entry->use_cnt)
		udp_tunnel_nic_ti_from_entry(entry, ti);
}

static void
__udp_tunnel_nic_set_port_priv(struct net_device *dev, unsigned int table,
			       unsigned int idx, u8 priv)
{
	dev->udp_tunnel_nic->entries[table][idx].hw_priv = priv;
}

static void
udp_tunnel_nic_entry_update_done(struct udp_tunnel_nic_table_entry *entry,
				 int err)
{
	bool dodgy = entry->flags & UDP_TUNNEL_NIC_ENTRY_OP_FAIL;

	WARN_ON_ONCE(entry->flags & UDP_TUNNEL_NIC_ENTRY_ADD &&
		     entry->flags & UDP_TUNNEL_NIC_ENTRY_DEL);

	if (entry->flags & UDP_TUNNEL_NIC_ENTRY_ADD &&
	    (!err || (err == -EEXIST && dodgy)))
		entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_ADD;

	if (entry->flags & UDP_TUNNEL_NIC_ENTRY_DEL &&
	    (!err || (err == -ENOENT && dodgy)))
		entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_DEL;

	if (!err)
		entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
	else
		entry->flags |= UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
}

static void
udp_tunnel_nic_device_sync_one(struct net_device *dev,
			       struct udp_tunnel_nic *utn,
			       unsigned int table, unsigned int idx)
{
	struct udp_tunnel_nic_table_entry *entry;
	struct udp_tunnel_info ti;
	int err;

	entry = &utn->entries[table][idx];
	if (!udp_tunnel_nic_entry_is_queued(entry))
		return;

	udp_tunnel_nic_ti_from_entry(entry, &ti);
	if (entry->flags & UDP_TUNNEL_NIC_ENTRY_ADD)
		err = dev->udp_tunnel_nic_info->set_port(dev, table, idx, &ti);
	else
		err = dev->udp_tunnel_nic_info->unset_port(dev, table, idx,
							   &ti);
	udp_tunnel_nic_entry_update_done(entry, err);

	if (err)
		netdev_warn(dev,
			    "UDP tunnel port sync failed port %d type %s: %d\n",
			    be16_to_cpu(entry->port),
			    udp_tunnel_nic_tunnel_type_name(entry->type),
			    err);
}

static void
udp_tunnel_nic_device_sync_by_port(struct net_device *dev,
				   struct udp_tunnel_nic *utn)
{
	const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
	unsigned int i, j;

	for (i = 0; i < utn->n_tables; i++)
		for (j = 0; j < info->tables[i].n_entries; j++)
			udp_tunnel_nic_device_sync_one(dev, utn, i, j);
}

static void
udp_tunnel_nic_device_sync_by_table(struct net_device *dev,
				    struct udp_tunnel_nic *utn)
{
	const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
	unsigned int i, j;
	int err;

	for (i = 0; i < utn->n_tables; i++) {
		/* Find something that needs sync in this table */
		for (j = 0; j < info->tables[i].n_entries; j++)
			if (udp_tunnel_nic_entry_is_queued(&utn->entries[i][j]))
				break;
		if (j == info->tables[i].n_entries)
			continue;

		err = info->sync_table(dev, i);
		if (err)
			netdev_warn(dev, "UDP tunnel port sync failed for table %d: %d\n",
				    i, err);

		for (j = 0; j < info->tables[i].n_entries; j++) {
			struct udp_tunnel_nic_table_entry *entry;

			entry = &utn->entries[i][j];
			if (udp_tunnel_nic_entry_is_queued(entry))
				udp_tunnel_nic_entry_update_done(entry, err);
		}
	}
}

static void
__udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn)
{
	if (!utn->need_sync)
		return;

	if (dev->udp_tunnel_nic_info->sync_table)
		udp_tunnel_nic_device_sync_by_table(dev, utn);
	else
		udp_tunnel_nic_device_sync_by_port(dev, utn);

	utn->need_sync = 0;
	/* Can't replay directly here, in case we come from the tunnel driver's
	 * notification - trying to replay may deadlock inside tunnel driver.
	 */
	utn->need_replay = udp_tunnel_nic_should_replay(dev, utn);
}

static void
udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn)
{
	if (!utn->need_sync)
		return;

	queue_work(udp_tunnel_nic_workqueue, &utn->work);
	utn->work_pending = 1;
}

static bool
udp_tunnel_nic_table_is_capable(const struct udp_tunnel_nic_table_info *table,
				struct udp_tunnel_info *ti)
{
	return table->tunnel_types & ti->type;
}

static bool
udp_tunnel_nic_is_capable(struct net_device *dev, struct udp_tunnel_nic *utn,
			  struct udp_tunnel_info *ti)
{
	const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
	unsigned int i;

	/* Special case IPv4-only NICs */
	if (info->flags & UDP_TUNNEL_NIC_INFO_IPV4_ONLY &&
	    ti->sa_family != AF_INET)
		return false;

	for (i = 0; i < utn->n_tables; i++)
		if (udp_tunnel_nic_table_is_capable(&info->tables[i], ti))
			return true;
	return false;
}

static int
udp_tunnel_nic_has_collision(struct net_device *dev, struct udp_tunnel_nic *utn,
			     struct udp_tunnel_info *ti)
{
	const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
	struct udp_tunnel_nic_table_entry *entry;
	unsigned int i, j;

	for (i = 0; i < utn->n_tables; i++)
		for (j = 0; j < info->tables[i].n_entries; j++) {
			entry =	&utn->entries[i][j];

			if (!udp_tunnel_nic_entry_is_free(entry) &&
			    entry->port == ti->port &&
			    entry->type != ti->type) {
				__set_bit(i, &utn->missed);
				return true;
			}
		}
	return false;
}

static void
udp_tunnel_nic_entry_adj(struct udp_tunnel_nic *utn,
			 unsigned int table, unsigned int idx, int use_cnt_adj)
{
	struct udp_tunnel_nic_table_entry *entry =  &utn->entries[table][idx];
	bool dodgy = entry->flags & UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
	unsigned int from, to;

	WARN_ON(entry->use_cnt + (u32)use_cnt_adj > U16_MAX);

	/* If not going from used to unused or vice versa - all done.
	 * For dodgy entries make sure we try to sync again (queue the entry).
	 */
	entry->use_cnt += use_cnt_adj;
	if (!dodgy && !entry->use_cnt == !(entry->use_cnt - use_cnt_adj))
		return;

	/* Cancel the op before it was sent to the device, if possible,
	 * otherwise we'd need to take special care to issue commands
	 * in the same order the ports arrived.
	 */
	if (use_cnt_adj < 0) {
		from = UDP_TUNNEL_NIC_ENTRY_ADD;
		to = UDP_TUNNEL_NIC_ENTRY_DEL;
	} else {
		from = UDP_TUNNEL_NIC_ENTRY_DEL;
		to = UDP_TUNNEL_NIC_ENTRY_ADD;
	}

	if (entry->flags & from) {
		entry->flags &= ~from;
		if (!dodgy)
			return;
	}

	udp_tunnel_nic_entry_queue(utn, entry, to);
}

static bool
udp_tunnel_nic_entry_try_adj(struct udp_tunnel_nic *utn,
			     unsigned int table, unsigned int idx,
			     struct udp_tunnel_info *ti, int use_cnt_adj)
{
	struct udp_tunnel_nic_table_entry *entry =  &utn->entries[table][idx];

	if (udp_tunnel_nic_entry_is_free(entry) ||
	    entry->port != ti->port ||
	    entry->type != ti->type)
		return false;

	if (udp_tunnel_nic_entry_is_frozen(entry))
		return true;

	udp_tunnel_nic_entry_adj(utn, table, idx, use_cnt_adj);
	return true;
}

/* Try to find existing matching entry and adjust its use count, instead of
 * adding a new one. Returns true if entry was found. In case of delete the
 * entry may have gotten removed in the process, in which case it will be
 * queued for removal.
 */
static bool
udp_tunnel_nic_try_existing(struct net_device *dev, struct udp_tunnel_nic *utn,
			    struct udp_tunnel_info *ti, int use_cnt_adj)
{
	const struct udp_tunnel_nic_table_info *table;
	unsigned int i, j;

	for (i = 0; i < utn->n_tables; i++) {
		table = &dev->udp_tunnel_nic_info->tables[i];
		if (!udp_tunnel_nic_table_is_capable(table, ti))
			continue;

		for (j = 0; j < table->n_entries; j++)
			if (udp_tunnel_nic_entry_try_adj(utn, i, j, ti,
							 use_cnt_adj))
				return true;
	}

	return false;
}

static bool
udp_tunnel_nic_add_existing(struct net_device *dev, struct udp_tunnel_nic *utn,
			    struct udp_tunnel_info *ti)
{
	return udp_tunnel_nic_try_existing(dev, utn, ti, +1);
}

static bool
udp_tunnel_nic_del_existing(struct net_device *dev, struct udp_tunnel_nic *utn,
			    struct udp_tunnel_info *ti)
{
	return udp_tunnel_nic_try_existing(dev, utn, ti, -1);
}

static bool
udp_tunnel_nic_add_new(struct net_device *dev, struct udp_tunnel_nic *utn,
		       struct udp_tunnel_info *ti)
{
	const struct udp_tunnel_nic_table_info *table;
	unsigned int i, j;

	for (i = 0; i < utn->n_tables; i++) {
		table = &dev->udp_tunnel_nic_info->tables[i];
		if (!udp_tunnel_nic_table_is_capable(table, ti))
			continue;

		for (j = 0; j < table->n_entries; j++) {
			struct udp_tunnel_nic_table_entry *entry;

			entry = &utn->entries[i][j];
			if (!udp_tunnel_nic_entry_is_free(entry))
				continue;

			entry->port = ti->port;
			entry->type = ti->type;
			entry->use_cnt = 1;
			udp_tunnel_nic_entry_queue(utn, entry,
						   UDP_TUNNEL_NIC_ENTRY_ADD);
			return true;
		}

		/* The different table may still fit this port in, but there
		 * are no devices currently which have multiple tables accepting
		 * the same tunnel type, and false positives are okay.
		 */
		__set_bit(i, &utn->missed);
	}

	return false;
}

static void
__udp_tunnel_nic_add_port(struct net_device *dev, struct udp_tunnel_info *ti)
{
	const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
	struct udp_tunnel_nic *utn;

	utn = dev->udp_tunnel_nic;
	if (!utn)
		return;
	if (!netif_running(dev) && info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY)
		return;
	if (info->flags & UDP_TUNNEL_NIC_INFO_STATIC_IANA_VXLAN &&
	    ti->port == htons(IANA_VXLAN_UDP_PORT)) {
		if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
			netdev_warn(dev, "device assumes port 4789 will be used by vxlan tunnels\n");
		return;
	}

	if (!udp_tunnel_nic_is_capable(dev, utn, ti))
		return;

	/* It may happen that a tunnel of one type is removed and different
	 * tunnel type tries to reuse its port before the device was informed.
	 * Rely on utn->missed to re-add this port later.
	 */
	if (udp_tunnel_nic_has_collision(dev, utn, ti))
		return;

	if (!udp_tunnel_nic_add_existing(dev, utn, ti))
		udp_tunnel_nic_add_new(dev, utn, ti);

	udp_tunnel_nic_device_sync(dev, utn);
}

static void
__udp_tunnel_nic_del_port(struct net_device *dev, struct udp_tunnel_info *ti)
{
	struct udp_tunnel_nic *utn;

	utn = dev->udp_tunnel_nic;
	if (!utn)
		return;

	if (!udp_tunnel_nic_is_capable(dev, utn, ti))
		return;

	udp_tunnel_nic_del_existing(dev, utn, ti);

	udp_tunnel_nic_device_sync(dev, utn);
}

static void __udp_tunnel_nic_reset_ntf(struct net_device *dev)
{
	const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
	struct udp_tunnel_nic *utn;
	unsigned int i, j;

	utn = dev->udp_tunnel_nic;
	if (!utn)
		return;

	mutex_lock(&utn->lock);

	utn->need_sync = false;
	for (i = 0; i < utn->n_tables; i++)
		for (j = 0; j < info->tables[i].n_entries; j++) {
			struct udp_tunnel_nic_table_entry *entry;

			entry = &utn->entries[i][j];

			entry->flags &= ~(UDP_TUNNEL_NIC_ENTRY_DEL |
					  UDP_TUNNEL_NIC_ENTRY_OP_FAIL);
			/* We don't release utn lock across ops */
			WARN_ON(entry->flags & UDP_TUNNEL_NIC_ENTRY_FROZEN);
			if (!entry->use_cnt)
				continue;

			udp_tunnel_nic_entry_queue(utn, entry,
						   UDP_TUNNEL_NIC_ENTRY_ADD);
		}

	__udp_tunnel_nic_device_sync(dev, utn);

	mutex_unlock(&utn->lock);
}

static size_t
__udp_tunnel_nic_dump_size(struct net_device *dev, unsigned int table)
{
	const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
	struct udp_tunnel_nic *utn;
	unsigned int j;
	size_t size;

	utn = dev->udp_tunnel_nic;
	if (!utn)
		return 0;

	size = 0;
	for (j = 0; j < info->tables[table].n_entries; j++) {
		if (!udp_tunnel_nic_entry_is_present(&utn->entries[table][j]))
			continue;

		size += nla_total_size(0) +		 /* _TABLE_ENTRY */
			nla_total_size(sizeof(__be16)) + /* _ENTRY_PORT */
			nla_total_size(sizeof(u32));	 /* _ENTRY_TYPE */
	}

	return size;
}

static int
__udp_tunnel_nic_dump_write(struct net_device *dev, unsigned int table,
			    struct sk_buff *skb)
{
	const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
	struct udp_tunnel_nic *utn;
	struct nlattr *nest;
	unsigned int j;

	utn = dev->udp_tunnel_nic;
	if (!utn)
		return 0;

	for (j = 0; j < info->tables[table].n_entries; j++) {
		if (!udp_tunnel_nic_entry_is_present(&utn->entries[table][j]))
			continue;

		nest = nla_nest_start(skb, ETHTOOL_A_TUNNEL_UDP_TABLE_ENTRY);
		if (!nest)
			return -EMSGSIZE;

		if (nla_put_be16(skb, ETHTOOL_A_TUNNEL_UDP_ENTRY_PORT,
				 utn->entries[table][j].port) ||
		    nla_put_u32(skb, ETHTOOL_A_TUNNEL_UDP_ENTRY_TYPE,
				ilog2(utn->entries[table][j].type)))
			goto err_cancel;

		nla_nest_end(skb, nest);
	}

	return 0;

err_cancel:
	nla_nest_cancel(skb, nest);
	return -EMSGSIZE;
}

static void __udp_tunnel_nic_assert_locked(struct net_device *dev)
{
	struct udp_tunnel_nic *utn;

	utn = dev->udp_tunnel_nic;
	if (utn)
		lockdep_assert_held(&utn->lock);
}

static void __udp_tunnel_nic_lock(struct net_device *dev)
{
	struct udp_tunnel_nic *utn;

	utn = dev->udp_tunnel_nic;
	if (utn)
		mutex_lock(&utn->lock);
}

static void __udp_tunnel_nic_unlock(struct net_device *dev)
{
	struct udp_tunnel_nic *utn;

	utn = dev->udp_tunnel_nic;
	if (utn)
		mutex_unlock(&utn->lock);
}

static const struct udp_tunnel_nic_ops __udp_tunnel_nic_ops = {
	.get_port	= __udp_tunnel_nic_get_port,
	.set_port_priv	= __udp_tunnel_nic_set_port_priv,
	.add_port	= __udp_tunnel_nic_add_port,
	.del_port	= __udp_tunnel_nic_del_port,
	.reset_ntf	= __udp_tunnel_nic_reset_ntf,
	.dump_size	= __udp_tunnel_nic_dump_size,
	.dump_write	= __udp_tunnel_nic_dump_write,
	.assert_locked	= __udp_tunnel_nic_assert_locked,
	.lock		= __udp_tunnel_nic_lock,
	.unlock		= __udp_tunnel_nic_unlock,
};

static void
udp_tunnel_nic_flush(struct net_device *dev, struct udp_tunnel_nic *utn)
{
	const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
	unsigned int i, j;

	for (i = 0; i < utn->n_tables; i++)
		for (j = 0; j < info->tables[i].n_entries; j++) {
			int adj_cnt = -utn->entries[i][j].use_cnt;

			if (adj_cnt)
				udp_tunnel_nic_entry_adj(utn, i, j, adj_cnt);
		}

	__udp_tunnel_nic_device_sync(dev, utn);

	for (i = 0; i < utn->n_tables; i++)
		memset(utn->entries[i], 0, array_size(info->tables[i].n_entries,
						      sizeof(**utn->entries)));
	WARN_ON(utn->need_sync);
	utn->need_replay = 0;
}

static void
udp_tunnel_nic_replay(struct net_device *dev, struct udp_tunnel_nic *utn)
{
	const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
	struct udp_tunnel_nic_shared_node *node;
	unsigned int i, j;

	/* Freeze all the ports we are already tracking so that the replay
	 * does not double up the refcount.
	 */
	for (i = 0; i < utn->n_tables; i++)
		for (j = 0; j < info->tables[i].n_entries; j++)
			udp_tunnel_nic_entry_freeze_used(&utn->entries[i][j]);
	utn->missed = 0;
	utn->need_replay = 0;

	if (!info->shared) {
		udp_tunnel_get_rx_info(dev);
	} else {
		list_for_each_entry(node, &info->shared->devices, list)
			udp_tunnel_get_rx_info(node->dev);
	}

	for (i = 0; i < utn->n_tables; i++)
		for (j = 0; j < info->tables[i].n_entries; j++)
			udp_tunnel_nic_entry_unfreeze(&utn->entries[i][j]);
}

static void udp_tunnel_nic_device_sync_work(struct work_struct *work)
{
	struct udp_tunnel_nic *utn =
		container_of(work, struct udp_tunnel_nic, work);

	rtnl_lock();
	mutex_lock(&utn->lock);

	utn->work_pending = 0;
	__udp_tunnel_nic_device_sync(utn->dev, utn);

	if (utn->need_replay)
		udp_tunnel_nic_replay(utn->dev, utn);

	mutex_unlock(&utn->lock);
	rtnl_unlock();
}

static struct udp_tunnel_nic *
udp_tunnel_nic_alloc(const struct udp_tunnel_nic_info *info,
		     unsigned int n_tables)
{
	struct udp_tunnel_nic *utn;
	unsigned int i;

	utn = kzalloc(struct_size(utn, entries, n_tables), GFP_KERNEL);
	if (!utn)
		return NULL;
	utn->n_tables = n_tables;
	INIT_WORK(&utn->work, udp_tunnel_nic_device_sync_work);
	mutex_init(&utn->lock);

	for (i = 0; i < n_tables; i++) {
		utn->entries[i] = kcalloc(info->tables[i].n_entries,
					  sizeof(*utn->entries[i]), GFP_KERNEL);
		if (!utn->entries[i])
			goto err_free_prev_entries;
	}

	return utn;

err_free_prev_entries:
	while (i--)
		kfree(utn->entries[i]);
	kfree(utn);
	return NULL;
}

static void udp_tunnel_nic_free(struct udp_tunnel_nic *utn)
{
	unsigned int i;

	for (i = 0; i < utn->n_tables; i++)
		kfree(utn->entries[i]);
	kfree(utn);
}

static int udp_tunnel_nic_register(struct net_device *dev)
{
	const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
	struct udp_tunnel_nic_shared_node *node = NULL;
	struct udp_tunnel_nic *utn;
	unsigned int n_tables, i;

	BUILD_BUG_ON(sizeof(utn->missed) * BITS_PER_BYTE <
		     UDP_TUNNEL_NIC_MAX_TABLES);
	/* Expect use count of at most 2 (IPv4, IPv6) per device */
	BUILD_BUG_ON(UDP_TUNNEL_NIC_USE_CNT_MAX <
		     UDP_TUNNEL_NIC_MAX_SHARING_DEVICES * 2);

	/* Check that the driver info is sane */
	if (WARN_ON(!info->set_port != !info->unset_port) ||
	    WARN_ON(!info->set_port == !info->sync_table) ||
	    WARN_ON(!info->tables[0].n_entries))
		return -EINVAL;

	if (WARN_ON(info->shared &&
		    info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY))
		return -EINVAL;

	n_tables = 1;
	for (i = 1; i < UDP_TUNNEL_NIC_MAX_TABLES; i++) {
		if (!info->tables[i].n_entries)
			continue;

		n_tables++;
		if (WARN_ON(!info->tables[i - 1].n_entries))
			return -EINVAL;
	}

	/* Create UDP tunnel state structures */
	if (info->shared) {
		node = kzalloc(sizeof(*node), GFP_KERNEL);
		if (!node)
			return -ENOMEM;

		node->dev = dev;
	}

	if (info->shared && info->shared->udp_tunnel_nic_info) {
		utn = info->shared->udp_tunnel_nic_info;
	} else {
		utn = udp_tunnel_nic_alloc(info, n_tables);
		if (!utn) {
			kfree(node);
			return -ENOMEM;
		}
	}

	if (info->shared) {
		if (!info->shared->udp_tunnel_nic_info) {
			INIT_LIST_HEAD(&info->shared->devices);
			info->shared->udp_tunnel_nic_info = utn;
		}

		list_add_tail(&node->list, &info->shared->devices);
	}

	utn->dev = dev;
	dev_hold(dev);
	dev->udp_tunnel_nic = utn;

	if (!(info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY)) {
		udp_tunnel_nic_lock(dev);
		udp_tunnel_get_rx_info(dev);
		udp_tunnel_nic_unlock(dev);
	}

	return 0;
}

static void
udp_tunnel_nic_unregister(struct net_device *dev, struct udp_tunnel_nic *utn)
{
	const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;

	udp_tunnel_nic_lock(dev);

	/* For a shared table remove this dev from the list of sharing devices
	 * and if there are other devices just detach.
	 */
	if (info->shared) {
		struct udp_tunnel_nic_shared_node *node, *first;

		list_for_each_entry(node, &info->shared->devices, list)
			if (node->dev == dev)
				break;
		if (list_entry_is_head(node, &info->shared->devices, list)) {
			udp_tunnel_nic_unlock(dev);
			return;
		}

		list_del(&node->list);
		kfree(node);

		first = list_first_entry_or_null(&info->shared->devices,
						 typeof(*first), list);
		if (first) {
			udp_tunnel_drop_rx_info(dev);
			utn->dev = first->dev;
			udp_tunnel_nic_unlock(dev);
			goto release_dev;
		}

		info->shared->udp_tunnel_nic_info = NULL;
	}

	/* Flush before we check work, so we don't waste time adding entries
	 * from the work which we will boot immediately.
	 */
	udp_tunnel_nic_flush(dev, utn);
	udp_tunnel_nic_unlock(dev);

	/* Wait for the work to be done using the state, netdev core will
	 * retry unregister until we give up our reference on this device.
	 */
	if (utn->work_pending)
		return;

	udp_tunnel_nic_free(utn);
release_dev:
	dev->udp_tunnel_nic = NULL;
	dev_put(dev);
}

static int
udp_tunnel_nic_netdevice_event(struct notifier_block *unused,
			       unsigned long event, void *ptr)
{
	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
	const struct udp_tunnel_nic_info *info;
	struct udp_tunnel_nic *utn;

	info = dev->udp_tunnel_nic_info;
	if (!info)
		return NOTIFY_DONE;

	if (event == NETDEV_REGISTER) {
		int err;

		err = udp_tunnel_nic_register(dev);
		if (err)
			netdev_WARN(dev, "failed to register for UDP tunnel offloads: %d", err);
		return notifier_from_errno(err);
	}
	/* All other events will need the udp_tunnel_nic state */
	utn = dev->udp_tunnel_nic;
	if (!utn)
		return NOTIFY_DONE;

	if (event == NETDEV_UNREGISTER) {
		udp_tunnel_nic_unregister(dev, utn);
		return NOTIFY_OK;
	}

	/* All other events only matter if NIC has to be programmed open */
	if (!(info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY))
		return NOTIFY_DONE;

	if (event == NETDEV_UP) {
		udp_tunnel_nic_lock(dev);
		WARN_ON(!udp_tunnel_nic_is_empty(dev, utn));
		udp_tunnel_get_rx_info(dev);
		udp_tunnel_nic_unlock(dev);
		return NOTIFY_OK;
	}
	if (event == NETDEV_GOING_DOWN) {
		udp_tunnel_nic_lock(dev);
		udp_tunnel_nic_flush(dev, utn);
		udp_tunnel_nic_unlock(dev);
		return NOTIFY_OK;
	}

	return NOTIFY_DONE;
}

static struct notifier_block udp_tunnel_nic_notifier_block __read_mostly = {
	.notifier_call = udp_tunnel_nic_netdevice_event,
};

static int __init udp_tunnel_nic_init_module(void)
{
	int err;

	udp_tunnel_nic_workqueue = alloc_ordered_workqueue("udp_tunnel_nic", 0);
	if (!udp_tunnel_nic_workqueue)
		return -ENOMEM;

	rtnl_lock();
	udp_tunnel_nic_ops = &__udp_tunnel_nic_ops;
	rtnl_unlock();

	err = register_netdevice_notifier(&udp_tunnel_nic_notifier_block);
	if (err)
		goto err_unset_ops;

	return 0;

err_unset_ops:
	rtnl_lock();
	udp_tunnel_nic_ops = NULL;
	rtnl_unlock();
	destroy_workqueue(udp_tunnel_nic_workqueue);
	return err;
}
late_initcall(udp_tunnel_nic_init_module);

static void __exit udp_tunnel_nic_cleanup_module(void)
{
	unregister_netdevice_notifier(&udp_tunnel_nic_notifier_block);

	rtnl_lock();
	udp_tunnel_nic_ops = NULL;
	rtnl_unlock();

	destroy_workqueue(udp_tunnel_nic_workqueue);
}
module_exit(udp_tunnel_nic_cleanup_module);

MODULE_LICENSE("GPL");