xref: /freebsd/sys/net/if_epair.c (revision 4d846d260e2b9a3d4d0a701462568268cbfe7a5b)
198c230c8SBjoern A. Zeeb /*-
2*4d846d26SWarner Losh  * SPDX-License-Identifier: BSD-2-Clause
3fe267a55SPedro F. Giffuni  *
498c230c8SBjoern A. Zeeb  * Copyright (c) 2008 The FreeBSD Foundation
53dd5760aSBjoern A. Zeeb  * Copyright (c) 2009-2021 Bjoern A. Zeeb <bz@FreeBSD.org>
698c230c8SBjoern A. Zeeb  *
798c230c8SBjoern A. Zeeb  * This software was developed by CK Software GmbH under sponsorship
898c230c8SBjoern A. Zeeb  * from the FreeBSD Foundation.
998c230c8SBjoern A. Zeeb  *
1098c230c8SBjoern A. Zeeb  * Redistribution and use in source and binary forms, with or without
1198c230c8SBjoern A. Zeeb  * modification, are permitted provided that the following conditions
1298c230c8SBjoern A. Zeeb  * are met:
1398c230c8SBjoern A. Zeeb  * 1. Redistributions of source code must retain the above copyright
1498c230c8SBjoern A. Zeeb  * notice, this list of conditions and the following disclaimer.
1598c230c8SBjoern A. Zeeb  * 2. Redistributions in binary form must reproduce the above copyright
1698c230c8SBjoern A. Zeeb  * notice, this list of conditions and the following disclaimer in the
1798c230c8SBjoern A. Zeeb  * documentation and/or other materials provided with the distribution.
1898c230c8SBjoern A. Zeeb  *
1998c230c8SBjoern A. Zeeb  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
2098c230c8SBjoern A. Zeeb  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
2198c230c8SBjoern A. Zeeb  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
2298c230c8SBjoern A. Zeeb  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
2398c230c8SBjoern A. Zeeb  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
2498c230c8SBjoern A. Zeeb  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
2598c230c8SBjoern A. Zeeb  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
2698c230c8SBjoern A. Zeeb  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
2798c230c8SBjoern A. Zeeb  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
2898c230c8SBjoern A. Zeeb  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
2998c230c8SBjoern A. Zeeb  * SUCH DAMAGE.
3098c230c8SBjoern A. Zeeb  */
3198c230c8SBjoern A. Zeeb 
3298c230c8SBjoern A. Zeeb /*
33d0ea4743SBjoern A. Zeeb  * A pair of virtual back-to-back connected ethernet like interfaces
34d0ea4743SBjoern A. Zeeb  * (``two interfaces with a virtual cross-over cable'').
35d0ea4743SBjoern A. Zeeb  *
3698c230c8SBjoern A. Zeeb  * This is mostly intended to be used to provide connectivity between
3798c230c8SBjoern A. Zeeb  * different virtual network stack instances.
3898c230c8SBjoern A. Zeeb  */
3998c230c8SBjoern A. Zeeb 
4098c230c8SBjoern A. Zeeb #include <sys/cdefs.h>
4198c230c8SBjoern A. Zeeb __FBSDID("$FreeBSD$");
4298c230c8SBjoern A. Zeeb 
4324f0bfbaSKristof Provost #include "opt_rss.h"
4452bcdc5bSSantiago Martinez #include "opt_inet.h"
4552bcdc5bSSantiago Martinez #include "opt_inet6.h"
4624f0bfbaSKristof Provost 
4798c230c8SBjoern A. Zeeb #include <sys/param.h>
4829c9b167SMark Johnston #include <sys/bus.h>
4911d41666SLuca Pizzamiglio #include <sys/hash.h>
5029c9b167SMark Johnston #include <sys/interrupt.h>
5111d41666SLuca Pizzamiglio #include <sys/jail.h>
5298c230c8SBjoern A. Zeeb #include <sys/kernel.h>
5311d41666SLuca Pizzamiglio #include <sys/libkern.h>
548ec07310SGleb Smirnoff #include <sys/malloc.h>
5598c230c8SBjoern A. Zeeb #include <sys/mbuf.h>
5698c230c8SBjoern A. Zeeb #include <sys/module.h>
5711d41666SLuca Pizzamiglio #include <sys/proc.h>
5898c230c8SBjoern A. Zeeb #include <sys/queue.h>
5924f0bfbaSKristof Provost #include <sys/sched.h>
60d0ea4743SBjoern A. Zeeb #include <sys/smp.h>
6198c230c8SBjoern A. Zeeb #include <sys/socket.h>
6298c230c8SBjoern A. Zeeb #include <sys/sockio.h>
6324f0bfbaSKristof Provost #include <sys/taskqueue.h>
6498c230c8SBjoern A. Zeeb 
6598c230c8SBjoern A. Zeeb #include <net/bpf.h>
6698c230c8SBjoern A. Zeeb #include <net/ethernet.h>
6798c230c8SBjoern A. Zeeb #include <net/if.h>
6876039bc8SGleb Smirnoff #include <net/if_var.h>
6998c230c8SBjoern A. Zeeb #include <net/if_clone.h>
702dccdd45SMarko Zec #include <net/if_media.h>
7198c230c8SBjoern A. Zeeb #include <net/if_var.h>
722c2b37adSJustin Hibbits #include <net/if_private.h>
7398c230c8SBjoern A. Zeeb #include <net/if_types.h>
7498c230c8SBjoern A. Zeeb #include <net/netisr.h>
7524f0bfbaSKristof Provost #ifdef RSS
7624f0bfbaSKristof Provost #include <net/rss_config.h>
7752bcdc5bSSantiago Martinez #ifdef INET
7824f0bfbaSKristof Provost #include <netinet/in_rss.h>
7952bcdc5bSSantiago Martinez #endif
8052bcdc5bSSantiago Martinez #ifdef INET6
8124f0bfbaSKristof Provost #include <netinet6/in6_rss.h>
8224f0bfbaSKristof Provost #endif
8352bcdc5bSSantiago Martinez #endif
84530c0060SRobert Watson #include <net/vnet.h>
8598c230c8SBjoern A. Zeeb 
8642a58907SGleb Smirnoff static const char epairname[] = "epair";
873dd5760aSBjoern A. Zeeb #define	RXRSIZE	4096	/* Probably overkill by 4-8x. */
88d0ea4743SBjoern A. Zeeb 
8942a58907SGleb Smirnoff static MALLOC_DEFINE(M_EPAIR, epairname,
90d0ea4743SBjoern A. Zeeb     "Pair of virtual cross-over connected Ethernet-like interfaces");
9198c230c8SBjoern A. Zeeb 
925f901c92SAndrew Turner VNET_DEFINE_STATIC(struct if_clone *, epair_cloner);
933c3136b1SHiroki Sato #define	V_epair_cloner	VNET(epair_cloner)
9498c230c8SBjoern A. Zeeb 
953dd5760aSBjoern A. Zeeb static unsigned int next_index = 0;
963dd5760aSBjoern A. Zeeb #define	EPAIR_LOCK_INIT()		mtx_init(&epair_n_index_mtx, "epairidx", \
973dd5760aSBjoern A. Zeeb 					    NULL, MTX_DEF)
983dd5760aSBjoern A. Zeeb #define	EPAIR_LOCK_DESTROY()		mtx_destroy(&epair_n_index_mtx)
993dd5760aSBjoern A. Zeeb #define	EPAIR_LOCK()			mtx_lock(&epair_n_index_mtx)
1003dd5760aSBjoern A. Zeeb #define	EPAIR_UNLOCK()			mtx_unlock(&epair_n_index_mtx)
1013dd5760aSBjoern A. Zeeb 
10224f0bfbaSKristof Provost struct epair_softc;
10324f0bfbaSKristof Provost struct epair_queue {
104df7bbd8cSMark Johnston 	struct mtx		 mtx;
105df7bbd8cSMark Johnston 	struct mbufq		 q;
10624f0bfbaSKristof Provost 	int			 id;
107df7bbd8cSMark Johnston 	enum {
108df7bbd8cSMark Johnston 		EPAIR_QUEUE_IDLE,
109df7bbd8cSMark Johnston 		EPAIR_QUEUE_WAKING,
110df7bbd8cSMark Johnston 		EPAIR_QUEUE_RUNNING,
111df7bbd8cSMark Johnston 	}			 state;
11224f0bfbaSKristof Provost 	struct task		 tx_task;
11324f0bfbaSKristof Provost 	struct epair_softc	*sc;
11424f0bfbaSKristof Provost };
1153dd5760aSBjoern A. Zeeb 
1163dd5760aSBjoern A. Zeeb static struct mtx epair_n_index_mtx;
1173dd5760aSBjoern A. Zeeb struct epair_softc {
1183dd5760aSBjoern A. Zeeb 	struct ifnet		*ifp;		/* This ifp. */
1193dd5760aSBjoern A. Zeeb 	struct ifnet		*oifp;		/* other ifp of pair. */
12024f0bfbaSKristof Provost 	int			 num_queues;
12124f0bfbaSKristof Provost 	struct epair_queue	*queues;
1223dd5760aSBjoern A. Zeeb 	struct ifmedia		 media;		/* Media config (fake). */
1233dd5760aSBjoern A. Zeeb 	STAILQ_ENTRY(epair_softc) entry;
124d0ea4743SBjoern A. Zeeb };
125d0ea4743SBjoern A. Zeeb 
12624f0bfbaSKristof Provost struct epair_tasks_t {
12724f0bfbaSKristof Provost 	int			 tasks;
12824f0bfbaSKristof Provost 	struct taskqueue	 *tq[MAXCPU];
12924f0bfbaSKristof Provost };
13024f0bfbaSKristof Provost 
13124f0bfbaSKristof Provost static struct epair_tasks_t epair_tasks;
13224f0bfbaSKristof Provost 
133d0ea4743SBjoern A. Zeeb static void
13462d2dcafSKristof Provost epair_clear_mbuf(struct mbuf *m)
13562d2dcafSKristof Provost {
136c69ae841SKristof Provost 	M_ASSERTPKTHDR(m);
137c69ae841SKristof Provost 
13862d2dcafSKristof Provost 	/* Remove any CSUM_SND_TAG as ether_input will barf. */
13962d2dcafSKristof Provost 	if (m->m_pkthdr.csum_flags & CSUM_SND_TAG) {
14062d2dcafSKristof Provost 		m_snd_tag_rele(m->m_pkthdr.snd_tag);
14162d2dcafSKristof Provost 		m->m_pkthdr.snd_tag = NULL;
14262d2dcafSKristof Provost 		m->m_pkthdr.csum_flags &= ~CSUM_SND_TAG;
14362d2dcafSKristof Provost 	}
14462d2dcafSKristof Provost 
145c69ae841SKristof Provost 	/* Clear vlan information. */
146c69ae841SKristof Provost 	m->m_flags &= ~M_VLANTAG;
147c69ae841SKristof Provost 	m->m_pkthdr.ether_vtag = 0;
148c69ae841SKristof Provost 
14962d2dcafSKristof Provost 	m_tag_delete_nonpersistent(m);
15062d2dcafSKristof Provost }
15162d2dcafSKristof Provost 
15262d2dcafSKristof Provost static void
15324f0bfbaSKristof Provost epair_tx_start_deferred(void *arg, int pending)
154d0ea4743SBjoern A. Zeeb {
15524f0bfbaSKristof Provost 	struct epair_queue *q = (struct epair_queue *)arg;
156df7bbd8cSMark Johnston 	if_t ifp;
157df7bbd8cSMark Johnston 	struct mbuf *m, *n;
158df7bbd8cSMark Johnston 	bool resched;
159d0ea4743SBjoern A. Zeeb 
160df7bbd8cSMark Johnston 	ifp = q->sc->ifp;
16124f0bfbaSKristof Provost 
162df7bbd8cSMark Johnston 	if_ref(ifp);
163df7bbd8cSMark Johnston 	CURVNET_SET(ifp->if_vnet);
164df7bbd8cSMark Johnston 
165df7bbd8cSMark Johnston 	mtx_lock(&q->mtx);
166df7bbd8cSMark Johnston 	m = mbufq_flush(&q->q);
167df7bbd8cSMark Johnston 	q->state = EPAIR_QUEUE_RUNNING;
168df7bbd8cSMark Johnston 	mtx_unlock(&q->mtx);
169df7bbd8cSMark Johnston 
170df7bbd8cSMark Johnston 	while (m != NULL) {
171df7bbd8cSMark Johnston 		n = STAILQ_NEXT(m, m_stailqpkt);
172df7bbd8cSMark Johnston 		m->m_nextpkt = NULL;
173df7bbd8cSMark Johnston 		if_input(ifp, m);
174df7bbd8cSMark Johnston 		m = n;
175df7bbd8cSMark Johnston 	}
176df7bbd8cSMark Johnston 
177df7bbd8cSMark Johnston 	/*
178df7bbd8cSMark Johnston 	 * Avoid flushing the queue more than once per task.  We can otherwise
179df7bbd8cSMark Johnston 	 * end up starving ourselves in a multi-epair routing configuration.
180df7bbd8cSMark Johnston 	 */
181df7bbd8cSMark Johnston 	mtx_lock(&q->mtx);
182df7bbd8cSMark Johnston 	if (mbufq_len(&q->q) > 0) {
183df7bbd8cSMark Johnston 		resched = true;
184df7bbd8cSMark Johnston 		q->state = EPAIR_QUEUE_WAKING;
185df7bbd8cSMark Johnston 	} else {
186df7bbd8cSMark Johnston 		resched = false;
187df7bbd8cSMark Johnston 		q->state = EPAIR_QUEUE_IDLE;
188df7bbd8cSMark Johnston 	}
189df7bbd8cSMark Johnston 	mtx_unlock(&q->mtx);
190df7bbd8cSMark Johnston 
191df7bbd8cSMark Johnston 	if (resched)
19224f0bfbaSKristof Provost 		taskqueue_enqueue(epair_tasks.tq[q->id], &q->tx_task);
193d0ea4743SBjoern A. Zeeb 
194df7bbd8cSMark Johnston 	CURVNET_RESTORE();
195df7bbd8cSMark Johnston 	if_rele(ifp);
196d0ea4743SBjoern A. Zeeb }
197d0ea4743SBjoern A. Zeeb 
19804a32b80SAlexander V. Chernikov static struct epair_queue *
1998a299958SKristof Provost epair_select_queue(struct epair_softc *sc, struct mbuf *m)
200d0ea4743SBjoern A. Zeeb {
20124f0bfbaSKristof Provost 	uint32_t bucket;
20224f0bfbaSKristof Provost #ifdef RSS
20324f0bfbaSKristof Provost 	struct ether_header *eh;
2048a299958SKristof Provost 	int ret;
205d0ea4743SBjoern A. Zeeb 
20624f0bfbaSKristof Provost 	ret = rss_m2bucket(m, &bucket);
20724f0bfbaSKristof Provost 	if (ret) {
20824f0bfbaSKristof Provost 		/* Actually hash the packet. */
20924f0bfbaSKristof Provost 		eh = mtod(m, struct ether_header *);
21024f0bfbaSKristof Provost 
21124f0bfbaSKristof Provost 		switch (ntohs(eh->ether_type)) {
21252bcdc5bSSantiago Martinez #ifdef INET
21324f0bfbaSKristof Provost 		case ETHERTYPE_IP:
21424f0bfbaSKristof Provost 			rss_soft_m2cpuid_v4(m, 0, &bucket);
21524f0bfbaSKristof Provost 			break;
21652bcdc5bSSantiago Martinez #endif
21752bcdc5bSSantiago Martinez #ifdef INET6
21824f0bfbaSKristof Provost 		case ETHERTYPE_IPV6:
21924f0bfbaSKristof Provost 			rss_soft_m2cpuid_v6(m, 0, &bucket);
22024f0bfbaSKristof Provost 			break;
22152bcdc5bSSantiago Martinez #endif
22224f0bfbaSKristof Provost 		default:
22324f0bfbaSKristof Provost 			bucket = 0;
22424f0bfbaSKristof Provost 			break;
22524f0bfbaSKristof Provost 		}
22624f0bfbaSKristof Provost 	}
22704a32b80SAlexander V. Chernikov 	bucket %= sc->num_queues;
22824f0bfbaSKristof Provost #else
22924f0bfbaSKristof Provost 	bucket = 0;
23024f0bfbaSKristof Provost #endif
23104a32b80SAlexander V. Chernikov 	return (&sc->queues[bucket]);
23204a32b80SAlexander V. Chernikov }
23304a32b80SAlexander V. Chernikov 
23404a32b80SAlexander V. Chernikov static void
23504a32b80SAlexander V. Chernikov epair_prepare_mbuf(struct mbuf *m, struct ifnet *src_ifp)
23604a32b80SAlexander V. Chernikov {
23704a32b80SAlexander V. Chernikov 	M_ASSERTPKTHDR(m);
23804a32b80SAlexander V. Chernikov 	epair_clear_mbuf(m);
23904a32b80SAlexander V. Chernikov 	if_setrcvif(m, src_ifp);
24004a32b80SAlexander V. Chernikov 	M_SETFIB(m, src_ifp->if_fib);
24104a32b80SAlexander V. Chernikov 
24204a32b80SAlexander V. Chernikov 	MPASS(m->m_nextpkt == NULL);
24304a32b80SAlexander V. Chernikov 	MPASS((m->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0);
24404a32b80SAlexander V. Chernikov }
24504a32b80SAlexander V. Chernikov 
24604a32b80SAlexander V. Chernikov static void
24704a32b80SAlexander V. Chernikov epair_menq(struct mbuf *m, struct epair_softc *osc)
24804a32b80SAlexander V. Chernikov {
249df7bbd8cSMark Johnston 	struct epair_queue *q;
25004a32b80SAlexander V. Chernikov 	struct ifnet *ifp, *oifp;
251df7bbd8cSMark Johnston 	int error, len;
25248227d1cSMark Johnston 	bool mcast;
25304a32b80SAlexander V. Chernikov 
25404a32b80SAlexander V. Chernikov 	/*
25504a32b80SAlexander V. Chernikov 	 * I know this looks weird. We pass the "other sc" as we need that one
25604a32b80SAlexander V. Chernikov 	 * and can get both ifps from it as well.
25704a32b80SAlexander V. Chernikov 	 */
25804a32b80SAlexander V. Chernikov 	oifp = osc->ifp;
25904a32b80SAlexander V. Chernikov 	ifp = osc->oifp;
26004a32b80SAlexander V. Chernikov 
26104a32b80SAlexander V. Chernikov 	epair_prepare_mbuf(m, oifp);
26204a32b80SAlexander V. Chernikov 
26304a32b80SAlexander V. Chernikov 	/* Save values as once the mbuf is queued, it's not ours anymore. */
26404a32b80SAlexander V. Chernikov 	len = m->m_pkthdr.len;
26548227d1cSMark Johnston 	mcast = (m->m_flags & (M_BCAST | M_MCAST)) != 0;
26604a32b80SAlexander V. Chernikov 
267df7bbd8cSMark Johnston 	q = epair_select_queue(osc, m);
26824f0bfbaSKristof Provost 
269df7bbd8cSMark Johnston 	mtx_lock(&q->mtx);
270df7bbd8cSMark Johnston 	if (q->state == EPAIR_QUEUE_IDLE) {
271df7bbd8cSMark Johnston 		q->state = EPAIR_QUEUE_WAKING;
272df7bbd8cSMark Johnston 		taskqueue_enqueue(epair_tasks.tq[q->id], &q->tx_task);
2733dd5760aSBjoern A. Zeeb 	}
274df7bbd8cSMark Johnston 	error = mbufq_enqueue(&q->q, m);
275df7bbd8cSMark Johnston 	mtx_unlock(&q->mtx);
276d0ea4743SBjoern A. Zeeb 
277df7bbd8cSMark Johnston 	if (error != 0) {
278df7bbd8cSMark Johnston 		m_freem(m);
279df7bbd8cSMark Johnston 		if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1);
280df7bbd8cSMark Johnston 	} else {
2813dd5760aSBjoern A. Zeeb 		if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
2823dd5760aSBjoern A. Zeeb 		if_inc_counter(ifp, IFCOUNTER_OBYTES, len);
28348227d1cSMark Johnston 		if (mcast)
2843dd5760aSBjoern A. Zeeb 			if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
2853dd5760aSBjoern A. Zeeb 		if_inc_counter(oifp, IFCOUNTER_IPACKETS, 1);
286df7bbd8cSMark Johnston 	}
287d0ea4743SBjoern A. Zeeb }
288d0ea4743SBjoern A. Zeeb 
28998c230c8SBjoern A. Zeeb static void
2903dd5760aSBjoern A. Zeeb epair_start(struct ifnet *ifp)
29198c230c8SBjoern A. Zeeb {
29298c230c8SBjoern A. Zeeb 	struct mbuf *m;
29398c230c8SBjoern A. Zeeb 	struct epair_softc *sc;
29498c230c8SBjoern A. Zeeb 	struct ifnet *oifp;
29598c230c8SBjoern A. Zeeb 
29698c230c8SBjoern A. Zeeb 	/*
29759f35a82SPatrick Kelsey 	 * We get packets here from ether_output via if_handoff()
29875580d58SPatrick Kelsey 	 * and need to put them into the input queue of the oifp
2993dd5760aSBjoern A. Zeeb 	 * and will put the packet into the receive-queue (rxq) of the
3003dd5760aSBjoern A. Zeeb 	 * other interface (oifp) of our pair.
30198c230c8SBjoern A. Zeeb 	 */
3023dd5760aSBjoern A. Zeeb 	sc = ifp->if_softc;
30398c230c8SBjoern A. Zeeb 	oifp = sc->oifp;
30498c230c8SBjoern A. Zeeb 	sc = oifp->if_softc;
30598c230c8SBjoern A. Zeeb 	for (;;) {
30698c230c8SBjoern A. Zeeb 		IFQ_DEQUEUE(&ifp->if_snd, m);
30798c230c8SBjoern A. Zeeb 		if (m == NULL)
30898c230c8SBjoern A. Zeeb 			break;
3093dd5760aSBjoern A. Zeeb 		M_ASSERTPKTHDR(m);
31098c230c8SBjoern A. Zeeb 		BPF_MTAP(ifp, m);
31198c230c8SBjoern A. Zeeb 
3123dd5760aSBjoern A. Zeeb 		/* In case either interface is not usable drop the packet. */
3133dd5760aSBjoern A. Zeeb 		if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
3143dd5760aSBjoern A. Zeeb 		    (ifp->if_flags & IFF_UP) == 0 ||
3153dd5760aSBjoern A. Zeeb 		    (oifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
31698c230c8SBjoern A. Zeeb 		    (oifp->if_flags & IFF_UP) == 0) {
31798c230c8SBjoern A. Zeeb 			m_freem(m);
31898c230c8SBjoern A. Zeeb 			continue;
31998c230c8SBjoern A. Zeeb 		}
32098c230c8SBjoern A. Zeeb 
32104a32b80SAlexander V. Chernikov 		epair_menq(m, sc);
32298c230c8SBjoern A. Zeeb 	}
32398c230c8SBjoern A. Zeeb }
32498c230c8SBjoern A. Zeeb 
32598c230c8SBjoern A. Zeeb static int
3263dd5760aSBjoern A. Zeeb epair_transmit(struct ifnet *ifp, struct mbuf *m)
32798c230c8SBjoern A. Zeeb {
32898c230c8SBjoern A. Zeeb 	struct epair_softc *sc;
32998c230c8SBjoern A. Zeeb 	struct ifnet *oifp;
3302cedfc3fSMateusz Guzik #ifdef ALTQ
3312cedfc3fSMateusz Guzik 	int len;
33248227d1cSMark Johnston 	bool mcast;
3332cedfc3fSMateusz Guzik #endif
33498c230c8SBjoern A. Zeeb 
33598c230c8SBjoern A. Zeeb 	if (m == NULL)
33698c230c8SBjoern A. Zeeb 		return (0);
3373dd5760aSBjoern A. Zeeb 	M_ASSERTPKTHDR(m);
33898c230c8SBjoern A. Zeeb 
33998c230c8SBjoern A. Zeeb 	/*
34098c230c8SBjoern A. Zeeb 	 * We are not going to use the interface en/dequeue mechanism
34198c230c8SBjoern A. Zeeb 	 * on the TX side. We are called from ether_output_frame()
3423dd5760aSBjoern A. Zeeb 	 * and will put the packet into the receive-queue (rxq) of the
3433dd5760aSBjoern A. Zeeb 	 * other interface (oifp) of our pair.
34498c230c8SBjoern A. Zeeb 	 */
34598c230c8SBjoern A. Zeeb 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
34698c230c8SBjoern A. Zeeb 		m_freem(m);
3473dd5760aSBjoern A. Zeeb 		if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
34898c230c8SBjoern A. Zeeb 		return (ENXIO);
34998c230c8SBjoern A. Zeeb 	}
35098c230c8SBjoern A. Zeeb 	if ((ifp->if_flags & IFF_UP) == 0) {
35198c230c8SBjoern A. Zeeb 		m_freem(m);
3523dd5760aSBjoern A. Zeeb 		if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
35398c230c8SBjoern A. Zeeb 		return (ENETDOWN);
35498c230c8SBjoern A. Zeeb 	}
35598c230c8SBjoern A. Zeeb 
35698c230c8SBjoern A. Zeeb 	BPF_MTAP(ifp, m);
35798c230c8SBjoern A. Zeeb 
35898c230c8SBjoern A. Zeeb 	/*
35998c230c8SBjoern A. Zeeb 	 * In case the outgoing interface is not usable,
36098c230c8SBjoern A. Zeeb 	 * drop the packet.
36198c230c8SBjoern A. Zeeb 	 */
3623dd5760aSBjoern A. Zeeb 	sc = ifp->if_softc;
36398c230c8SBjoern A. Zeeb 	oifp = sc->oifp;
36498c230c8SBjoern A. Zeeb 	if ((oifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
36598c230c8SBjoern A. Zeeb 	    (oifp->if_flags & IFF_UP) == 0) {
3663751dddbSGleb Smirnoff 		if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
36798c230c8SBjoern A. Zeeb 		m_freem(m);
36898c230c8SBjoern A. Zeeb 		return (0);
36998c230c8SBjoern A. Zeeb 	}
3702cedfc3fSMateusz Guzik 
3712cedfc3fSMateusz Guzik #ifdef ALTQ
37298c230c8SBjoern A. Zeeb 	len = m->m_pkthdr.len;
37348227d1cSMark Johnston 	mcast = (m->m_flags & (M_BCAST | M_MCAST)) != 0;
37404a32b80SAlexander V. Chernikov 	int error = 0;
37598c230c8SBjoern A. Zeeb 
376a4641f4eSPedro F. Giffuni 	/* Support ALTQ via the classic if_start() path. */
37798c230c8SBjoern A. Zeeb 	IF_LOCK(&ifp->if_snd);
37898c230c8SBjoern A. Zeeb 	if (ALTQ_IS_ENABLED(&ifp->if_snd)) {
37998c230c8SBjoern A. Zeeb 		ALTQ_ENQUEUE(&ifp->if_snd, m, NULL, error);
38098c230c8SBjoern A. Zeeb 		if (error)
3813751dddbSGleb Smirnoff 			if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1);
38298c230c8SBjoern A. Zeeb 		IF_UNLOCK(&ifp->if_snd);
38398c230c8SBjoern A. Zeeb 		if (!error) {
3843751dddbSGleb Smirnoff 			if_inc_counter(ifp, IFCOUNTER_OBYTES, len);
38548227d1cSMark Johnston 			if (mcast)
3863751dddbSGleb Smirnoff 				if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
3873dd5760aSBjoern A. Zeeb 			epair_start(ifp);
38898c230c8SBjoern A. Zeeb 		}
38998c230c8SBjoern A. Zeeb 		return (error);
39098c230c8SBjoern A. Zeeb 	}
39198c230c8SBjoern A. Zeeb 	IF_UNLOCK(&ifp->if_snd);
39298c230c8SBjoern A. Zeeb #endif
39398c230c8SBjoern A. Zeeb 
39404a32b80SAlexander V. Chernikov 	epair_menq(m, oifp->if_softc);
39504a32b80SAlexander V. Chernikov 	return (0);
39698c230c8SBjoern A. Zeeb }
39762d2dcafSKristof Provost 
39812aeeb91SAlexander V. Chernikov static void
39912aeeb91SAlexander V. Chernikov epair_qflush(struct ifnet *ifp __unused)
40012aeeb91SAlexander V. Chernikov {
40112aeeb91SAlexander V. Chernikov }
40212aeeb91SAlexander V. Chernikov 
40398c230c8SBjoern A. Zeeb static int
4042dccdd45SMarko Zec epair_media_change(struct ifnet *ifp __unused)
4052dccdd45SMarko Zec {
4062dccdd45SMarko Zec 
4072dccdd45SMarko Zec 	/* Do nothing. */
4082dccdd45SMarko Zec 	return (0);
4092dccdd45SMarko Zec }
4102dccdd45SMarko Zec 
4112dccdd45SMarko Zec static void
4122dccdd45SMarko Zec epair_media_status(struct ifnet *ifp __unused, struct ifmediareq *imr)
4132dccdd45SMarko Zec {
4142dccdd45SMarko Zec 
4152dccdd45SMarko Zec 	imr->ifm_status = IFM_AVALID | IFM_ACTIVE;
4162dccdd45SMarko Zec 	imr->ifm_active = IFM_ETHER | IFM_10G_T | IFM_FDX;
4172dccdd45SMarko Zec }
4182dccdd45SMarko Zec 
4192dccdd45SMarko Zec static int
42098c230c8SBjoern A. Zeeb epair_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
42198c230c8SBjoern A. Zeeb {
4222dccdd45SMarko Zec 	struct epair_softc *sc;
42398c230c8SBjoern A. Zeeb 	struct ifreq *ifr;
42498c230c8SBjoern A. Zeeb 	int error;
42598c230c8SBjoern A. Zeeb 
42698c230c8SBjoern A. Zeeb 	ifr = (struct ifreq *)data;
42798c230c8SBjoern A. Zeeb 	switch (cmd) {
42898c230c8SBjoern A. Zeeb 	case SIOCSIFFLAGS:
42998c230c8SBjoern A. Zeeb 	case SIOCADDMULTI:
43098c230c8SBjoern A. Zeeb 	case SIOCDELMULTI:
43198c230c8SBjoern A. Zeeb 		error = 0;
43298c230c8SBjoern A. Zeeb 		break;
43398c230c8SBjoern A. Zeeb 
4342dccdd45SMarko Zec 	case SIOCSIFMEDIA:
4352dccdd45SMarko Zec 	case SIOCGIFMEDIA:
4362dccdd45SMarko Zec 		sc = ifp->if_softc;
4372dccdd45SMarko Zec 		error = ifmedia_ioctl(ifp, ifr, &sc->media, cmd);
4382dccdd45SMarko Zec 		break;
4392dccdd45SMarko Zec 
440d0ea4743SBjoern A. Zeeb 	case SIOCSIFMTU:
441d0ea4743SBjoern A. Zeeb 		/* We basically allow all kinds of MTUs. */
442d0ea4743SBjoern A. Zeeb 		ifp->if_mtu = ifr->ifr_mtu;
443d0ea4743SBjoern A. Zeeb 		error = 0;
444d0ea4743SBjoern A. Zeeb 		break;
445d0ea4743SBjoern A. Zeeb 
44698c230c8SBjoern A. Zeeb 	default:
44798c230c8SBjoern A. Zeeb 		/* Let the common ethernet handler process this. */
44898c230c8SBjoern A. Zeeb 		error = ether_ioctl(ifp, cmd, data);
44998c230c8SBjoern A. Zeeb 		break;
45098c230c8SBjoern A. Zeeb 	}
45198c230c8SBjoern A. Zeeb 
45298c230c8SBjoern A. Zeeb 	return (error);
45398c230c8SBjoern A. Zeeb }
45498c230c8SBjoern A. Zeeb 
45598c230c8SBjoern A. Zeeb static void
45698c230c8SBjoern A. Zeeb epair_init(void *dummy __unused)
45798c230c8SBjoern A. Zeeb {
45898c230c8SBjoern A. Zeeb }
45998c230c8SBjoern A. Zeeb 
46098c230c8SBjoern A. Zeeb /*
46198c230c8SBjoern A. Zeeb  * Interface cloning functions.
46298c230c8SBjoern A. Zeeb  * We use our private ones so that we can create/destroy our secondary
46398c230c8SBjoern A. Zeeb  * device along with the primary one.
46498c230c8SBjoern A. Zeeb  */
46598c230c8SBjoern A. Zeeb static int
46698c230c8SBjoern A. Zeeb epair_clone_match(struct if_clone *ifc, const char *name)
46798c230c8SBjoern A. Zeeb {
46898c230c8SBjoern A. Zeeb 	const char *cp;
46998c230c8SBjoern A. Zeeb 
47098c230c8SBjoern A. Zeeb 	/*
47198c230c8SBjoern A. Zeeb 	 * Our base name is epair.
47298c230c8SBjoern A. Zeeb 	 * Our interfaces will be named epair<n>[ab].
47398c230c8SBjoern A. Zeeb 	 * So accept anything of the following list:
47498c230c8SBjoern A. Zeeb 	 * - epair
47598c230c8SBjoern A. Zeeb 	 * - epair<n>
47698c230c8SBjoern A. Zeeb 	 * but not the epair<n>[ab] versions.
47798c230c8SBjoern A. Zeeb 	 */
47842a58907SGleb Smirnoff 	if (strncmp(epairname, name, sizeof(epairname)-1) != 0)
47998c230c8SBjoern A. Zeeb 		return (0);
48098c230c8SBjoern A. Zeeb 
48142a58907SGleb Smirnoff 	for (cp = name + sizeof(epairname) - 1; *cp != '\0'; cp++) {
48298c230c8SBjoern A. Zeeb 		if (*cp < '0' || *cp > '9')
48398c230c8SBjoern A. Zeeb 			return (0);
48498c230c8SBjoern A. Zeeb 	}
48598c230c8SBjoern A. Zeeb 
48698c230c8SBjoern A. Zeeb 	return (1);
48798c230c8SBjoern A. Zeeb }
48898c230c8SBjoern A. Zeeb 
489b02fd8b7SKristof Provost static void
490b02fd8b7SKristof Provost epair_clone_add(struct if_clone *ifc, struct epair_softc *scb)
491b02fd8b7SKristof Provost {
492b02fd8b7SKristof Provost 	struct ifnet *ifp;
493b02fd8b7SKristof Provost 	uint8_t eaddr[ETHER_ADDR_LEN];	/* 00:00:00:00:00:00 */
494b02fd8b7SKristof Provost 
495b02fd8b7SKristof Provost 	ifp = scb->ifp;
496b02fd8b7SKristof Provost 	/* Copy epairNa etheraddr and change the last byte. */
497b02fd8b7SKristof Provost 	memcpy(eaddr, scb->oifp->if_hw_addr, ETHER_ADDR_LEN);
498b02fd8b7SKristof Provost 	eaddr[5] = 0x0b;
499b02fd8b7SKristof Provost 	ether_ifattach(ifp, eaddr);
500b02fd8b7SKristof Provost 
501b02fd8b7SKristof Provost 	if_clone_addif(ifc, ifp);
502b02fd8b7SKristof Provost }
503b02fd8b7SKristof Provost 
50412aeeb91SAlexander V. Chernikov static struct epair_softc *
50512aeeb91SAlexander V. Chernikov epair_alloc_sc(struct if_clone *ifc)
50612aeeb91SAlexander V. Chernikov {
50712aeeb91SAlexander V. Chernikov 	struct epair_softc *sc;
50812aeeb91SAlexander V. Chernikov 
50912aeeb91SAlexander V. Chernikov 	struct ifnet *ifp = if_alloc(IFT_ETHER);
51012aeeb91SAlexander V. Chernikov 	if (ifp == NULL)
51112aeeb91SAlexander V. Chernikov 		return (NULL);
51212aeeb91SAlexander V. Chernikov 
51312aeeb91SAlexander V. Chernikov 	sc = malloc(sizeof(struct epair_softc), M_EPAIR, M_WAITOK | M_ZERO);
51412aeeb91SAlexander V. Chernikov 	sc->ifp = ifp;
51512aeeb91SAlexander V. Chernikov 	sc->num_queues = epair_tasks.tasks;
51612aeeb91SAlexander V. Chernikov 	sc->queues = mallocarray(sc->num_queues, sizeof(struct epair_queue),
51712aeeb91SAlexander V. Chernikov 	    M_EPAIR, M_WAITOK);
51812aeeb91SAlexander V. Chernikov 	for (int i = 0; i < sc->num_queues; i++) {
51912aeeb91SAlexander V. Chernikov 		struct epair_queue *q = &sc->queues[i];
52012aeeb91SAlexander V. Chernikov 		q->id = i;
521df7bbd8cSMark Johnston 		q->state = EPAIR_QUEUE_IDLE;
522df7bbd8cSMark Johnston 		mtx_init(&q->mtx, "epairq", NULL, MTX_DEF | MTX_NEW);
523df7bbd8cSMark Johnston 		mbufq_init(&q->q, RXRSIZE);
52412aeeb91SAlexander V. Chernikov 		q->sc = sc;
52512aeeb91SAlexander V. Chernikov 		NET_TASK_INIT(&q->tx_task, 0, epair_tx_start_deferred, q);
52612aeeb91SAlexander V. Chernikov 	}
52712aeeb91SAlexander V. Chernikov 
52812aeeb91SAlexander V. Chernikov 	/* Initialise pseudo media types. */
52912aeeb91SAlexander V. Chernikov 	ifmedia_init(&sc->media, 0, epair_media_change, epair_media_status);
53012aeeb91SAlexander V. Chernikov 	ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_T, 0, NULL);
53112aeeb91SAlexander V. Chernikov 	ifmedia_set(&sc->media, IFM_ETHER | IFM_10G_T);
53212aeeb91SAlexander V. Chernikov 
53312aeeb91SAlexander V. Chernikov 	return (sc);
53412aeeb91SAlexander V. Chernikov }
53512aeeb91SAlexander V. Chernikov 
53612aeeb91SAlexander V. Chernikov static void
53712aeeb91SAlexander V. Chernikov epair_setup_ifp(struct epair_softc *sc, char *name, int unit)
53812aeeb91SAlexander V. Chernikov {
53912aeeb91SAlexander V. Chernikov 	struct ifnet *ifp = sc->ifp;
54012aeeb91SAlexander V. Chernikov 
54112aeeb91SAlexander V. Chernikov 	ifp->if_softc = sc;
54212aeeb91SAlexander V. Chernikov 	strlcpy(ifp->if_xname, name, IFNAMSIZ);
54312aeeb91SAlexander V. Chernikov 	ifp->if_dname = epairname;
54412aeeb91SAlexander V. Chernikov 	ifp->if_dunit = unit;
54512aeeb91SAlexander V. Chernikov 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
54612aeeb91SAlexander V. Chernikov 	ifp->if_capabilities = IFCAP_VLAN_MTU;
54712aeeb91SAlexander V. Chernikov 	ifp->if_capenable = IFCAP_VLAN_MTU;
54812aeeb91SAlexander V. Chernikov 	ifp->if_transmit = epair_transmit;
54912aeeb91SAlexander V. Chernikov 	ifp->if_qflush = epair_qflush;
55012aeeb91SAlexander V. Chernikov 	ifp->if_start = epair_start;
55112aeeb91SAlexander V. Chernikov 	ifp->if_ioctl = epair_ioctl;
55212aeeb91SAlexander V. Chernikov 	ifp->if_init  = epair_init;
55312aeeb91SAlexander V. Chernikov 	if_setsendqlen(ifp, ifqmaxlen);
55412aeeb91SAlexander V. Chernikov 	if_setsendqready(ifp);
55512aeeb91SAlexander V. Chernikov 
55612aeeb91SAlexander V. Chernikov 	ifp->if_baudrate = IF_Gbps(10);	/* arbitrary maximum */
55712aeeb91SAlexander V. Chernikov }
55812aeeb91SAlexander V. Chernikov 
55912aeeb91SAlexander V. Chernikov static void
56012aeeb91SAlexander V. Chernikov epair_generate_mac(struct epair_softc *sc, uint8_t *eaddr)
56112aeeb91SAlexander V. Chernikov {
56212aeeb91SAlexander V. Chernikov 	uint32_t key[3];
56312aeeb91SAlexander V. Chernikov 	uint32_t hash;
56412aeeb91SAlexander V. Chernikov 	uint64_t hostid;
56512aeeb91SAlexander V. Chernikov 
56612aeeb91SAlexander V. Chernikov 	EPAIR_LOCK();
56712aeeb91SAlexander V. Chernikov #ifdef SMP
56812aeeb91SAlexander V. Chernikov 	/* Get an approximate distribution. */
56912aeeb91SAlexander V. Chernikov 	hash = next_index % mp_ncpus;
57012aeeb91SAlexander V. Chernikov #else
57112aeeb91SAlexander V. Chernikov 	hash = 0;
57212aeeb91SAlexander V. Chernikov #endif
57312aeeb91SAlexander V. Chernikov 	EPAIR_UNLOCK();
57412aeeb91SAlexander V. Chernikov 
57512aeeb91SAlexander V. Chernikov 	/*
57612aeeb91SAlexander V. Chernikov 	 * Calculate the etheraddr hashing the hostid and the
57712aeeb91SAlexander V. Chernikov 	 * interface index. The result would be hopefully unique.
57812aeeb91SAlexander V. Chernikov 	 * Note that the "a" component of an epair instance may get moved
57912aeeb91SAlexander V. Chernikov 	 * to a different VNET after creation. In that case its index
58012aeeb91SAlexander V. Chernikov 	 * will be freed and the index can get reused by new epair instance.
58112aeeb91SAlexander V. Chernikov 	 * Make sure we do not create same etheraddr again.
58212aeeb91SAlexander V. Chernikov 	 */
58312aeeb91SAlexander V. Chernikov 	getcredhostid(curthread->td_ucred, (unsigned long *)&hostid);
58412aeeb91SAlexander V. Chernikov 	if (hostid == 0)
58512aeeb91SAlexander V. Chernikov 		arc4rand(&hostid, sizeof(hostid), 0);
58612aeeb91SAlexander V. Chernikov 
58712aeeb91SAlexander V. Chernikov 	struct ifnet *ifp = sc->ifp;
58812aeeb91SAlexander V. Chernikov 	EPAIR_LOCK();
58912aeeb91SAlexander V. Chernikov 	if (ifp->if_index > next_index)
59012aeeb91SAlexander V. Chernikov 		next_index = ifp->if_index;
59112aeeb91SAlexander V. Chernikov 	else
59212aeeb91SAlexander V. Chernikov 		next_index++;
59312aeeb91SAlexander V. Chernikov 
59412aeeb91SAlexander V. Chernikov 	key[0] = (uint32_t)next_index;
59512aeeb91SAlexander V. Chernikov 	EPAIR_UNLOCK();
59612aeeb91SAlexander V. Chernikov 	key[1] = (uint32_t)(hostid & 0xffffffff);
59712aeeb91SAlexander V. Chernikov 	key[2] = (uint32_t)((hostid >> 32) & 0xfffffffff);
59812aeeb91SAlexander V. Chernikov 	hash = jenkins_hash32(key, 3, 0);
59912aeeb91SAlexander V. Chernikov 
60012aeeb91SAlexander V. Chernikov 	eaddr[0] = 0x02;
60112aeeb91SAlexander V. Chernikov 	memcpy(&eaddr[1], &hash, 4);
60212aeeb91SAlexander V. Chernikov 	eaddr[5] = 0x0a;
60312aeeb91SAlexander V. Chernikov }
60412aeeb91SAlexander V. Chernikov 
60512aeeb91SAlexander V. Chernikov static void
60612aeeb91SAlexander V. Chernikov epair_free_sc(struct epair_softc *sc)
60712aeeb91SAlexander V. Chernikov {
60812aeeb91SAlexander V. Chernikov 	if (sc == NULL)
60912aeeb91SAlexander V. Chernikov 		return;
61012aeeb91SAlexander V. Chernikov 
61112aeeb91SAlexander V. Chernikov 	if_free(sc->ifp);
61212aeeb91SAlexander V. Chernikov 	ifmedia_removeall(&sc->media);
61312aeeb91SAlexander V. Chernikov 	for (int i = 0; i < sc->num_queues; i++) {
61412aeeb91SAlexander V. Chernikov 		struct epair_queue *q = &sc->queues[i];
615df7bbd8cSMark Johnston 		mtx_destroy(&q->mtx);
61612aeeb91SAlexander V. Chernikov 	}
61712aeeb91SAlexander V. Chernikov 	free(sc->queues, M_EPAIR);
61812aeeb91SAlexander V. Chernikov 	free(sc, M_EPAIR);
61912aeeb91SAlexander V. Chernikov }
62012aeeb91SAlexander V. Chernikov 
62104a32b80SAlexander V. Chernikov static void
62204a32b80SAlexander V. Chernikov epair_set_state(struct ifnet *ifp, bool running)
62398c230c8SBjoern A. Zeeb {
62404a32b80SAlexander V. Chernikov 	if (running) {
62504a32b80SAlexander V. Chernikov 		ifp->if_drv_flags |= IFF_DRV_RUNNING;
62604a32b80SAlexander V. Chernikov 		if_link_state_change(ifp, LINK_STATE_UP);
62704a32b80SAlexander V. Chernikov 	} else {
62804a32b80SAlexander V. Chernikov 		if_link_state_change(ifp, LINK_STATE_DOWN);
62904a32b80SAlexander V. Chernikov 		ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
63004a32b80SAlexander V. Chernikov 	}
63104a32b80SAlexander V. Chernikov }
63204a32b80SAlexander V. Chernikov 
63304a32b80SAlexander V. Chernikov static int
63404a32b80SAlexander V. Chernikov epair_handle_unit(struct if_clone *ifc, char *name, size_t len, int *punit)
63504a32b80SAlexander V. Chernikov {
63604a32b80SAlexander V. Chernikov 	int error = 0, unit, wildcard;
63798c230c8SBjoern A. Zeeb 	char *dp;
63898c230c8SBjoern A. Zeeb 
63998c230c8SBjoern A. Zeeb 	/* Try to see if a special unit was requested. */
64098c230c8SBjoern A. Zeeb 	error = ifc_name2unit(name, &unit);
64198c230c8SBjoern A. Zeeb 	if (error != 0)
64298c230c8SBjoern A. Zeeb 		return (error);
64398c230c8SBjoern A. Zeeb 	wildcard = (unit < 0);
64498c230c8SBjoern A. Zeeb 
64598c230c8SBjoern A. Zeeb 	error = ifc_alloc_unit(ifc, &unit);
64698c230c8SBjoern A. Zeeb 	if (error != 0)
64798c230c8SBjoern A. Zeeb 		return (error);
64898c230c8SBjoern A. Zeeb 
64998c230c8SBjoern A. Zeeb 	/*
65098c230c8SBjoern A. Zeeb 	 * If no unit had been given, we need to adjust the ifName.
65198c230c8SBjoern A. Zeeb 	 * Also make sure there is space for our extra [ab] suffix.
65298c230c8SBjoern A. Zeeb 	 */
65398c230c8SBjoern A. Zeeb 	for (dp = name; *dp != '\0'; dp++);
65498c230c8SBjoern A. Zeeb 	if (wildcard) {
65504a32b80SAlexander V. Chernikov 		int slen = snprintf(dp, len - (dp - name), "%d", unit);
65604a32b80SAlexander V. Chernikov 		if (slen > len - (dp - name) - 1) {
65798c230c8SBjoern A. Zeeb 			/* ifName too long. */
65804a32b80SAlexander V. Chernikov 			error = ENOSPC;
65904a32b80SAlexander V. Chernikov 			goto done;
66098c230c8SBjoern A. Zeeb 		}
66104a32b80SAlexander V. Chernikov 		dp += slen;
66298c230c8SBjoern A. Zeeb 	}
66398c230c8SBjoern A. Zeeb 	if (len - (dp - name) - 1 < 1) {
66498c230c8SBjoern A. Zeeb 		/* No space left for our [ab] suffix. */
66504a32b80SAlexander V. Chernikov 		error = ENOSPC;
66604a32b80SAlexander V. Chernikov 		goto done;
66798c230c8SBjoern A. Zeeb 	}
6683c3136b1SHiroki Sato 	*dp = 'b';
66998c230c8SBjoern A. Zeeb 	/* Must not change dp so we can replace 'a' by 'b' later. */
67098c230c8SBjoern A. Zeeb 	*(dp+1) = '\0';
67198c230c8SBjoern A. Zeeb 
6723c3136b1SHiroki Sato 	/* Check if 'a' and 'b' interfaces already exist. */
67304a32b80SAlexander V. Chernikov 	if (ifunit(name) != NULL) {
67404a32b80SAlexander V. Chernikov 		error = EEXIST;
67504a32b80SAlexander V. Chernikov 		goto done;
67604a32b80SAlexander V. Chernikov 	}
67704a32b80SAlexander V. Chernikov 
6783c3136b1SHiroki Sato 	*dp = 'a';
67904a32b80SAlexander V. Chernikov 	if (ifunit(name) != NULL) {
68004a32b80SAlexander V. Chernikov 		error = EEXIST;
68104a32b80SAlexander V. Chernikov 		goto done;
68204a32b80SAlexander V. Chernikov 	}
68304a32b80SAlexander V. Chernikov 	*punit = unit;
68404a32b80SAlexander V. Chernikov done:
68504a32b80SAlexander V. Chernikov 	if (error != 0)
68604a32b80SAlexander V. Chernikov 		ifc_free_unit(ifc, unit);
68704a32b80SAlexander V. Chernikov 
68804a32b80SAlexander V. Chernikov 	return (error);
68904a32b80SAlexander V. Chernikov }
69004a32b80SAlexander V. Chernikov 
69104a32b80SAlexander V. Chernikov static int
69204a32b80SAlexander V. Chernikov epair_clone_create(struct if_clone *ifc, char *name, size_t len,
69304a32b80SAlexander V. Chernikov     struct ifc_data *ifd, struct ifnet **ifpp)
69404a32b80SAlexander V. Chernikov {
69504a32b80SAlexander V. Chernikov 	struct epair_softc *sca, *scb;
69604a32b80SAlexander V. Chernikov 	struct ifnet *ifp;
69704a32b80SAlexander V. Chernikov 	char *dp;
69804a32b80SAlexander V. Chernikov 	int error, unit;
69904a32b80SAlexander V. Chernikov 	uint8_t eaddr[ETHER_ADDR_LEN];	/* 00:00:00:00:00:00 */
70004a32b80SAlexander V. Chernikov 
70104a32b80SAlexander V. Chernikov 	error = epair_handle_unit(ifc, name, len, &unit);
70204a32b80SAlexander V. Chernikov 	if (error != 0)
70304a32b80SAlexander V. Chernikov 		return (error);
7043c3136b1SHiroki Sato 
70598c230c8SBjoern A. Zeeb 	/* Allocate memory for both [ab] interfaces */
70612aeeb91SAlexander V. Chernikov 	sca = epair_alloc_sc(ifc);
70712aeeb91SAlexander V. Chernikov 	scb = epair_alloc_sc(ifc);
70812aeeb91SAlexander V. Chernikov 	if (sca == NULL || scb == NULL) {
70912aeeb91SAlexander V. Chernikov 		epair_free_sc(sca);
71012aeeb91SAlexander V. Chernikov 		epair_free_sc(scb);
71198c230c8SBjoern A. Zeeb 		ifc_free_unit(ifc, unit);
71298c230c8SBjoern A. Zeeb 		return (ENOSPC);
71398c230c8SBjoern A. Zeeb 	}
71498c230c8SBjoern A. Zeeb 
71598c230c8SBjoern A. Zeeb 	/*
71698c230c8SBjoern A. Zeeb 	 * Cross-reference the interfaces so we will be able to free both.
71798c230c8SBjoern A. Zeeb 	 */
71898c230c8SBjoern A. Zeeb 	sca->oifp = scb->ifp;
71998c230c8SBjoern A. Zeeb 	scb->oifp = sca->ifp;
72098c230c8SBjoern A. Zeeb 
72198c230c8SBjoern A. Zeeb 	/* Finish initialization of interface <n>a. */
72298c230c8SBjoern A. Zeeb 	ifp = sca->ifp;
72312aeeb91SAlexander V. Chernikov 	epair_setup_ifp(sca, name, unit);
72412aeeb91SAlexander V. Chernikov 	epair_generate_mac(sca, eaddr);
72511d41666SLuca Pizzamiglio 
72698c230c8SBjoern A. Zeeb 	ether_ifattach(ifp, eaddr);
72798c230c8SBjoern A. Zeeb 
72898c230c8SBjoern A. Zeeb 	/* Swap the name and finish initialization of interface <n>b. */
72904a32b80SAlexander V. Chernikov 	dp = name + strlen(name) - 1;
73098c230c8SBjoern A. Zeeb 	*dp = 'b';
73198c230c8SBjoern A. Zeeb 
73212aeeb91SAlexander V. Chernikov 	epair_setup_ifp(scb, name, unit);
73312aeeb91SAlexander V. Chernikov 
73498c230c8SBjoern A. Zeeb 	ifp = scb->ifp;
73598c230c8SBjoern A. Zeeb 	/* We need to play some tricks here for the second interface. */
73642a58907SGleb Smirnoff 	strlcpy(name, epairname, len);
737b02fd8b7SKristof Provost 	/* Correctly set the name for the cloner list. */
738b02fd8b7SKristof Provost 	strlcpy(name, scb->ifp->if_xname, len);
739b02fd8b7SKristof Provost 
74012aeeb91SAlexander V. Chernikov 	epair_clone_add(ifc, scb);
74198c230c8SBjoern A. Zeeb 
74298c230c8SBjoern A. Zeeb 	/*
74398c230c8SBjoern A. Zeeb 	 * Restore name to <n>a as the ifp for this will go into the
74498c230c8SBjoern A. Zeeb 	 * cloner list for the initial call.
74598c230c8SBjoern A. Zeeb 	 */
74698c230c8SBjoern A. Zeeb 	strlcpy(name, sca->ifp->if_xname, len);
74798c230c8SBjoern A. Zeeb 
74898c230c8SBjoern A. Zeeb 	/* Tell the world, that we are ready to rock. */
74904a32b80SAlexander V. Chernikov 	epair_set_state(sca->ifp, true);
75004a32b80SAlexander V. Chernikov 	epair_set_state(scb->ifp, true);
75198c230c8SBjoern A. Zeeb 
75291ebcbe0SAlexander V. Chernikov 	*ifpp = sca->ifp;
75391ebcbe0SAlexander V. Chernikov 
75498c230c8SBjoern A. Zeeb 	return (0);
75598c230c8SBjoern A. Zeeb }
75698c230c8SBjoern A. Zeeb 
7573dd5760aSBjoern A. Zeeb static void
7583dd5760aSBjoern A. Zeeb epair_drain_rings(struct epair_softc *sc)
7593dd5760aSBjoern A. Zeeb {
76024f0bfbaSKristof Provost 	for (int i = 0; i < sc->num_queues; i++) {
761df7bbd8cSMark Johnston 		struct epair_queue *q;
762df7bbd8cSMark Johnston 		struct mbuf *m, *n;
763df7bbd8cSMark Johnston 
764df7bbd8cSMark Johnston 		q = &sc->queues[i];
765df7bbd8cSMark Johnston 		mtx_lock(&q->mtx);
766df7bbd8cSMark Johnston 		m = mbufq_flush(&q->q);
767df7bbd8cSMark Johnston 		mtx_unlock(&q->mtx);
768df7bbd8cSMark Johnston 
769df7bbd8cSMark Johnston 		for (; m != NULL; m = n) {
770df7bbd8cSMark Johnston 			n = m->m_nextpkt;
7713dd5760aSBjoern A. Zeeb 			m_freem(m);
7723dd5760aSBjoern A. Zeeb 		}
7733dd5760aSBjoern A. Zeeb 	}
77424f0bfbaSKristof Provost }
7753dd5760aSBjoern A. Zeeb 
77698c230c8SBjoern A. Zeeb static int
77791ebcbe0SAlexander V. Chernikov epair_clone_destroy(struct if_clone *ifc, struct ifnet *ifp, uint32_t flags)
77898c230c8SBjoern A. Zeeb {
77998c230c8SBjoern A. Zeeb 	struct ifnet *oifp;
78098c230c8SBjoern A. Zeeb 	struct epair_softc *sca, *scb;
78198c230c8SBjoern A. Zeeb 	int unit, error;
78298c230c8SBjoern A. Zeeb 
78398c230c8SBjoern A. Zeeb 	/*
78498c230c8SBjoern A. Zeeb 	 * In case we called into if_clone_destroyif() ourselves
78598c230c8SBjoern A. Zeeb 	 * again to remove the second interface, the softc will be
78698c230c8SBjoern A. Zeeb 	 * NULL. In that case so not do anything but return success.
78798c230c8SBjoern A. Zeeb 	 */
78898c230c8SBjoern A. Zeeb 	if (ifp->if_softc == NULL)
78998c230c8SBjoern A. Zeeb 		return (0);
79098c230c8SBjoern A. Zeeb 
79198c230c8SBjoern A. Zeeb 	unit = ifp->if_dunit;
79298c230c8SBjoern A. Zeeb 	sca = ifp->if_softc;
79398c230c8SBjoern A. Zeeb 	oifp = sca->oifp;
79498c230c8SBjoern A. Zeeb 	scb = oifp->if_softc;
79598c230c8SBjoern A. Zeeb 
7963dd5760aSBjoern A. Zeeb 	/* Frist get the interfaces down and detached. */
79704a32b80SAlexander V. Chernikov 	epair_set_state(ifp, false);
79804a32b80SAlexander V. Chernikov 	epair_set_state(oifp, false);
79998c230c8SBjoern A. Zeeb 
8003dd5760aSBjoern A. Zeeb 	ether_ifdetach(ifp);
8017edc3d88SMikolaj Golub 	ether_ifdetach(oifp);
8023dd5760aSBjoern A. Zeeb 
8033dd5760aSBjoern A. Zeeb 	/* Third free any queued packets and all the resources. */
8043dd5760aSBjoern A. Zeeb 	CURVNET_SET_QUIET(oifp->if_vnet);
8053dd5760aSBjoern A. Zeeb 	epair_drain_rings(scb);
80698c230c8SBjoern A. Zeeb 	oifp->if_softc = NULL;
80798c230c8SBjoern A. Zeeb 	error = if_clone_destroyif(ifc, oifp);
80898c230c8SBjoern A. Zeeb 	if (error)
80998c230c8SBjoern A. Zeeb 		panic("%s: if_clone_destroyif() for our 2nd iface failed: %d",
81098c230c8SBjoern A. Zeeb 		    __func__, error);
81112aeeb91SAlexander V. Chernikov 	epair_free_sc(scb);
8127edc3d88SMikolaj Golub 	CURVNET_RESTORE();
8137edc3d88SMikolaj Golub 
8143dd5760aSBjoern A. Zeeb 	epair_drain_rings(sca);
81512aeeb91SAlexander V. Chernikov 	epair_free_sc(sca);
8163dd5760aSBjoern A. Zeeb 
8173dd5760aSBjoern A. Zeeb 	/* Last free the cloner unit. */
81898c230c8SBjoern A. Zeeb 	ifc_free_unit(ifc, unit);
81998c230c8SBjoern A. Zeeb 
82098c230c8SBjoern A. Zeeb 	return (0);
82198c230c8SBjoern A. Zeeb }
82298c230c8SBjoern A. Zeeb 
8233c3136b1SHiroki Sato static void
8243c3136b1SHiroki Sato vnet_epair_init(const void *unused __unused)
8253c3136b1SHiroki Sato {
82691ebcbe0SAlexander V. Chernikov 	struct if_clone_addreq req = {
82791ebcbe0SAlexander V. Chernikov 		.match_f = epair_clone_match,
82891ebcbe0SAlexander V. Chernikov 		.create_f = epair_clone_create,
82991ebcbe0SAlexander V. Chernikov 		.destroy_f = epair_clone_destroy,
83091ebcbe0SAlexander V. Chernikov 	};
83191ebcbe0SAlexander V. Chernikov 	V_epair_cloner = ifc_attach_cloner(epairname, &req);
8323c3136b1SHiroki Sato }
83389856f7eSBjoern A. Zeeb VNET_SYSINIT(vnet_epair_init, SI_SUB_PSEUDO, SI_ORDER_ANY,
8343c3136b1SHiroki Sato     vnet_epair_init, NULL);
8353c3136b1SHiroki Sato 
8363c3136b1SHiroki Sato static void
8373c3136b1SHiroki Sato vnet_epair_uninit(const void *unused __unused)
8383c3136b1SHiroki Sato {
8393c3136b1SHiroki Sato 
84091ebcbe0SAlexander V. Chernikov 	ifc_detach_cloner(V_epair_cloner);
8413c3136b1SHiroki Sato }
84289856f7eSBjoern A. Zeeb VNET_SYSUNINIT(vnet_epair_uninit, SI_SUB_INIT_IF, SI_ORDER_ANY,
8433c3136b1SHiroki Sato     vnet_epair_uninit, NULL);
8443c3136b1SHiroki Sato 
84598c230c8SBjoern A. Zeeb static int
8467442b632SLi-Wen Hsu epair_mod_init(void)
84724f0bfbaSKristof Provost {
84824f0bfbaSKristof Provost 	char name[32];
84924f0bfbaSKristof Provost 	epair_tasks.tasks = 0;
85024f0bfbaSKristof Provost 
85124f0bfbaSKristof Provost #ifdef RSS
85224f0bfbaSKristof Provost 	int cpu;
85324f0bfbaSKristof Provost 
85424f0bfbaSKristof Provost 	CPU_FOREACH(cpu) {
85524f0bfbaSKristof Provost 		cpuset_t cpu_mask;
85624f0bfbaSKristof Provost 
85724f0bfbaSKristof Provost 		/* Pin to this CPU so we get appropriate NUMA allocations. */
85824f0bfbaSKristof Provost 		thread_lock(curthread);
85924f0bfbaSKristof Provost 		sched_bind(curthread, cpu);
86024f0bfbaSKristof Provost 		thread_unlock(curthread);
86124f0bfbaSKristof Provost 
86224f0bfbaSKristof Provost 		snprintf(name, sizeof(name), "epair_task_%d", cpu);
86324f0bfbaSKristof Provost 
86424f0bfbaSKristof Provost 		epair_tasks.tq[cpu] = taskqueue_create(name, M_WAITOK,
86524f0bfbaSKristof Provost 		    taskqueue_thread_enqueue,
86624f0bfbaSKristof Provost 		    &epair_tasks.tq[cpu]);
86724f0bfbaSKristof Provost 		CPU_SETOF(cpu, &cpu_mask);
86824f0bfbaSKristof Provost 		taskqueue_start_threads_cpuset(&epair_tasks.tq[cpu], 1, PI_NET,
86924f0bfbaSKristof Provost 		    &cpu_mask, "%s", name);
87024f0bfbaSKristof Provost 
87124f0bfbaSKristof Provost 		epair_tasks.tasks++;
87224f0bfbaSKristof Provost 	}
873cbbce423SKristof Provost 	thread_lock(curthread);
874cbbce423SKristof Provost 	sched_unbind(curthread);
875cbbce423SKristof Provost 	thread_unlock(curthread);
87624f0bfbaSKristof Provost #else
87724f0bfbaSKristof Provost 	snprintf(name, sizeof(name), "epair_task");
87824f0bfbaSKristof Provost 
87924f0bfbaSKristof Provost 	epair_tasks.tq[0] = taskqueue_create(name, M_WAITOK,
88024f0bfbaSKristof Provost 	    taskqueue_thread_enqueue,
88124f0bfbaSKristof Provost 	    &epair_tasks.tq[0]);
88224f0bfbaSKristof Provost 	taskqueue_start_threads(&epair_tasks.tq[0], 1, PI_NET, "%s", name);
88324f0bfbaSKristof Provost 
88424f0bfbaSKristof Provost 	epair_tasks.tasks = 1;
88524f0bfbaSKristof Provost #endif
88624f0bfbaSKristof Provost 
88724f0bfbaSKristof Provost 	return (0);
88824f0bfbaSKristof Provost }
88924f0bfbaSKristof Provost 
89024f0bfbaSKristof Provost static void
8917442b632SLi-Wen Hsu epair_mod_cleanup(void)
89224f0bfbaSKristof Provost {
89324f0bfbaSKristof Provost 
89424f0bfbaSKristof Provost 	for (int i = 0; i < epair_tasks.tasks; i++) {
89524f0bfbaSKristof Provost 		taskqueue_drain_all(epair_tasks.tq[i]);
89624f0bfbaSKristof Provost 		taskqueue_free(epair_tasks.tq[i]);
89724f0bfbaSKristof Provost 	}
89824f0bfbaSKristof Provost }
89924f0bfbaSKristof Provost 
90024f0bfbaSKristof Provost static int
90198c230c8SBjoern A. Zeeb epair_modevent(module_t mod, int type, void *data)
90298c230c8SBjoern A. Zeeb {
90324f0bfbaSKristof Provost 	int ret;
90498c230c8SBjoern A. Zeeb 
90598c230c8SBjoern A. Zeeb 	switch (type) {
90698c230c8SBjoern A. Zeeb 	case MOD_LOAD:
9073dd5760aSBjoern A. Zeeb 		EPAIR_LOCK_INIT();
90824f0bfbaSKristof Provost 		ret = epair_mod_init();
90924f0bfbaSKristof Provost 		if (ret != 0)
91024f0bfbaSKristof Provost 			return (ret);
91198c230c8SBjoern A. Zeeb 		if (bootverbose)
9123dd5760aSBjoern A. Zeeb 			printf("%s: %s initialized.\n", __func__, epairname);
91398c230c8SBjoern A. Zeeb 		break;
91498c230c8SBjoern A. Zeeb 	case MOD_UNLOAD:
91524f0bfbaSKristof Provost 		epair_mod_cleanup();
9163dd5760aSBjoern A. Zeeb 		EPAIR_LOCK_DESTROY();
9173dd5760aSBjoern A. Zeeb 		if (bootverbose)
9183dd5760aSBjoern A. Zeeb 			printf("%s: %s unloaded.\n", __func__, epairname);
91998c230c8SBjoern A. Zeeb 		break;
92098c230c8SBjoern A. Zeeb 	default:
92198c230c8SBjoern A. Zeeb 		return (EOPNOTSUPP);
92298c230c8SBjoern A. Zeeb 	}
92398c230c8SBjoern A. Zeeb 	return (0);
92498c230c8SBjoern A. Zeeb }
92598c230c8SBjoern A. Zeeb 
92698c230c8SBjoern A. Zeeb static moduledata_t epair_mod = {
92798c230c8SBjoern A. Zeeb 	"if_epair",
92898c230c8SBjoern A. Zeeb 	epair_modevent,
9299823d527SKevin Lo 	0
93098c230c8SBjoern A. Zeeb };
93198c230c8SBjoern A. Zeeb 
93289856f7eSBjoern A. Zeeb DECLARE_MODULE(if_epair, epair_mod, SI_SUB_PSEUDO, SI_ORDER_MIDDLE);
9333dd5760aSBjoern A. Zeeb MODULE_VERSION(if_epair, 3);
934