198c230c8SBjoern A. Zeeb /*- 2*4d846d26SWarner Losh * SPDX-License-Identifier: BSD-2-Clause 3fe267a55SPedro F. Giffuni * 498c230c8SBjoern A. Zeeb * Copyright (c) 2008 The FreeBSD Foundation 53dd5760aSBjoern A. Zeeb * Copyright (c) 2009-2021 Bjoern A. Zeeb <bz@FreeBSD.org> 698c230c8SBjoern A. Zeeb * 798c230c8SBjoern A. Zeeb * This software was developed by CK Software GmbH under sponsorship 898c230c8SBjoern A. Zeeb * from the FreeBSD Foundation. 998c230c8SBjoern A. Zeeb * 1098c230c8SBjoern A. Zeeb * Redistribution and use in source and binary forms, with or without 1198c230c8SBjoern A. Zeeb * modification, are permitted provided that the following conditions 1298c230c8SBjoern A. Zeeb * are met: 1398c230c8SBjoern A. Zeeb * 1. Redistributions of source code must retain the above copyright 1498c230c8SBjoern A. Zeeb * notice, this list of conditions and the following disclaimer. 1598c230c8SBjoern A. Zeeb * 2. Redistributions in binary form must reproduce the above copyright 1698c230c8SBjoern A. Zeeb * notice, this list of conditions and the following disclaimer in the 1798c230c8SBjoern A. Zeeb * documentation and/or other materials provided with the distribution. 1898c230c8SBjoern A. Zeeb * 1998c230c8SBjoern A. Zeeb * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 2098c230c8SBjoern A. Zeeb * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2198c230c8SBjoern A. Zeeb * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2298c230c8SBjoern A. Zeeb * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 2398c230c8SBjoern A. Zeeb * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2498c230c8SBjoern A. Zeeb * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2598c230c8SBjoern A. Zeeb * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2698c230c8SBjoern A. Zeeb * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2798c230c8SBjoern A. Zeeb * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2898c230c8SBjoern A. Zeeb * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2998c230c8SBjoern A. Zeeb * SUCH DAMAGE. 3098c230c8SBjoern A. Zeeb */ 3198c230c8SBjoern A. Zeeb 3298c230c8SBjoern A. Zeeb /* 33d0ea4743SBjoern A. Zeeb * A pair of virtual back-to-back connected ethernet like interfaces 34d0ea4743SBjoern A. Zeeb * (``two interfaces with a virtual cross-over cable''). 35d0ea4743SBjoern A. Zeeb * 3698c230c8SBjoern A. Zeeb * This is mostly intended to be used to provide connectivity between 3798c230c8SBjoern A. Zeeb * different virtual network stack instances. 3898c230c8SBjoern A. Zeeb */ 3998c230c8SBjoern A. Zeeb 4098c230c8SBjoern A. Zeeb #include <sys/cdefs.h> 4198c230c8SBjoern A. Zeeb __FBSDID("$FreeBSD$"); 4298c230c8SBjoern A. Zeeb 4324f0bfbaSKristof Provost #include "opt_rss.h" 4452bcdc5bSSantiago Martinez #include "opt_inet.h" 4552bcdc5bSSantiago Martinez #include "opt_inet6.h" 4624f0bfbaSKristof Provost 4798c230c8SBjoern A. Zeeb #include <sys/param.h> 4829c9b167SMark Johnston #include <sys/bus.h> 4911d41666SLuca Pizzamiglio #include <sys/hash.h> 5029c9b167SMark Johnston #include <sys/interrupt.h> 5111d41666SLuca Pizzamiglio #include <sys/jail.h> 5298c230c8SBjoern A. Zeeb #include <sys/kernel.h> 5311d41666SLuca Pizzamiglio #include <sys/libkern.h> 548ec07310SGleb Smirnoff #include <sys/malloc.h> 5598c230c8SBjoern A. Zeeb #include <sys/mbuf.h> 5698c230c8SBjoern A. Zeeb #include <sys/module.h> 5711d41666SLuca Pizzamiglio #include <sys/proc.h> 5898c230c8SBjoern A. Zeeb #include <sys/queue.h> 5924f0bfbaSKristof Provost #include <sys/sched.h> 60d0ea4743SBjoern A. Zeeb #include <sys/smp.h> 6198c230c8SBjoern A. Zeeb #include <sys/socket.h> 6298c230c8SBjoern A. Zeeb #include <sys/sockio.h> 6324f0bfbaSKristof Provost #include <sys/taskqueue.h> 6498c230c8SBjoern A. Zeeb 6598c230c8SBjoern A. Zeeb #include <net/bpf.h> 6698c230c8SBjoern A. Zeeb #include <net/ethernet.h> 6798c230c8SBjoern A. Zeeb #include <net/if.h> 6876039bc8SGleb Smirnoff #include <net/if_var.h> 6998c230c8SBjoern A. Zeeb #include <net/if_clone.h> 702dccdd45SMarko Zec #include <net/if_media.h> 7198c230c8SBjoern A. Zeeb #include <net/if_var.h> 722c2b37adSJustin Hibbits #include <net/if_private.h> 7398c230c8SBjoern A. Zeeb #include <net/if_types.h> 7498c230c8SBjoern A. Zeeb #include <net/netisr.h> 7524f0bfbaSKristof Provost #ifdef RSS 7624f0bfbaSKristof Provost #include <net/rss_config.h> 7752bcdc5bSSantiago Martinez #ifdef INET 7824f0bfbaSKristof Provost #include <netinet/in_rss.h> 7952bcdc5bSSantiago Martinez #endif 8052bcdc5bSSantiago Martinez #ifdef INET6 8124f0bfbaSKristof Provost #include <netinet6/in6_rss.h> 8224f0bfbaSKristof Provost #endif 8352bcdc5bSSantiago Martinez #endif 84530c0060SRobert Watson #include <net/vnet.h> 8598c230c8SBjoern A. Zeeb 8642a58907SGleb Smirnoff static const char epairname[] = "epair"; 873dd5760aSBjoern A. Zeeb #define RXRSIZE 4096 /* Probably overkill by 4-8x. */ 88d0ea4743SBjoern A. Zeeb 8942a58907SGleb Smirnoff static MALLOC_DEFINE(M_EPAIR, epairname, 90d0ea4743SBjoern A. Zeeb "Pair of virtual cross-over connected Ethernet-like interfaces"); 9198c230c8SBjoern A. Zeeb 925f901c92SAndrew Turner VNET_DEFINE_STATIC(struct if_clone *, epair_cloner); 933c3136b1SHiroki Sato #define V_epair_cloner VNET(epair_cloner) 9498c230c8SBjoern A. Zeeb 953dd5760aSBjoern A. Zeeb static unsigned int next_index = 0; 963dd5760aSBjoern A. Zeeb #define EPAIR_LOCK_INIT() mtx_init(&epair_n_index_mtx, "epairidx", \ 973dd5760aSBjoern A. Zeeb NULL, MTX_DEF) 983dd5760aSBjoern A. Zeeb #define EPAIR_LOCK_DESTROY() mtx_destroy(&epair_n_index_mtx) 993dd5760aSBjoern A. Zeeb #define EPAIR_LOCK() mtx_lock(&epair_n_index_mtx) 1003dd5760aSBjoern A. Zeeb #define EPAIR_UNLOCK() mtx_unlock(&epair_n_index_mtx) 1013dd5760aSBjoern A. Zeeb 10224f0bfbaSKristof Provost struct epair_softc; 10324f0bfbaSKristof Provost struct epair_queue { 104df7bbd8cSMark Johnston struct mtx mtx; 105df7bbd8cSMark Johnston struct mbufq q; 10624f0bfbaSKristof Provost int id; 107df7bbd8cSMark Johnston enum { 108df7bbd8cSMark Johnston EPAIR_QUEUE_IDLE, 109df7bbd8cSMark Johnston EPAIR_QUEUE_WAKING, 110df7bbd8cSMark Johnston EPAIR_QUEUE_RUNNING, 111df7bbd8cSMark Johnston } state; 11224f0bfbaSKristof Provost struct task tx_task; 11324f0bfbaSKristof Provost struct epair_softc *sc; 11424f0bfbaSKristof Provost }; 1153dd5760aSBjoern A. Zeeb 1163dd5760aSBjoern A. Zeeb static struct mtx epair_n_index_mtx; 1173dd5760aSBjoern A. Zeeb struct epair_softc { 1183dd5760aSBjoern A. Zeeb struct ifnet *ifp; /* This ifp. */ 1193dd5760aSBjoern A. Zeeb struct ifnet *oifp; /* other ifp of pair. */ 12024f0bfbaSKristof Provost int num_queues; 12124f0bfbaSKristof Provost struct epair_queue *queues; 1223dd5760aSBjoern A. Zeeb struct ifmedia media; /* Media config (fake). */ 1233dd5760aSBjoern A. Zeeb STAILQ_ENTRY(epair_softc) entry; 124d0ea4743SBjoern A. Zeeb }; 125d0ea4743SBjoern A. Zeeb 12624f0bfbaSKristof Provost struct epair_tasks_t { 12724f0bfbaSKristof Provost int tasks; 12824f0bfbaSKristof Provost struct taskqueue *tq[MAXCPU]; 12924f0bfbaSKristof Provost }; 13024f0bfbaSKristof Provost 13124f0bfbaSKristof Provost static struct epair_tasks_t epair_tasks; 13224f0bfbaSKristof Provost 133d0ea4743SBjoern A. Zeeb static void 13462d2dcafSKristof Provost epair_clear_mbuf(struct mbuf *m) 13562d2dcafSKristof Provost { 136c69ae841SKristof Provost M_ASSERTPKTHDR(m); 137c69ae841SKristof Provost 13862d2dcafSKristof Provost /* Remove any CSUM_SND_TAG as ether_input will barf. */ 13962d2dcafSKristof Provost if (m->m_pkthdr.csum_flags & CSUM_SND_TAG) { 14062d2dcafSKristof Provost m_snd_tag_rele(m->m_pkthdr.snd_tag); 14162d2dcafSKristof Provost m->m_pkthdr.snd_tag = NULL; 14262d2dcafSKristof Provost m->m_pkthdr.csum_flags &= ~CSUM_SND_TAG; 14362d2dcafSKristof Provost } 14462d2dcafSKristof Provost 145c69ae841SKristof Provost /* Clear vlan information. */ 146c69ae841SKristof Provost m->m_flags &= ~M_VLANTAG; 147c69ae841SKristof Provost m->m_pkthdr.ether_vtag = 0; 148c69ae841SKristof Provost 14962d2dcafSKristof Provost m_tag_delete_nonpersistent(m); 15062d2dcafSKristof Provost } 15162d2dcafSKristof Provost 15262d2dcafSKristof Provost static void 15324f0bfbaSKristof Provost epair_tx_start_deferred(void *arg, int pending) 154d0ea4743SBjoern A. Zeeb { 15524f0bfbaSKristof Provost struct epair_queue *q = (struct epair_queue *)arg; 156df7bbd8cSMark Johnston if_t ifp; 157df7bbd8cSMark Johnston struct mbuf *m, *n; 158df7bbd8cSMark Johnston bool resched; 159d0ea4743SBjoern A. Zeeb 160df7bbd8cSMark Johnston ifp = q->sc->ifp; 16124f0bfbaSKristof Provost 162df7bbd8cSMark Johnston if_ref(ifp); 163df7bbd8cSMark Johnston CURVNET_SET(ifp->if_vnet); 164df7bbd8cSMark Johnston 165df7bbd8cSMark Johnston mtx_lock(&q->mtx); 166df7bbd8cSMark Johnston m = mbufq_flush(&q->q); 167df7bbd8cSMark Johnston q->state = EPAIR_QUEUE_RUNNING; 168df7bbd8cSMark Johnston mtx_unlock(&q->mtx); 169df7bbd8cSMark Johnston 170df7bbd8cSMark Johnston while (m != NULL) { 171df7bbd8cSMark Johnston n = STAILQ_NEXT(m, m_stailqpkt); 172df7bbd8cSMark Johnston m->m_nextpkt = NULL; 173df7bbd8cSMark Johnston if_input(ifp, m); 174df7bbd8cSMark Johnston m = n; 175df7bbd8cSMark Johnston } 176df7bbd8cSMark Johnston 177df7bbd8cSMark Johnston /* 178df7bbd8cSMark Johnston * Avoid flushing the queue more than once per task. We can otherwise 179df7bbd8cSMark Johnston * end up starving ourselves in a multi-epair routing configuration. 180df7bbd8cSMark Johnston */ 181df7bbd8cSMark Johnston mtx_lock(&q->mtx); 182df7bbd8cSMark Johnston if (mbufq_len(&q->q) > 0) { 183df7bbd8cSMark Johnston resched = true; 184df7bbd8cSMark Johnston q->state = EPAIR_QUEUE_WAKING; 185df7bbd8cSMark Johnston } else { 186df7bbd8cSMark Johnston resched = false; 187df7bbd8cSMark Johnston q->state = EPAIR_QUEUE_IDLE; 188df7bbd8cSMark Johnston } 189df7bbd8cSMark Johnston mtx_unlock(&q->mtx); 190df7bbd8cSMark Johnston 191df7bbd8cSMark Johnston if (resched) 19224f0bfbaSKristof Provost taskqueue_enqueue(epair_tasks.tq[q->id], &q->tx_task); 193d0ea4743SBjoern A. Zeeb 194df7bbd8cSMark Johnston CURVNET_RESTORE(); 195df7bbd8cSMark Johnston if_rele(ifp); 196d0ea4743SBjoern A. Zeeb } 197d0ea4743SBjoern A. Zeeb 19804a32b80SAlexander V. Chernikov static struct epair_queue * 1998a299958SKristof Provost epair_select_queue(struct epair_softc *sc, struct mbuf *m) 200d0ea4743SBjoern A. Zeeb { 20124f0bfbaSKristof Provost uint32_t bucket; 20224f0bfbaSKristof Provost #ifdef RSS 20324f0bfbaSKristof Provost struct ether_header *eh; 2048a299958SKristof Provost int ret; 205d0ea4743SBjoern A. Zeeb 20624f0bfbaSKristof Provost ret = rss_m2bucket(m, &bucket); 20724f0bfbaSKristof Provost if (ret) { 20824f0bfbaSKristof Provost /* Actually hash the packet. */ 20924f0bfbaSKristof Provost eh = mtod(m, struct ether_header *); 21024f0bfbaSKristof Provost 21124f0bfbaSKristof Provost switch (ntohs(eh->ether_type)) { 21252bcdc5bSSantiago Martinez #ifdef INET 21324f0bfbaSKristof Provost case ETHERTYPE_IP: 21424f0bfbaSKristof Provost rss_soft_m2cpuid_v4(m, 0, &bucket); 21524f0bfbaSKristof Provost break; 21652bcdc5bSSantiago Martinez #endif 21752bcdc5bSSantiago Martinez #ifdef INET6 21824f0bfbaSKristof Provost case ETHERTYPE_IPV6: 21924f0bfbaSKristof Provost rss_soft_m2cpuid_v6(m, 0, &bucket); 22024f0bfbaSKristof Provost break; 22152bcdc5bSSantiago Martinez #endif 22224f0bfbaSKristof Provost default: 22324f0bfbaSKristof Provost bucket = 0; 22424f0bfbaSKristof Provost break; 22524f0bfbaSKristof Provost } 22624f0bfbaSKristof Provost } 22704a32b80SAlexander V. Chernikov bucket %= sc->num_queues; 22824f0bfbaSKristof Provost #else 22924f0bfbaSKristof Provost bucket = 0; 23024f0bfbaSKristof Provost #endif 23104a32b80SAlexander V. Chernikov return (&sc->queues[bucket]); 23204a32b80SAlexander V. Chernikov } 23304a32b80SAlexander V. Chernikov 23404a32b80SAlexander V. Chernikov static void 23504a32b80SAlexander V. Chernikov epair_prepare_mbuf(struct mbuf *m, struct ifnet *src_ifp) 23604a32b80SAlexander V. Chernikov { 23704a32b80SAlexander V. Chernikov M_ASSERTPKTHDR(m); 23804a32b80SAlexander V. Chernikov epair_clear_mbuf(m); 23904a32b80SAlexander V. Chernikov if_setrcvif(m, src_ifp); 24004a32b80SAlexander V. Chernikov M_SETFIB(m, src_ifp->if_fib); 24104a32b80SAlexander V. Chernikov 24204a32b80SAlexander V. Chernikov MPASS(m->m_nextpkt == NULL); 24304a32b80SAlexander V. Chernikov MPASS((m->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0); 24404a32b80SAlexander V. Chernikov } 24504a32b80SAlexander V. Chernikov 24604a32b80SAlexander V. Chernikov static void 24704a32b80SAlexander V. Chernikov epair_menq(struct mbuf *m, struct epair_softc *osc) 24804a32b80SAlexander V. Chernikov { 249df7bbd8cSMark Johnston struct epair_queue *q; 25004a32b80SAlexander V. Chernikov struct ifnet *ifp, *oifp; 251df7bbd8cSMark Johnston int error, len; 25248227d1cSMark Johnston bool mcast; 25304a32b80SAlexander V. Chernikov 25404a32b80SAlexander V. Chernikov /* 25504a32b80SAlexander V. Chernikov * I know this looks weird. We pass the "other sc" as we need that one 25604a32b80SAlexander V. Chernikov * and can get both ifps from it as well. 25704a32b80SAlexander V. Chernikov */ 25804a32b80SAlexander V. Chernikov oifp = osc->ifp; 25904a32b80SAlexander V. Chernikov ifp = osc->oifp; 26004a32b80SAlexander V. Chernikov 26104a32b80SAlexander V. Chernikov epair_prepare_mbuf(m, oifp); 26204a32b80SAlexander V. Chernikov 26304a32b80SAlexander V. Chernikov /* Save values as once the mbuf is queued, it's not ours anymore. */ 26404a32b80SAlexander V. Chernikov len = m->m_pkthdr.len; 26548227d1cSMark Johnston mcast = (m->m_flags & (M_BCAST | M_MCAST)) != 0; 26604a32b80SAlexander V. Chernikov 267df7bbd8cSMark Johnston q = epair_select_queue(osc, m); 26824f0bfbaSKristof Provost 269df7bbd8cSMark Johnston mtx_lock(&q->mtx); 270df7bbd8cSMark Johnston if (q->state == EPAIR_QUEUE_IDLE) { 271df7bbd8cSMark Johnston q->state = EPAIR_QUEUE_WAKING; 272df7bbd8cSMark Johnston taskqueue_enqueue(epair_tasks.tq[q->id], &q->tx_task); 2733dd5760aSBjoern A. Zeeb } 274df7bbd8cSMark Johnston error = mbufq_enqueue(&q->q, m); 275df7bbd8cSMark Johnston mtx_unlock(&q->mtx); 276d0ea4743SBjoern A. Zeeb 277df7bbd8cSMark Johnston if (error != 0) { 278df7bbd8cSMark Johnston m_freem(m); 279df7bbd8cSMark Johnston if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); 280df7bbd8cSMark Johnston } else { 2813dd5760aSBjoern A. Zeeb if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); 2823dd5760aSBjoern A. Zeeb if_inc_counter(ifp, IFCOUNTER_OBYTES, len); 28348227d1cSMark Johnston if (mcast) 2843dd5760aSBjoern A. Zeeb if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); 2853dd5760aSBjoern A. Zeeb if_inc_counter(oifp, IFCOUNTER_IPACKETS, 1); 286df7bbd8cSMark Johnston } 287d0ea4743SBjoern A. Zeeb } 288d0ea4743SBjoern A. Zeeb 28998c230c8SBjoern A. Zeeb static void 2903dd5760aSBjoern A. Zeeb epair_start(struct ifnet *ifp) 29198c230c8SBjoern A. Zeeb { 29298c230c8SBjoern A. Zeeb struct mbuf *m; 29398c230c8SBjoern A. Zeeb struct epair_softc *sc; 29498c230c8SBjoern A. Zeeb struct ifnet *oifp; 29598c230c8SBjoern A. Zeeb 29698c230c8SBjoern A. Zeeb /* 29759f35a82SPatrick Kelsey * We get packets here from ether_output via if_handoff() 29875580d58SPatrick Kelsey * and need to put them into the input queue of the oifp 2993dd5760aSBjoern A. Zeeb * and will put the packet into the receive-queue (rxq) of the 3003dd5760aSBjoern A. Zeeb * other interface (oifp) of our pair. 30198c230c8SBjoern A. Zeeb */ 3023dd5760aSBjoern A. Zeeb sc = ifp->if_softc; 30398c230c8SBjoern A. Zeeb oifp = sc->oifp; 30498c230c8SBjoern A. Zeeb sc = oifp->if_softc; 30598c230c8SBjoern A. Zeeb for (;;) { 30698c230c8SBjoern A. Zeeb IFQ_DEQUEUE(&ifp->if_snd, m); 30798c230c8SBjoern A. Zeeb if (m == NULL) 30898c230c8SBjoern A. Zeeb break; 3093dd5760aSBjoern A. Zeeb M_ASSERTPKTHDR(m); 31098c230c8SBjoern A. Zeeb BPF_MTAP(ifp, m); 31198c230c8SBjoern A. Zeeb 3123dd5760aSBjoern A. Zeeb /* In case either interface is not usable drop the packet. */ 3133dd5760aSBjoern A. Zeeb if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 3143dd5760aSBjoern A. Zeeb (ifp->if_flags & IFF_UP) == 0 || 3153dd5760aSBjoern A. Zeeb (oifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 31698c230c8SBjoern A. Zeeb (oifp->if_flags & IFF_UP) == 0) { 31798c230c8SBjoern A. Zeeb m_freem(m); 31898c230c8SBjoern A. Zeeb continue; 31998c230c8SBjoern A. Zeeb } 32098c230c8SBjoern A. Zeeb 32104a32b80SAlexander V. Chernikov epair_menq(m, sc); 32298c230c8SBjoern A. Zeeb } 32398c230c8SBjoern A. Zeeb } 32498c230c8SBjoern A. Zeeb 32598c230c8SBjoern A. Zeeb static int 3263dd5760aSBjoern A. Zeeb epair_transmit(struct ifnet *ifp, struct mbuf *m) 32798c230c8SBjoern A. Zeeb { 32898c230c8SBjoern A. Zeeb struct epair_softc *sc; 32998c230c8SBjoern A. Zeeb struct ifnet *oifp; 3302cedfc3fSMateusz Guzik #ifdef ALTQ 3312cedfc3fSMateusz Guzik int len; 33248227d1cSMark Johnston bool mcast; 3332cedfc3fSMateusz Guzik #endif 33498c230c8SBjoern A. Zeeb 33598c230c8SBjoern A. Zeeb if (m == NULL) 33698c230c8SBjoern A. Zeeb return (0); 3373dd5760aSBjoern A. Zeeb M_ASSERTPKTHDR(m); 33898c230c8SBjoern A. Zeeb 33998c230c8SBjoern A. Zeeb /* 34098c230c8SBjoern A. Zeeb * We are not going to use the interface en/dequeue mechanism 34198c230c8SBjoern A. Zeeb * on the TX side. We are called from ether_output_frame() 3423dd5760aSBjoern A. Zeeb * and will put the packet into the receive-queue (rxq) of the 3433dd5760aSBjoern A. Zeeb * other interface (oifp) of our pair. 34498c230c8SBjoern A. Zeeb */ 34598c230c8SBjoern A. Zeeb if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 34698c230c8SBjoern A. Zeeb m_freem(m); 3473dd5760aSBjoern A. Zeeb if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 34898c230c8SBjoern A. Zeeb return (ENXIO); 34998c230c8SBjoern A. Zeeb } 35098c230c8SBjoern A. Zeeb if ((ifp->if_flags & IFF_UP) == 0) { 35198c230c8SBjoern A. Zeeb m_freem(m); 3523dd5760aSBjoern A. Zeeb if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 35398c230c8SBjoern A. Zeeb return (ENETDOWN); 35498c230c8SBjoern A. Zeeb } 35598c230c8SBjoern A. Zeeb 35698c230c8SBjoern A. Zeeb BPF_MTAP(ifp, m); 35798c230c8SBjoern A. Zeeb 35898c230c8SBjoern A. Zeeb /* 35998c230c8SBjoern A. Zeeb * In case the outgoing interface is not usable, 36098c230c8SBjoern A. Zeeb * drop the packet. 36198c230c8SBjoern A. Zeeb */ 3623dd5760aSBjoern A. Zeeb sc = ifp->if_softc; 36398c230c8SBjoern A. Zeeb oifp = sc->oifp; 36498c230c8SBjoern A. Zeeb if ((oifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 36598c230c8SBjoern A. Zeeb (oifp->if_flags & IFF_UP) == 0) { 3663751dddbSGleb Smirnoff if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 36798c230c8SBjoern A. Zeeb m_freem(m); 36898c230c8SBjoern A. Zeeb return (0); 36998c230c8SBjoern A. Zeeb } 3702cedfc3fSMateusz Guzik 3712cedfc3fSMateusz Guzik #ifdef ALTQ 37298c230c8SBjoern A. Zeeb len = m->m_pkthdr.len; 37348227d1cSMark Johnston mcast = (m->m_flags & (M_BCAST | M_MCAST)) != 0; 37404a32b80SAlexander V. Chernikov int error = 0; 37598c230c8SBjoern A. Zeeb 376a4641f4eSPedro F. Giffuni /* Support ALTQ via the classic if_start() path. */ 37798c230c8SBjoern A. Zeeb IF_LOCK(&ifp->if_snd); 37898c230c8SBjoern A. Zeeb if (ALTQ_IS_ENABLED(&ifp->if_snd)) { 37998c230c8SBjoern A. Zeeb ALTQ_ENQUEUE(&ifp->if_snd, m, NULL, error); 38098c230c8SBjoern A. Zeeb if (error) 3813751dddbSGleb Smirnoff if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); 38298c230c8SBjoern A. Zeeb IF_UNLOCK(&ifp->if_snd); 38398c230c8SBjoern A. Zeeb if (!error) { 3843751dddbSGleb Smirnoff if_inc_counter(ifp, IFCOUNTER_OBYTES, len); 38548227d1cSMark Johnston if (mcast) 3863751dddbSGleb Smirnoff if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); 3873dd5760aSBjoern A. Zeeb epair_start(ifp); 38898c230c8SBjoern A. Zeeb } 38998c230c8SBjoern A. Zeeb return (error); 39098c230c8SBjoern A. Zeeb } 39198c230c8SBjoern A. Zeeb IF_UNLOCK(&ifp->if_snd); 39298c230c8SBjoern A. Zeeb #endif 39398c230c8SBjoern A. Zeeb 39404a32b80SAlexander V. Chernikov epair_menq(m, oifp->if_softc); 39504a32b80SAlexander V. Chernikov return (0); 39698c230c8SBjoern A. Zeeb } 39762d2dcafSKristof Provost 39812aeeb91SAlexander V. Chernikov static void 39912aeeb91SAlexander V. Chernikov epair_qflush(struct ifnet *ifp __unused) 40012aeeb91SAlexander V. Chernikov { 40112aeeb91SAlexander V. Chernikov } 40212aeeb91SAlexander V. Chernikov 40398c230c8SBjoern A. Zeeb static int 4042dccdd45SMarko Zec epair_media_change(struct ifnet *ifp __unused) 4052dccdd45SMarko Zec { 4062dccdd45SMarko Zec 4072dccdd45SMarko Zec /* Do nothing. */ 4082dccdd45SMarko Zec return (0); 4092dccdd45SMarko Zec } 4102dccdd45SMarko Zec 4112dccdd45SMarko Zec static void 4122dccdd45SMarko Zec epair_media_status(struct ifnet *ifp __unused, struct ifmediareq *imr) 4132dccdd45SMarko Zec { 4142dccdd45SMarko Zec 4152dccdd45SMarko Zec imr->ifm_status = IFM_AVALID | IFM_ACTIVE; 4162dccdd45SMarko Zec imr->ifm_active = IFM_ETHER | IFM_10G_T | IFM_FDX; 4172dccdd45SMarko Zec } 4182dccdd45SMarko Zec 4192dccdd45SMarko Zec static int 42098c230c8SBjoern A. Zeeb epair_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 42198c230c8SBjoern A. Zeeb { 4222dccdd45SMarko Zec struct epair_softc *sc; 42398c230c8SBjoern A. Zeeb struct ifreq *ifr; 42498c230c8SBjoern A. Zeeb int error; 42598c230c8SBjoern A. Zeeb 42698c230c8SBjoern A. Zeeb ifr = (struct ifreq *)data; 42798c230c8SBjoern A. Zeeb switch (cmd) { 42898c230c8SBjoern A. Zeeb case SIOCSIFFLAGS: 42998c230c8SBjoern A. Zeeb case SIOCADDMULTI: 43098c230c8SBjoern A. Zeeb case SIOCDELMULTI: 43198c230c8SBjoern A. Zeeb error = 0; 43298c230c8SBjoern A. Zeeb break; 43398c230c8SBjoern A. Zeeb 4342dccdd45SMarko Zec case SIOCSIFMEDIA: 4352dccdd45SMarko Zec case SIOCGIFMEDIA: 4362dccdd45SMarko Zec sc = ifp->if_softc; 4372dccdd45SMarko Zec error = ifmedia_ioctl(ifp, ifr, &sc->media, cmd); 4382dccdd45SMarko Zec break; 4392dccdd45SMarko Zec 440d0ea4743SBjoern A. Zeeb case SIOCSIFMTU: 441d0ea4743SBjoern A. Zeeb /* We basically allow all kinds of MTUs. */ 442d0ea4743SBjoern A. Zeeb ifp->if_mtu = ifr->ifr_mtu; 443d0ea4743SBjoern A. Zeeb error = 0; 444d0ea4743SBjoern A. Zeeb break; 445d0ea4743SBjoern A. Zeeb 44698c230c8SBjoern A. Zeeb default: 44798c230c8SBjoern A. Zeeb /* Let the common ethernet handler process this. */ 44898c230c8SBjoern A. Zeeb error = ether_ioctl(ifp, cmd, data); 44998c230c8SBjoern A. Zeeb break; 45098c230c8SBjoern A. Zeeb } 45198c230c8SBjoern A. Zeeb 45298c230c8SBjoern A. Zeeb return (error); 45398c230c8SBjoern A. Zeeb } 45498c230c8SBjoern A. Zeeb 45598c230c8SBjoern A. Zeeb static void 45698c230c8SBjoern A. Zeeb epair_init(void *dummy __unused) 45798c230c8SBjoern A. Zeeb { 45898c230c8SBjoern A. Zeeb } 45998c230c8SBjoern A. Zeeb 46098c230c8SBjoern A. Zeeb /* 46198c230c8SBjoern A. Zeeb * Interface cloning functions. 46298c230c8SBjoern A. Zeeb * We use our private ones so that we can create/destroy our secondary 46398c230c8SBjoern A. Zeeb * device along with the primary one. 46498c230c8SBjoern A. Zeeb */ 46598c230c8SBjoern A. Zeeb static int 46698c230c8SBjoern A. Zeeb epair_clone_match(struct if_clone *ifc, const char *name) 46798c230c8SBjoern A. Zeeb { 46898c230c8SBjoern A. Zeeb const char *cp; 46998c230c8SBjoern A. Zeeb 47098c230c8SBjoern A. Zeeb /* 47198c230c8SBjoern A. Zeeb * Our base name is epair. 47298c230c8SBjoern A. Zeeb * Our interfaces will be named epair<n>[ab]. 47398c230c8SBjoern A. Zeeb * So accept anything of the following list: 47498c230c8SBjoern A. Zeeb * - epair 47598c230c8SBjoern A. Zeeb * - epair<n> 47698c230c8SBjoern A. Zeeb * but not the epair<n>[ab] versions. 47798c230c8SBjoern A. Zeeb */ 47842a58907SGleb Smirnoff if (strncmp(epairname, name, sizeof(epairname)-1) != 0) 47998c230c8SBjoern A. Zeeb return (0); 48098c230c8SBjoern A. Zeeb 48142a58907SGleb Smirnoff for (cp = name + sizeof(epairname) - 1; *cp != '\0'; cp++) { 48298c230c8SBjoern A. Zeeb if (*cp < '0' || *cp > '9') 48398c230c8SBjoern A. Zeeb return (0); 48498c230c8SBjoern A. Zeeb } 48598c230c8SBjoern A. Zeeb 48698c230c8SBjoern A. Zeeb return (1); 48798c230c8SBjoern A. Zeeb } 48898c230c8SBjoern A. Zeeb 489b02fd8b7SKristof Provost static void 490b02fd8b7SKristof Provost epair_clone_add(struct if_clone *ifc, struct epair_softc *scb) 491b02fd8b7SKristof Provost { 492b02fd8b7SKristof Provost struct ifnet *ifp; 493b02fd8b7SKristof Provost uint8_t eaddr[ETHER_ADDR_LEN]; /* 00:00:00:00:00:00 */ 494b02fd8b7SKristof Provost 495b02fd8b7SKristof Provost ifp = scb->ifp; 496b02fd8b7SKristof Provost /* Copy epairNa etheraddr and change the last byte. */ 497b02fd8b7SKristof Provost memcpy(eaddr, scb->oifp->if_hw_addr, ETHER_ADDR_LEN); 498b02fd8b7SKristof Provost eaddr[5] = 0x0b; 499b02fd8b7SKristof Provost ether_ifattach(ifp, eaddr); 500b02fd8b7SKristof Provost 501b02fd8b7SKristof Provost if_clone_addif(ifc, ifp); 502b02fd8b7SKristof Provost } 503b02fd8b7SKristof Provost 50412aeeb91SAlexander V. Chernikov static struct epair_softc * 50512aeeb91SAlexander V. Chernikov epair_alloc_sc(struct if_clone *ifc) 50612aeeb91SAlexander V. Chernikov { 50712aeeb91SAlexander V. Chernikov struct epair_softc *sc; 50812aeeb91SAlexander V. Chernikov 50912aeeb91SAlexander V. Chernikov struct ifnet *ifp = if_alloc(IFT_ETHER); 51012aeeb91SAlexander V. Chernikov if (ifp == NULL) 51112aeeb91SAlexander V. Chernikov return (NULL); 51212aeeb91SAlexander V. Chernikov 51312aeeb91SAlexander V. Chernikov sc = malloc(sizeof(struct epair_softc), M_EPAIR, M_WAITOK | M_ZERO); 51412aeeb91SAlexander V. Chernikov sc->ifp = ifp; 51512aeeb91SAlexander V. Chernikov sc->num_queues = epair_tasks.tasks; 51612aeeb91SAlexander V. Chernikov sc->queues = mallocarray(sc->num_queues, sizeof(struct epair_queue), 51712aeeb91SAlexander V. Chernikov M_EPAIR, M_WAITOK); 51812aeeb91SAlexander V. Chernikov for (int i = 0; i < sc->num_queues; i++) { 51912aeeb91SAlexander V. Chernikov struct epair_queue *q = &sc->queues[i]; 52012aeeb91SAlexander V. Chernikov q->id = i; 521df7bbd8cSMark Johnston q->state = EPAIR_QUEUE_IDLE; 522df7bbd8cSMark Johnston mtx_init(&q->mtx, "epairq", NULL, MTX_DEF | MTX_NEW); 523df7bbd8cSMark Johnston mbufq_init(&q->q, RXRSIZE); 52412aeeb91SAlexander V. Chernikov q->sc = sc; 52512aeeb91SAlexander V. Chernikov NET_TASK_INIT(&q->tx_task, 0, epair_tx_start_deferred, q); 52612aeeb91SAlexander V. Chernikov } 52712aeeb91SAlexander V. Chernikov 52812aeeb91SAlexander V. Chernikov /* Initialise pseudo media types. */ 52912aeeb91SAlexander V. Chernikov ifmedia_init(&sc->media, 0, epair_media_change, epair_media_status); 53012aeeb91SAlexander V. Chernikov ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_T, 0, NULL); 53112aeeb91SAlexander V. Chernikov ifmedia_set(&sc->media, IFM_ETHER | IFM_10G_T); 53212aeeb91SAlexander V. Chernikov 53312aeeb91SAlexander V. Chernikov return (sc); 53412aeeb91SAlexander V. Chernikov } 53512aeeb91SAlexander V. Chernikov 53612aeeb91SAlexander V. Chernikov static void 53712aeeb91SAlexander V. Chernikov epair_setup_ifp(struct epair_softc *sc, char *name, int unit) 53812aeeb91SAlexander V. Chernikov { 53912aeeb91SAlexander V. Chernikov struct ifnet *ifp = sc->ifp; 54012aeeb91SAlexander V. Chernikov 54112aeeb91SAlexander V. Chernikov ifp->if_softc = sc; 54212aeeb91SAlexander V. Chernikov strlcpy(ifp->if_xname, name, IFNAMSIZ); 54312aeeb91SAlexander V. Chernikov ifp->if_dname = epairname; 54412aeeb91SAlexander V. Chernikov ifp->if_dunit = unit; 54512aeeb91SAlexander V. Chernikov ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 54612aeeb91SAlexander V. Chernikov ifp->if_capabilities = IFCAP_VLAN_MTU; 54712aeeb91SAlexander V. Chernikov ifp->if_capenable = IFCAP_VLAN_MTU; 54812aeeb91SAlexander V. Chernikov ifp->if_transmit = epair_transmit; 54912aeeb91SAlexander V. Chernikov ifp->if_qflush = epair_qflush; 55012aeeb91SAlexander V. Chernikov ifp->if_start = epair_start; 55112aeeb91SAlexander V. Chernikov ifp->if_ioctl = epair_ioctl; 55212aeeb91SAlexander V. Chernikov ifp->if_init = epair_init; 55312aeeb91SAlexander V. Chernikov if_setsendqlen(ifp, ifqmaxlen); 55412aeeb91SAlexander V. Chernikov if_setsendqready(ifp); 55512aeeb91SAlexander V. Chernikov 55612aeeb91SAlexander V. Chernikov ifp->if_baudrate = IF_Gbps(10); /* arbitrary maximum */ 55712aeeb91SAlexander V. Chernikov } 55812aeeb91SAlexander V. Chernikov 55912aeeb91SAlexander V. Chernikov static void 56012aeeb91SAlexander V. Chernikov epair_generate_mac(struct epair_softc *sc, uint8_t *eaddr) 56112aeeb91SAlexander V. Chernikov { 56212aeeb91SAlexander V. Chernikov uint32_t key[3]; 56312aeeb91SAlexander V. Chernikov uint32_t hash; 56412aeeb91SAlexander V. Chernikov uint64_t hostid; 56512aeeb91SAlexander V. Chernikov 56612aeeb91SAlexander V. Chernikov EPAIR_LOCK(); 56712aeeb91SAlexander V. Chernikov #ifdef SMP 56812aeeb91SAlexander V. Chernikov /* Get an approximate distribution. */ 56912aeeb91SAlexander V. Chernikov hash = next_index % mp_ncpus; 57012aeeb91SAlexander V. Chernikov #else 57112aeeb91SAlexander V. Chernikov hash = 0; 57212aeeb91SAlexander V. Chernikov #endif 57312aeeb91SAlexander V. Chernikov EPAIR_UNLOCK(); 57412aeeb91SAlexander V. Chernikov 57512aeeb91SAlexander V. Chernikov /* 57612aeeb91SAlexander V. Chernikov * Calculate the etheraddr hashing the hostid and the 57712aeeb91SAlexander V. Chernikov * interface index. The result would be hopefully unique. 57812aeeb91SAlexander V. Chernikov * Note that the "a" component of an epair instance may get moved 57912aeeb91SAlexander V. Chernikov * to a different VNET after creation. In that case its index 58012aeeb91SAlexander V. Chernikov * will be freed and the index can get reused by new epair instance. 58112aeeb91SAlexander V. Chernikov * Make sure we do not create same etheraddr again. 58212aeeb91SAlexander V. Chernikov */ 58312aeeb91SAlexander V. Chernikov getcredhostid(curthread->td_ucred, (unsigned long *)&hostid); 58412aeeb91SAlexander V. Chernikov if (hostid == 0) 58512aeeb91SAlexander V. Chernikov arc4rand(&hostid, sizeof(hostid), 0); 58612aeeb91SAlexander V. Chernikov 58712aeeb91SAlexander V. Chernikov struct ifnet *ifp = sc->ifp; 58812aeeb91SAlexander V. Chernikov EPAIR_LOCK(); 58912aeeb91SAlexander V. Chernikov if (ifp->if_index > next_index) 59012aeeb91SAlexander V. Chernikov next_index = ifp->if_index; 59112aeeb91SAlexander V. Chernikov else 59212aeeb91SAlexander V. Chernikov next_index++; 59312aeeb91SAlexander V. Chernikov 59412aeeb91SAlexander V. Chernikov key[0] = (uint32_t)next_index; 59512aeeb91SAlexander V. Chernikov EPAIR_UNLOCK(); 59612aeeb91SAlexander V. Chernikov key[1] = (uint32_t)(hostid & 0xffffffff); 59712aeeb91SAlexander V. Chernikov key[2] = (uint32_t)((hostid >> 32) & 0xfffffffff); 59812aeeb91SAlexander V. Chernikov hash = jenkins_hash32(key, 3, 0); 59912aeeb91SAlexander V. Chernikov 60012aeeb91SAlexander V. Chernikov eaddr[0] = 0x02; 60112aeeb91SAlexander V. Chernikov memcpy(&eaddr[1], &hash, 4); 60212aeeb91SAlexander V. Chernikov eaddr[5] = 0x0a; 60312aeeb91SAlexander V. Chernikov } 60412aeeb91SAlexander V. Chernikov 60512aeeb91SAlexander V. Chernikov static void 60612aeeb91SAlexander V. Chernikov epair_free_sc(struct epair_softc *sc) 60712aeeb91SAlexander V. Chernikov { 60812aeeb91SAlexander V. Chernikov if (sc == NULL) 60912aeeb91SAlexander V. Chernikov return; 61012aeeb91SAlexander V. Chernikov 61112aeeb91SAlexander V. Chernikov if_free(sc->ifp); 61212aeeb91SAlexander V. Chernikov ifmedia_removeall(&sc->media); 61312aeeb91SAlexander V. Chernikov for (int i = 0; i < sc->num_queues; i++) { 61412aeeb91SAlexander V. Chernikov struct epair_queue *q = &sc->queues[i]; 615df7bbd8cSMark Johnston mtx_destroy(&q->mtx); 61612aeeb91SAlexander V. Chernikov } 61712aeeb91SAlexander V. Chernikov free(sc->queues, M_EPAIR); 61812aeeb91SAlexander V. Chernikov free(sc, M_EPAIR); 61912aeeb91SAlexander V. Chernikov } 62012aeeb91SAlexander V. Chernikov 62104a32b80SAlexander V. Chernikov static void 62204a32b80SAlexander V. Chernikov epair_set_state(struct ifnet *ifp, bool running) 62398c230c8SBjoern A. Zeeb { 62404a32b80SAlexander V. Chernikov if (running) { 62504a32b80SAlexander V. Chernikov ifp->if_drv_flags |= IFF_DRV_RUNNING; 62604a32b80SAlexander V. Chernikov if_link_state_change(ifp, LINK_STATE_UP); 62704a32b80SAlexander V. Chernikov } else { 62804a32b80SAlexander V. Chernikov if_link_state_change(ifp, LINK_STATE_DOWN); 62904a32b80SAlexander V. Chernikov ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 63004a32b80SAlexander V. Chernikov } 63104a32b80SAlexander V. Chernikov } 63204a32b80SAlexander V. Chernikov 63304a32b80SAlexander V. Chernikov static int 63404a32b80SAlexander V. Chernikov epair_handle_unit(struct if_clone *ifc, char *name, size_t len, int *punit) 63504a32b80SAlexander V. Chernikov { 63604a32b80SAlexander V. Chernikov int error = 0, unit, wildcard; 63798c230c8SBjoern A. Zeeb char *dp; 63898c230c8SBjoern A. Zeeb 63998c230c8SBjoern A. Zeeb /* Try to see if a special unit was requested. */ 64098c230c8SBjoern A. Zeeb error = ifc_name2unit(name, &unit); 64198c230c8SBjoern A. Zeeb if (error != 0) 64298c230c8SBjoern A. Zeeb return (error); 64398c230c8SBjoern A. Zeeb wildcard = (unit < 0); 64498c230c8SBjoern A. Zeeb 64598c230c8SBjoern A. Zeeb error = ifc_alloc_unit(ifc, &unit); 64698c230c8SBjoern A. Zeeb if (error != 0) 64798c230c8SBjoern A. Zeeb return (error); 64898c230c8SBjoern A. Zeeb 64998c230c8SBjoern A. Zeeb /* 65098c230c8SBjoern A. Zeeb * If no unit had been given, we need to adjust the ifName. 65198c230c8SBjoern A. Zeeb * Also make sure there is space for our extra [ab] suffix. 65298c230c8SBjoern A. Zeeb */ 65398c230c8SBjoern A. Zeeb for (dp = name; *dp != '\0'; dp++); 65498c230c8SBjoern A. Zeeb if (wildcard) { 65504a32b80SAlexander V. Chernikov int slen = snprintf(dp, len - (dp - name), "%d", unit); 65604a32b80SAlexander V. Chernikov if (slen > len - (dp - name) - 1) { 65798c230c8SBjoern A. Zeeb /* ifName too long. */ 65804a32b80SAlexander V. Chernikov error = ENOSPC; 65904a32b80SAlexander V. Chernikov goto done; 66098c230c8SBjoern A. Zeeb } 66104a32b80SAlexander V. Chernikov dp += slen; 66298c230c8SBjoern A. Zeeb } 66398c230c8SBjoern A. Zeeb if (len - (dp - name) - 1 < 1) { 66498c230c8SBjoern A. Zeeb /* No space left for our [ab] suffix. */ 66504a32b80SAlexander V. Chernikov error = ENOSPC; 66604a32b80SAlexander V. Chernikov goto done; 66798c230c8SBjoern A. Zeeb } 6683c3136b1SHiroki Sato *dp = 'b'; 66998c230c8SBjoern A. Zeeb /* Must not change dp so we can replace 'a' by 'b' later. */ 67098c230c8SBjoern A. Zeeb *(dp+1) = '\0'; 67198c230c8SBjoern A. Zeeb 6723c3136b1SHiroki Sato /* Check if 'a' and 'b' interfaces already exist. */ 67304a32b80SAlexander V. Chernikov if (ifunit(name) != NULL) { 67404a32b80SAlexander V. Chernikov error = EEXIST; 67504a32b80SAlexander V. Chernikov goto done; 67604a32b80SAlexander V. Chernikov } 67704a32b80SAlexander V. Chernikov 6783c3136b1SHiroki Sato *dp = 'a'; 67904a32b80SAlexander V. Chernikov if (ifunit(name) != NULL) { 68004a32b80SAlexander V. Chernikov error = EEXIST; 68104a32b80SAlexander V. Chernikov goto done; 68204a32b80SAlexander V. Chernikov } 68304a32b80SAlexander V. Chernikov *punit = unit; 68404a32b80SAlexander V. Chernikov done: 68504a32b80SAlexander V. Chernikov if (error != 0) 68604a32b80SAlexander V. Chernikov ifc_free_unit(ifc, unit); 68704a32b80SAlexander V. Chernikov 68804a32b80SAlexander V. Chernikov return (error); 68904a32b80SAlexander V. Chernikov } 69004a32b80SAlexander V. Chernikov 69104a32b80SAlexander V. Chernikov static int 69204a32b80SAlexander V. Chernikov epair_clone_create(struct if_clone *ifc, char *name, size_t len, 69304a32b80SAlexander V. Chernikov struct ifc_data *ifd, struct ifnet **ifpp) 69404a32b80SAlexander V. Chernikov { 69504a32b80SAlexander V. Chernikov struct epair_softc *sca, *scb; 69604a32b80SAlexander V. Chernikov struct ifnet *ifp; 69704a32b80SAlexander V. Chernikov char *dp; 69804a32b80SAlexander V. Chernikov int error, unit; 69904a32b80SAlexander V. Chernikov uint8_t eaddr[ETHER_ADDR_LEN]; /* 00:00:00:00:00:00 */ 70004a32b80SAlexander V. Chernikov 70104a32b80SAlexander V. Chernikov error = epair_handle_unit(ifc, name, len, &unit); 70204a32b80SAlexander V. Chernikov if (error != 0) 70304a32b80SAlexander V. Chernikov return (error); 7043c3136b1SHiroki Sato 70598c230c8SBjoern A. Zeeb /* Allocate memory for both [ab] interfaces */ 70612aeeb91SAlexander V. Chernikov sca = epair_alloc_sc(ifc); 70712aeeb91SAlexander V. Chernikov scb = epair_alloc_sc(ifc); 70812aeeb91SAlexander V. Chernikov if (sca == NULL || scb == NULL) { 70912aeeb91SAlexander V. Chernikov epair_free_sc(sca); 71012aeeb91SAlexander V. Chernikov epair_free_sc(scb); 71198c230c8SBjoern A. Zeeb ifc_free_unit(ifc, unit); 71298c230c8SBjoern A. Zeeb return (ENOSPC); 71398c230c8SBjoern A. Zeeb } 71498c230c8SBjoern A. Zeeb 71598c230c8SBjoern A. Zeeb /* 71698c230c8SBjoern A. Zeeb * Cross-reference the interfaces so we will be able to free both. 71798c230c8SBjoern A. Zeeb */ 71898c230c8SBjoern A. Zeeb sca->oifp = scb->ifp; 71998c230c8SBjoern A. Zeeb scb->oifp = sca->ifp; 72098c230c8SBjoern A. Zeeb 72198c230c8SBjoern A. Zeeb /* Finish initialization of interface <n>a. */ 72298c230c8SBjoern A. Zeeb ifp = sca->ifp; 72312aeeb91SAlexander V. Chernikov epair_setup_ifp(sca, name, unit); 72412aeeb91SAlexander V. Chernikov epair_generate_mac(sca, eaddr); 72511d41666SLuca Pizzamiglio 72698c230c8SBjoern A. Zeeb ether_ifattach(ifp, eaddr); 72798c230c8SBjoern A. Zeeb 72898c230c8SBjoern A. Zeeb /* Swap the name and finish initialization of interface <n>b. */ 72904a32b80SAlexander V. Chernikov dp = name + strlen(name) - 1; 73098c230c8SBjoern A. Zeeb *dp = 'b'; 73198c230c8SBjoern A. Zeeb 73212aeeb91SAlexander V. Chernikov epair_setup_ifp(scb, name, unit); 73312aeeb91SAlexander V. Chernikov 73498c230c8SBjoern A. Zeeb ifp = scb->ifp; 73598c230c8SBjoern A. Zeeb /* We need to play some tricks here for the second interface. */ 73642a58907SGleb Smirnoff strlcpy(name, epairname, len); 737b02fd8b7SKristof Provost /* Correctly set the name for the cloner list. */ 738b02fd8b7SKristof Provost strlcpy(name, scb->ifp->if_xname, len); 739b02fd8b7SKristof Provost 74012aeeb91SAlexander V. Chernikov epair_clone_add(ifc, scb); 74198c230c8SBjoern A. Zeeb 74298c230c8SBjoern A. Zeeb /* 74398c230c8SBjoern A. Zeeb * Restore name to <n>a as the ifp for this will go into the 74498c230c8SBjoern A. Zeeb * cloner list for the initial call. 74598c230c8SBjoern A. Zeeb */ 74698c230c8SBjoern A. Zeeb strlcpy(name, sca->ifp->if_xname, len); 74798c230c8SBjoern A. Zeeb 74898c230c8SBjoern A. Zeeb /* Tell the world, that we are ready to rock. */ 74904a32b80SAlexander V. Chernikov epair_set_state(sca->ifp, true); 75004a32b80SAlexander V. Chernikov epair_set_state(scb->ifp, true); 75198c230c8SBjoern A. Zeeb 75291ebcbe0SAlexander V. Chernikov *ifpp = sca->ifp; 75391ebcbe0SAlexander V. Chernikov 75498c230c8SBjoern A. Zeeb return (0); 75598c230c8SBjoern A. Zeeb } 75698c230c8SBjoern A. Zeeb 7573dd5760aSBjoern A. Zeeb static void 7583dd5760aSBjoern A. Zeeb epair_drain_rings(struct epair_softc *sc) 7593dd5760aSBjoern A. Zeeb { 76024f0bfbaSKristof Provost for (int i = 0; i < sc->num_queues; i++) { 761df7bbd8cSMark Johnston struct epair_queue *q; 762df7bbd8cSMark Johnston struct mbuf *m, *n; 763df7bbd8cSMark Johnston 764df7bbd8cSMark Johnston q = &sc->queues[i]; 765df7bbd8cSMark Johnston mtx_lock(&q->mtx); 766df7bbd8cSMark Johnston m = mbufq_flush(&q->q); 767df7bbd8cSMark Johnston mtx_unlock(&q->mtx); 768df7bbd8cSMark Johnston 769df7bbd8cSMark Johnston for (; m != NULL; m = n) { 770df7bbd8cSMark Johnston n = m->m_nextpkt; 7713dd5760aSBjoern A. Zeeb m_freem(m); 7723dd5760aSBjoern A. Zeeb } 7733dd5760aSBjoern A. Zeeb } 77424f0bfbaSKristof Provost } 7753dd5760aSBjoern A. Zeeb 77698c230c8SBjoern A. Zeeb static int 77791ebcbe0SAlexander V. Chernikov epair_clone_destroy(struct if_clone *ifc, struct ifnet *ifp, uint32_t flags) 77898c230c8SBjoern A. Zeeb { 77998c230c8SBjoern A. Zeeb struct ifnet *oifp; 78098c230c8SBjoern A. Zeeb struct epair_softc *sca, *scb; 78198c230c8SBjoern A. Zeeb int unit, error; 78298c230c8SBjoern A. Zeeb 78398c230c8SBjoern A. Zeeb /* 78498c230c8SBjoern A. Zeeb * In case we called into if_clone_destroyif() ourselves 78598c230c8SBjoern A. Zeeb * again to remove the second interface, the softc will be 78698c230c8SBjoern A. Zeeb * NULL. In that case so not do anything but return success. 78798c230c8SBjoern A. Zeeb */ 78898c230c8SBjoern A. Zeeb if (ifp->if_softc == NULL) 78998c230c8SBjoern A. Zeeb return (0); 79098c230c8SBjoern A. Zeeb 79198c230c8SBjoern A. Zeeb unit = ifp->if_dunit; 79298c230c8SBjoern A. Zeeb sca = ifp->if_softc; 79398c230c8SBjoern A. Zeeb oifp = sca->oifp; 79498c230c8SBjoern A. Zeeb scb = oifp->if_softc; 79598c230c8SBjoern A. Zeeb 7963dd5760aSBjoern A. Zeeb /* Frist get the interfaces down and detached. */ 79704a32b80SAlexander V. Chernikov epair_set_state(ifp, false); 79804a32b80SAlexander V. Chernikov epair_set_state(oifp, false); 79998c230c8SBjoern A. Zeeb 8003dd5760aSBjoern A. Zeeb ether_ifdetach(ifp); 8017edc3d88SMikolaj Golub ether_ifdetach(oifp); 8023dd5760aSBjoern A. Zeeb 8033dd5760aSBjoern A. Zeeb /* Third free any queued packets and all the resources. */ 8043dd5760aSBjoern A. Zeeb CURVNET_SET_QUIET(oifp->if_vnet); 8053dd5760aSBjoern A. Zeeb epair_drain_rings(scb); 80698c230c8SBjoern A. Zeeb oifp->if_softc = NULL; 80798c230c8SBjoern A. Zeeb error = if_clone_destroyif(ifc, oifp); 80898c230c8SBjoern A. Zeeb if (error) 80998c230c8SBjoern A. Zeeb panic("%s: if_clone_destroyif() for our 2nd iface failed: %d", 81098c230c8SBjoern A. Zeeb __func__, error); 81112aeeb91SAlexander V. Chernikov epair_free_sc(scb); 8127edc3d88SMikolaj Golub CURVNET_RESTORE(); 8137edc3d88SMikolaj Golub 8143dd5760aSBjoern A. Zeeb epair_drain_rings(sca); 81512aeeb91SAlexander V. Chernikov epair_free_sc(sca); 8163dd5760aSBjoern A. Zeeb 8173dd5760aSBjoern A. Zeeb /* Last free the cloner unit. */ 81898c230c8SBjoern A. Zeeb ifc_free_unit(ifc, unit); 81998c230c8SBjoern A. Zeeb 82098c230c8SBjoern A. Zeeb return (0); 82198c230c8SBjoern A. Zeeb } 82298c230c8SBjoern A. Zeeb 8233c3136b1SHiroki Sato static void 8243c3136b1SHiroki Sato vnet_epair_init(const void *unused __unused) 8253c3136b1SHiroki Sato { 82691ebcbe0SAlexander V. Chernikov struct if_clone_addreq req = { 82791ebcbe0SAlexander V. Chernikov .match_f = epair_clone_match, 82891ebcbe0SAlexander V. Chernikov .create_f = epair_clone_create, 82991ebcbe0SAlexander V. Chernikov .destroy_f = epair_clone_destroy, 83091ebcbe0SAlexander V. Chernikov }; 83191ebcbe0SAlexander V. Chernikov V_epair_cloner = ifc_attach_cloner(epairname, &req); 8323c3136b1SHiroki Sato } 83389856f7eSBjoern A. Zeeb VNET_SYSINIT(vnet_epair_init, SI_SUB_PSEUDO, SI_ORDER_ANY, 8343c3136b1SHiroki Sato vnet_epair_init, NULL); 8353c3136b1SHiroki Sato 8363c3136b1SHiroki Sato static void 8373c3136b1SHiroki Sato vnet_epair_uninit(const void *unused __unused) 8383c3136b1SHiroki Sato { 8393c3136b1SHiroki Sato 84091ebcbe0SAlexander V. Chernikov ifc_detach_cloner(V_epair_cloner); 8413c3136b1SHiroki Sato } 84289856f7eSBjoern A. Zeeb VNET_SYSUNINIT(vnet_epair_uninit, SI_SUB_INIT_IF, SI_ORDER_ANY, 8433c3136b1SHiroki Sato vnet_epair_uninit, NULL); 8443c3136b1SHiroki Sato 84598c230c8SBjoern A. Zeeb static int 8467442b632SLi-Wen Hsu epair_mod_init(void) 84724f0bfbaSKristof Provost { 84824f0bfbaSKristof Provost char name[32]; 84924f0bfbaSKristof Provost epair_tasks.tasks = 0; 85024f0bfbaSKristof Provost 85124f0bfbaSKristof Provost #ifdef RSS 85224f0bfbaSKristof Provost int cpu; 85324f0bfbaSKristof Provost 85424f0bfbaSKristof Provost CPU_FOREACH(cpu) { 85524f0bfbaSKristof Provost cpuset_t cpu_mask; 85624f0bfbaSKristof Provost 85724f0bfbaSKristof Provost /* Pin to this CPU so we get appropriate NUMA allocations. */ 85824f0bfbaSKristof Provost thread_lock(curthread); 85924f0bfbaSKristof Provost sched_bind(curthread, cpu); 86024f0bfbaSKristof Provost thread_unlock(curthread); 86124f0bfbaSKristof Provost 86224f0bfbaSKristof Provost snprintf(name, sizeof(name), "epair_task_%d", cpu); 86324f0bfbaSKristof Provost 86424f0bfbaSKristof Provost epair_tasks.tq[cpu] = taskqueue_create(name, M_WAITOK, 86524f0bfbaSKristof Provost taskqueue_thread_enqueue, 86624f0bfbaSKristof Provost &epair_tasks.tq[cpu]); 86724f0bfbaSKristof Provost CPU_SETOF(cpu, &cpu_mask); 86824f0bfbaSKristof Provost taskqueue_start_threads_cpuset(&epair_tasks.tq[cpu], 1, PI_NET, 86924f0bfbaSKristof Provost &cpu_mask, "%s", name); 87024f0bfbaSKristof Provost 87124f0bfbaSKristof Provost epair_tasks.tasks++; 87224f0bfbaSKristof Provost } 873cbbce423SKristof Provost thread_lock(curthread); 874cbbce423SKristof Provost sched_unbind(curthread); 875cbbce423SKristof Provost thread_unlock(curthread); 87624f0bfbaSKristof Provost #else 87724f0bfbaSKristof Provost snprintf(name, sizeof(name), "epair_task"); 87824f0bfbaSKristof Provost 87924f0bfbaSKristof Provost epair_tasks.tq[0] = taskqueue_create(name, M_WAITOK, 88024f0bfbaSKristof Provost taskqueue_thread_enqueue, 88124f0bfbaSKristof Provost &epair_tasks.tq[0]); 88224f0bfbaSKristof Provost taskqueue_start_threads(&epair_tasks.tq[0], 1, PI_NET, "%s", name); 88324f0bfbaSKristof Provost 88424f0bfbaSKristof Provost epair_tasks.tasks = 1; 88524f0bfbaSKristof Provost #endif 88624f0bfbaSKristof Provost 88724f0bfbaSKristof Provost return (0); 88824f0bfbaSKristof Provost } 88924f0bfbaSKristof Provost 89024f0bfbaSKristof Provost static void 8917442b632SLi-Wen Hsu epair_mod_cleanup(void) 89224f0bfbaSKristof Provost { 89324f0bfbaSKristof Provost 89424f0bfbaSKristof Provost for (int i = 0; i < epair_tasks.tasks; i++) { 89524f0bfbaSKristof Provost taskqueue_drain_all(epair_tasks.tq[i]); 89624f0bfbaSKristof Provost taskqueue_free(epair_tasks.tq[i]); 89724f0bfbaSKristof Provost } 89824f0bfbaSKristof Provost } 89924f0bfbaSKristof Provost 90024f0bfbaSKristof Provost static int 90198c230c8SBjoern A. Zeeb epair_modevent(module_t mod, int type, void *data) 90298c230c8SBjoern A. Zeeb { 90324f0bfbaSKristof Provost int ret; 90498c230c8SBjoern A. Zeeb 90598c230c8SBjoern A. Zeeb switch (type) { 90698c230c8SBjoern A. Zeeb case MOD_LOAD: 9073dd5760aSBjoern A. Zeeb EPAIR_LOCK_INIT(); 90824f0bfbaSKristof Provost ret = epair_mod_init(); 90924f0bfbaSKristof Provost if (ret != 0) 91024f0bfbaSKristof Provost return (ret); 91198c230c8SBjoern A. Zeeb if (bootverbose) 9123dd5760aSBjoern A. Zeeb printf("%s: %s initialized.\n", __func__, epairname); 91398c230c8SBjoern A. Zeeb break; 91498c230c8SBjoern A. Zeeb case MOD_UNLOAD: 91524f0bfbaSKristof Provost epair_mod_cleanup(); 9163dd5760aSBjoern A. Zeeb EPAIR_LOCK_DESTROY(); 9173dd5760aSBjoern A. Zeeb if (bootverbose) 9183dd5760aSBjoern A. Zeeb printf("%s: %s unloaded.\n", __func__, epairname); 91998c230c8SBjoern A. Zeeb break; 92098c230c8SBjoern A. Zeeb default: 92198c230c8SBjoern A. Zeeb return (EOPNOTSUPP); 92298c230c8SBjoern A. Zeeb } 92398c230c8SBjoern A. Zeeb return (0); 92498c230c8SBjoern A. Zeeb } 92598c230c8SBjoern A. Zeeb 92698c230c8SBjoern A. Zeeb static moduledata_t epair_mod = { 92798c230c8SBjoern A. Zeeb "if_epair", 92898c230c8SBjoern A. Zeeb epair_modevent, 9299823d527SKevin Lo 0 93098c230c8SBjoern A. Zeeb }; 93198c230c8SBjoern A. Zeeb 93289856f7eSBjoern A. Zeeb DECLARE_MODULE(if_epair, epair_mod, SI_SUB_PSEUDO, SI_ORDER_MIDDLE); 9333dd5760aSBjoern A. Zeeb MODULE_VERSION(if_epair, 3); 934