1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2008 The FreeBSD Foundation 5 * Copyright (c) 2009-2010 Bjoern A. Zeeb <bz@FreeBSD.org> 6 * All rights reserved. 7 * 8 * This software was developed by CK Software GmbH under sponsorship 9 * from the FreeBSD Foundation. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 /* 34 * A pair of virtual back-to-back connected ethernet like interfaces 35 * (``two interfaces with a virtual cross-over cable''). 36 * 37 * This is mostly intended to be used to provide connectivity between 38 * different virtual network stack instances. 39 */ 40 /* 41 * Things to re-think once we have more experience: 42 * - ifp->if_reassign function once we can test with vimage. Depending on 43 * how if_vmove() is going to be improved. 44 * - Real random etheraddrs that are checked to be uniquish; we would need 45 * to re-do them in case we move the interface between network stacks 46 * in a private if_reassign function. 47 * In case we bridge to a real interface/network or between indepedent 48 * epairs on multiple stacks/machines, we may need this. 49 * For now let the user handle that case. 50 */ 51 52 #include <sys/cdefs.h> 53 __FBSDID("$FreeBSD$"); 54 55 #include <sys/param.h> 56 #include <sys/hash.h> 57 #include <sys/jail.h> 58 #include <sys/kernel.h> 59 #include <sys/libkern.h> 60 #include <sys/malloc.h> 61 #include <sys/mbuf.h> 62 #include <sys/module.h> 63 #include <sys/proc.h> 64 #include <sys/refcount.h> 65 #include <sys/queue.h> 66 #include <sys/smp.h> 67 #include <sys/socket.h> 68 #include <sys/sockio.h> 69 #include <sys/sysctl.h> 70 #include <sys/types.h> 71 72 #include <net/bpf.h> 73 #include <net/ethernet.h> 74 #include <net/if.h> 75 #include <net/if_var.h> 76 #include <net/if_clone.h> 77 #include <net/if_media.h> 78 #include <net/if_var.h> 79 #include <net/if_types.h> 80 #include <net/netisr.h> 81 #include <net/vnet.h> 82 83 SYSCTL_DECL(_net_link); 84 static SYSCTL_NODE(_net_link, OID_AUTO, epair, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 85 "epair sysctl"); 86 87 #ifdef EPAIR_DEBUG 88 static int epair_debug = 0; 89 SYSCTL_INT(_net_link_epair, OID_AUTO, epair_debug, CTLFLAG_RW, 90 &epair_debug, 0, "if_epair(4) debugging."); 91 #define DPRINTF(fmt, arg...) \ 92 if (epair_debug) \ 93 printf("[%s:%d] " fmt, __func__, __LINE__, ##arg) 94 #else 95 #define DPRINTF(fmt, arg...) 96 #endif 97 98 static void epair_nh_sintr(struct mbuf *); 99 static struct mbuf *epair_nh_m2cpuid(struct mbuf *, uintptr_t, u_int *); 100 static void epair_nh_drainedcpu(u_int); 101 102 static void epair_start_locked(struct ifnet *); 103 static int epair_media_change(struct ifnet *); 104 static void epair_media_status(struct ifnet *, struct ifmediareq *); 105 106 static int epair_clone_match(struct if_clone *, const char *); 107 static int epair_clone_create(struct if_clone *, char *, size_t, caddr_t); 108 static int epair_clone_destroy(struct if_clone *, struct ifnet *); 109 110 static const char epairname[] = "epair"; 111 static unsigned int next_index = 0; 112 113 /* Netisr related definitions and sysctl. */ 114 static struct netisr_handler epair_nh = { 115 .nh_name = epairname, 116 .nh_proto = NETISR_EPAIR, 117 .nh_policy = NETISR_POLICY_CPU, 118 .nh_handler = epair_nh_sintr, 119 .nh_m2cpuid = epair_nh_m2cpuid, 120 .nh_drainedcpu = epair_nh_drainedcpu, 121 }; 122 123 static int 124 sysctl_epair_netisr_maxqlen(SYSCTL_HANDLER_ARGS) 125 { 126 int error, qlimit; 127 128 netisr_getqlimit(&epair_nh, &qlimit); 129 error = sysctl_handle_int(oidp, &qlimit, 0, req); 130 if (error || !req->newptr) 131 return (error); 132 if (qlimit < 1) 133 return (EINVAL); 134 return (netisr_setqlimit(&epair_nh, qlimit)); 135 } 136 SYSCTL_PROC(_net_link_epair, OID_AUTO, netisr_maxqlen, 137 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0, 138 sysctl_epair_netisr_maxqlen, "I", 139 "Maximum if_epair(4) netisr \"hw\" queue length"); 140 141 struct epair_softc { 142 struct ifnet *ifp; /* This ifp. */ 143 struct ifnet *oifp; /* other ifp of pair. */ 144 struct ifmedia media; /* Media config (fake). */ 145 u_int refcount; /* # of mbufs in flight. */ 146 u_int cpuid; /* CPU ID assigned upon creation. */ 147 void (*if_qflush)(struct ifnet *); 148 /* Original if_qflush routine. */ 149 }; 150 151 /* 152 * Per-CPU list of ifps with data in the ifq that needs to be flushed 153 * to the netisr ``hw'' queue before we allow any further direct queuing 154 * to the ``hw'' queue. 155 */ 156 struct epair_ifp_drain { 157 STAILQ_ENTRY(epair_ifp_drain) ifp_next; 158 struct ifnet *ifp; 159 }; 160 STAILQ_HEAD(eid_list, epair_ifp_drain); 161 162 #define EPAIR_LOCK_INIT(dpcpu) mtx_init(&(dpcpu)->if_epair_mtx, \ 163 "if_epair", NULL, MTX_DEF) 164 #define EPAIR_LOCK_DESTROY(dpcpu) mtx_destroy(&(dpcpu)->if_epair_mtx) 165 #define EPAIR_LOCK_ASSERT(dpcpu) mtx_assert(&(dpcpu)->if_epair_mtx, \ 166 MA_OWNED) 167 #define EPAIR_LOCK(dpcpu) mtx_lock(&(dpcpu)->if_epair_mtx) 168 #define EPAIR_UNLOCK(dpcpu) mtx_unlock(&(dpcpu)->if_epair_mtx) 169 170 #ifdef INVARIANTS 171 #define EPAIR_REFCOUNT_INIT(r, v) refcount_init((r), (v)) 172 #define EPAIR_REFCOUNT_AQUIRE(r) refcount_acquire((r)) 173 #define EPAIR_REFCOUNT_RELEASE(r) refcount_release((r)) 174 #define EPAIR_REFCOUNT_ASSERT(a, p) KASSERT(a, p) 175 #else 176 #define EPAIR_REFCOUNT_INIT(r, v) 177 #define EPAIR_REFCOUNT_AQUIRE(r) 178 #define EPAIR_REFCOUNT_RELEASE(r) 179 #define EPAIR_REFCOUNT_ASSERT(a, p) 180 #endif 181 182 static MALLOC_DEFINE(M_EPAIR, epairname, 183 "Pair of virtual cross-over connected Ethernet-like interfaces"); 184 185 VNET_DEFINE_STATIC(struct if_clone *, epair_cloner); 186 #define V_epair_cloner VNET(epair_cloner) 187 188 /* 189 * DPCPU area and functions. 190 */ 191 struct epair_dpcpu { 192 struct mtx if_epair_mtx; /* Per-CPU locking. */ 193 int epair_drv_flags; /* Per-CPU ``hw'' drv flags. */ 194 struct eid_list epair_ifp_drain_list; /* Per-CPU list of ifps with 195 * data in the ifq. */ 196 }; 197 DPCPU_DEFINE(struct epair_dpcpu, epair_dpcpu); 198 199 static void 200 epair_dpcpu_init(void) 201 { 202 struct epair_dpcpu *epair_dpcpu; 203 struct eid_list *s; 204 u_int cpuid; 205 206 CPU_FOREACH(cpuid) { 207 epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); 208 209 /* Initialize per-cpu lock. */ 210 EPAIR_LOCK_INIT(epair_dpcpu); 211 212 /* Driver flags are per-cpu as are our netisr "hw" queues. */ 213 epair_dpcpu->epair_drv_flags = 0; 214 215 /* 216 * Initialize per-cpu drain list. 217 * Manually do what STAILQ_HEAD_INITIALIZER would do. 218 */ 219 s = &epair_dpcpu->epair_ifp_drain_list; 220 s->stqh_first = NULL; 221 s->stqh_last = &s->stqh_first; 222 } 223 } 224 225 static void 226 epair_dpcpu_detach(void) 227 { 228 struct epair_dpcpu *epair_dpcpu; 229 u_int cpuid; 230 231 CPU_FOREACH(cpuid) { 232 epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); 233 234 /* Destroy per-cpu lock. */ 235 EPAIR_LOCK_DESTROY(epair_dpcpu); 236 } 237 } 238 239 /* 240 * Helper functions. 241 */ 242 static u_int 243 cpuid_from_ifp(struct ifnet *ifp) 244 { 245 struct epair_softc *sc; 246 247 if (ifp == NULL) 248 return (0); 249 sc = ifp->if_softc; 250 251 return (sc->cpuid); 252 } 253 254 /* 255 * Netisr handler functions. 256 */ 257 static void 258 epair_nh_sintr(struct mbuf *m) 259 { 260 struct ifnet *ifp; 261 struct epair_softc *sc __unused; 262 263 ifp = m->m_pkthdr.rcvif; 264 (*ifp->if_input)(ifp, m); 265 sc = ifp->if_softc; 266 EPAIR_REFCOUNT_RELEASE(&sc->refcount); 267 EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, 268 ("%s: ifp=%p sc->refcount not >= 1: %d", 269 __func__, ifp, sc->refcount)); 270 DPRINTF("ifp=%p refcount=%u\n", ifp, sc->refcount); 271 } 272 273 static struct mbuf * 274 epair_nh_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) 275 { 276 277 *cpuid = cpuid_from_ifp(m->m_pkthdr.rcvif); 278 279 return (m); 280 } 281 282 static void 283 epair_nh_drainedcpu(u_int cpuid) 284 { 285 struct epair_dpcpu *epair_dpcpu; 286 struct epair_ifp_drain *elm, *tvar; 287 struct ifnet *ifp; 288 289 epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); 290 EPAIR_LOCK(epair_dpcpu); 291 /* 292 * Assume our "hw" queue and possibly ifq will be emptied 293 * again. In case we will overflow the "hw" queue while 294 * draining, epair_start_locked will set IFF_DRV_OACTIVE 295 * again and we will stop and return. 296 */ 297 STAILQ_FOREACH_SAFE(elm, &epair_dpcpu->epair_ifp_drain_list, 298 ifp_next, tvar) { 299 ifp = elm->ifp; 300 epair_dpcpu->epair_drv_flags &= ~IFF_DRV_OACTIVE; 301 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 302 epair_start_locked(ifp); 303 304 IFQ_LOCK(&ifp->if_snd); 305 if (IFQ_IS_EMPTY(&ifp->if_snd)) { 306 struct epair_softc *sc __unused; 307 308 STAILQ_REMOVE(&epair_dpcpu->epair_ifp_drain_list, 309 elm, epair_ifp_drain, ifp_next); 310 /* The cached ifp goes off the list. */ 311 sc = ifp->if_softc; 312 EPAIR_REFCOUNT_RELEASE(&sc->refcount); 313 EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, 314 ("%s: ifp=%p sc->refcount not >= 1: %d", 315 __func__, ifp, sc->refcount)); 316 free(elm, M_EPAIR); 317 } 318 IFQ_UNLOCK(&ifp->if_snd); 319 320 if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) != 0) { 321 /* Our "hw"q overflew again. */ 322 epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE; 323 DPRINTF("hw queue length overflow at %u\n", 324 epair_nh.nh_qlimit); 325 break; 326 } 327 } 328 EPAIR_UNLOCK(epair_dpcpu); 329 } 330 331 /* 332 * Network interface (`if') related functions. 333 */ 334 static void 335 epair_remove_ifp_from_draining(struct ifnet *ifp) 336 { 337 struct epair_dpcpu *epair_dpcpu; 338 struct epair_ifp_drain *elm, *tvar; 339 u_int cpuid; 340 341 CPU_FOREACH(cpuid) { 342 epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); 343 EPAIR_LOCK(epair_dpcpu); 344 STAILQ_FOREACH_SAFE(elm, &epair_dpcpu->epair_ifp_drain_list, 345 ifp_next, tvar) { 346 if (ifp == elm->ifp) { 347 struct epair_softc *sc __unused; 348 349 STAILQ_REMOVE( 350 &epair_dpcpu->epair_ifp_drain_list, elm, 351 epair_ifp_drain, ifp_next); 352 /* The cached ifp goes off the list. */ 353 sc = ifp->if_softc; 354 EPAIR_REFCOUNT_RELEASE(&sc->refcount); 355 EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, 356 ("%s: ifp=%p sc->refcount not >= 1: %d", 357 __func__, ifp, sc->refcount)); 358 free(elm, M_EPAIR); 359 } 360 } 361 EPAIR_UNLOCK(epair_dpcpu); 362 } 363 } 364 365 static int 366 epair_add_ifp_for_draining(struct ifnet *ifp) 367 { 368 struct epair_dpcpu *epair_dpcpu; 369 struct epair_softc *sc; 370 struct epair_ifp_drain *elm = NULL; 371 372 sc = ifp->if_softc; 373 epair_dpcpu = DPCPU_ID_PTR(sc->cpuid, epair_dpcpu); 374 EPAIR_LOCK_ASSERT(epair_dpcpu); 375 STAILQ_FOREACH(elm, &epair_dpcpu->epair_ifp_drain_list, ifp_next) 376 if (elm->ifp == ifp) 377 break; 378 /* If the ifp is there already, return success. */ 379 if (elm != NULL) 380 return (0); 381 382 elm = malloc(sizeof(struct epair_ifp_drain), M_EPAIR, M_NOWAIT|M_ZERO); 383 if (elm == NULL) 384 return (ENOMEM); 385 386 elm->ifp = ifp; 387 /* Add a reference for the ifp pointer on the list. */ 388 EPAIR_REFCOUNT_AQUIRE(&sc->refcount); 389 STAILQ_INSERT_TAIL(&epair_dpcpu->epair_ifp_drain_list, elm, ifp_next); 390 391 return (0); 392 } 393 394 static void 395 epair_start_locked(struct ifnet *ifp) 396 { 397 struct epair_dpcpu *epair_dpcpu; 398 struct mbuf *m; 399 struct epair_softc *sc; 400 struct ifnet *oifp; 401 int error; 402 403 DPRINTF("ifp=%p\n", ifp); 404 sc = ifp->if_softc; 405 epair_dpcpu = DPCPU_ID_PTR(sc->cpuid, epair_dpcpu); 406 EPAIR_LOCK_ASSERT(epair_dpcpu); 407 408 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 409 return; 410 if ((ifp->if_flags & IFF_UP) == 0) 411 return; 412 413 /* 414 * We get packets here from ether_output via if_handoff() 415 * and need to put them into the input queue of the oifp 416 * and call oifp->if_input() via netisr/epair_sintr(). 417 */ 418 oifp = sc->oifp; 419 sc = oifp->if_softc; 420 for (;;) { 421 IFQ_DEQUEUE(&ifp->if_snd, m); 422 if (m == NULL) 423 break; 424 BPF_MTAP(ifp, m); 425 426 /* 427 * In case the outgoing interface is not usable, 428 * drop the packet. 429 */ 430 if ((oifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 431 (oifp->if_flags & IFF_UP) ==0) { 432 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 433 m_freem(m); 434 continue; 435 } 436 DPRINTF("packet %s -> %s\n", ifp->if_xname, oifp->if_xname); 437 438 /* 439 * Add a reference so the interface cannot go while the 440 * packet is in transit as we rely on rcvif to stay valid. 441 */ 442 EPAIR_REFCOUNT_AQUIRE(&sc->refcount); 443 m->m_pkthdr.rcvif = oifp; 444 CURVNET_SET_QUIET(oifp->if_vnet); 445 error = netisr_queue(NETISR_EPAIR, m); 446 CURVNET_RESTORE(); 447 if (!error) { 448 if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); 449 /* Someone else received the packet. */ 450 if_inc_counter(oifp, IFCOUNTER_IPACKETS, 1); 451 } else { 452 /* The packet was freed already. */ 453 epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE; 454 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 455 (void) epair_add_ifp_for_draining(ifp); 456 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 457 EPAIR_REFCOUNT_RELEASE(&sc->refcount); 458 EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, 459 ("%s: ifp=%p sc->refcount not >= 1: %d", 460 __func__, oifp, sc->refcount)); 461 } 462 } 463 } 464 465 static void 466 epair_start(struct ifnet *ifp) 467 { 468 struct epair_dpcpu *epair_dpcpu; 469 470 epair_dpcpu = DPCPU_ID_PTR(cpuid_from_ifp(ifp), epair_dpcpu); 471 EPAIR_LOCK(epair_dpcpu); 472 epair_start_locked(ifp); 473 EPAIR_UNLOCK(epair_dpcpu); 474 } 475 476 static int 477 epair_transmit_locked(struct ifnet *ifp, struct mbuf *m) 478 { 479 struct epair_dpcpu *epair_dpcpu; 480 struct epair_softc *sc; 481 struct ifnet *oifp; 482 int error, len; 483 short mflags; 484 485 DPRINTF("ifp=%p m=%p\n", ifp, m); 486 sc = ifp->if_softc; 487 epair_dpcpu = DPCPU_ID_PTR(sc->cpuid, epair_dpcpu); 488 EPAIR_LOCK_ASSERT(epair_dpcpu); 489 490 if (m == NULL) 491 return (0); 492 493 /* 494 * We are not going to use the interface en/dequeue mechanism 495 * on the TX side. We are called from ether_output_frame() 496 * and will put the packet into the incoming queue of the 497 * other interface of our pair via the netsir. 498 */ 499 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 500 m_freem(m); 501 return (ENXIO); 502 } 503 if ((ifp->if_flags & IFF_UP) == 0) { 504 m_freem(m); 505 return (ENETDOWN); 506 } 507 508 BPF_MTAP(ifp, m); 509 510 /* 511 * In case the outgoing interface is not usable, 512 * drop the packet. 513 */ 514 oifp = sc->oifp; 515 if ((oifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 516 (oifp->if_flags & IFF_UP) ==0) { 517 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 518 m_freem(m); 519 return (0); 520 } 521 len = m->m_pkthdr.len; 522 mflags = m->m_flags; 523 DPRINTF("packet %s -> %s\n", ifp->if_xname, oifp->if_xname); 524 525 #ifdef ALTQ 526 /* Support ALTQ via the classic if_start() path. */ 527 IF_LOCK(&ifp->if_snd); 528 if (ALTQ_IS_ENABLED(&ifp->if_snd)) { 529 ALTQ_ENQUEUE(&ifp->if_snd, m, NULL, error); 530 if (error) 531 if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); 532 IF_UNLOCK(&ifp->if_snd); 533 if (!error) { 534 if_inc_counter(ifp, IFCOUNTER_OBYTES, len); 535 if (mflags & (M_BCAST|M_MCAST)) 536 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); 537 538 if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) 539 epair_start_locked(ifp); 540 else 541 (void)epair_add_ifp_for_draining(ifp); 542 } 543 return (error); 544 } 545 IF_UNLOCK(&ifp->if_snd); 546 #endif 547 548 if ((epair_dpcpu->epair_drv_flags & IFF_DRV_OACTIVE) != 0) { 549 /* 550 * Our hardware queue is full, try to fall back 551 * queuing to the ifq but do not call ifp->if_start. 552 * Either we are lucky or the packet is gone. 553 */ 554 IFQ_ENQUEUE(&ifp->if_snd, m, error); 555 if (!error) 556 (void)epair_add_ifp_for_draining(ifp); 557 return (error); 558 } 559 sc = oifp->if_softc; 560 /* 561 * Add a reference so the interface cannot go while the 562 * packet is in transit as we rely on rcvif to stay valid. 563 */ 564 EPAIR_REFCOUNT_AQUIRE(&sc->refcount); 565 m->m_pkthdr.rcvif = oifp; 566 CURVNET_SET_QUIET(oifp->if_vnet); 567 error = netisr_queue(NETISR_EPAIR, m); 568 CURVNET_RESTORE(); 569 if (!error) { 570 if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); 571 /* 572 * IFQ_HANDOFF_ADJ/ip_handoff() update statistics, 573 * but as we bypass all this we have to duplicate 574 * the logic another time. 575 */ 576 if_inc_counter(ifp, IFCOUNTER_OBYTES, len); 577 if (mflags & (M_BCAST|M_MCAST)) 578 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); 579 /* Someone else received the packet. */ 580 if_inc_counter(oifp, IFCOUNTER_IPACKETS, 1); 581 } else { 582 /* The packet was freed already. */ 583 epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE; 584 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 585 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 586 EPAIR_REFCOUNT_RELEASE(&sc->refcount); 587 EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, 588 ("%s: ifp=%p sc->refcount not >= 1: %d", 589 __func__, oifp, sc->refcount)); 590 } 591 592 return (error); 593 } 594 595 static int 596 epair_transmit(struct ifnet *ifp, struct mbuf *m) 597 { 598 struct epair_dpcpu *epair_dpcpu; 599 int error; 600 601 epair_dpcpu = DPCPU_ID_PTR(cpuid_from_ifp(ifp), epair_dpcpu); 602 EPAIR_LOCK(epair_dpcpu); 603 error = epair_transmit_locked(ifp, m); 604 EPAIR_UNLOCK(epair_dpcpu); 605 return (error); 606 } 607 608 static void 609 epair_qflush(struct ifnet *ifp) 610 { 611 struct epair_softc *sc; 612 613 sc = ifp->if_softc; 614 615 /* 616 * See epair_clone_destroy(), we can end up getting called twice. 617 * Don't do anything on the second call. 618 */ 619 if (sc == NULL) 620 return; 621 622 /* 623 * Remove this ifp from all backpointer lists. The interface will not 624 * usable for flushing anyway nor should it have anything to flush 625 * after if_qflush(). 626 */ 627 epair_remove_ifp_from_draining(ifp); 628 629 if (sc->if_qflush) 630 sc->if_qflush(ifp); 631 } 632 633 static int 634 epair_media_change(struct ifnet *ifp __unused) 635 { 636 637 /* Do nothing. */ 638 return (0); 639 } 640 641 static void 642 epair_media_status(struct ifnet *ifp __unused, struct ifmediareq *imr) 643 { 644 645 imr->ifm_status = IFM_AVALID | IFM_ACTIVE; 646 imr->ifm_active = IFM_ETHER | IFM_10G_T | IFM_FDX; 647 } 648 649 static int 650 epair_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 651 { 652 struct epair_softc *sc; 653 struct ifreq *ifr; 654 int error; 655 656 ifr = (struct ifreq *)data; 657 switch (cmd) { 658 case SIOCSIFFLAGS: 659 case SIOCADDMULTI: 660 case SIOCDELMULTI: 661 error = 0; 662 break; 663 664 case SIOCSIFMEDIA: 665 case SIOCGIFMEDIA: 666 sc = ifp->if_softc; 667 error = ifmedia_ioctl(ifp, ifr, &sc->media, cmd); 668 break; 669 670 case SIOCSIFMTU: 671 /* We basically allow all kinds of MTUs. */ 672 ifp->if_mtu = ifr->ifr_mtu; 673 error = 0; 674 break; 675 676 default: 677 /* Let the common ethernet handler process this. */ 678 error = ether_ioctl(ifp, cmd, data); 679 break; 680 } 681 682 return (error); 683 } 684 685 static void 686 epair_init(void *dummy __unused) 687 { 688 } 689 690 /* 691 * Interface cloning functions. 692 * We use our private ones so that we can create/destroy our secondary 693 * device along with the primary one. 694 */ 695 static int 696 epair_clone_match(struct if_clone *ifc, const char *name) 697 { 698 const char *cp; 699 700 DPRINTF("name='%s'\n", name); 701 702 /* 703 * Our base name is epair. 704 * Our interfaces will be named epair<n>[ab]. 705 * So accept anything of the following list: 706 * - epair 707 * - epair<n> 708 * but not the epair<n>[ab] versions. 709 */ 710 if (strncmp(epairname, name, sizeof(epairname)-1) != 0) 711 return (0); 712 713 for (cp = name + sizeof(epairname) - 1; *cp != '\0'; cp++) { 714 if (*cp < '0' || *cp > '9') 715 return (0); 716 } 717 718 return (1); 719 } 720 721 static void 722 epair_clone_add(struct if_clone *ifc, struct epair_softc *scb) 723 { 724 struct ifnet *ifp; 725 uint8_t eaddr[ETHER_ADDR_LEN]; /* 00:00:00:00:00:00 */ 726 727 ifp = scb->ifp; 728 /* Copy epairNa etheraddr and change the last byte. */ 729 memcpy(eaddr, scb->oifp->if_hw_addr, ETHER_ADDR_LEN); 730 eaddr[5] = 0x0b; 731 ether_ifattach(ifp, eaddr); 732 733 if_clone_addif(ifc, ifp); 734 } 735 736 static int 737 epair_clone_create(struct if_clone *ifc, char *name, size_t len, caddr_t params) 738 { 739 struct epair_softc *sca, *scb; 740 struct ifnet *ifp; 741 char *dp; 742 int error, unit, wildcard; 743 uint64_t hostid; 744 uint32_t key[3]; 745 uint32_t hash; 746 uint8_t eaddr[ETHER_ADDR_LEN]; /* 00:00:00:00:00:00 */ 747 748 /* Try to see if a special unit was requested. */ 749 error = ifc_name2unit(name, &unit); 750 if (error != 0) 751 return (error); 752 wildcard = (unit < 0); 753 754 error = ifc_alloc_unit(ifc, &unit); 755 if (error != 0) 756 return (error); 757 758 /* 759 * If no unit had been given, we need to adjust the ifName. 760 * Also make sure there is space for our extra [ab] suffix. 761 */ 762 for (dp = name; *dp != '\0'; dp++); 763 if (wildcard) { 764 error = snprintf(dp, len - (dp - name), "%d", unit); 765 if (error > len - (dp - name) - 1) { 766 /* ifName too long. */ 767 ifc_free_unit(ifc, unit); 768 return (ENOSPC); 769 } 770 dp += error; 771 } 772 if (len - (dp - name) - 1 < 1) { 773 /* No space left for our [ab] suffix. */ 774 ifc_free_unit(ifc, unit); 775 return (ENOSPC); 776 } 777 *dp = 'b'; 778 /* Must not change dp so we can replace 'a' by 'b' later. */ 779 *(dp+1) = '\0'; 780 781 /* Check if 'a' and 'b' interfaces already exist. */ 782 if (ifunit(name) != NULL) 783 return (EEXIST); 784 *dp = 'a'; 785 if (ifunit(name) != NULL) 786 return (EEXIST); 787 788 /* Allocate memory for both [ab] interfaces */ 789 sca = malloc(sizeof(struct epair_softc), M_EPAIR, M_WAITOK | M_ZERO); 790 EPAIR_REFCOUNT_INIT(&sca->refcount, 1); 791 sca->ifp = if_alloc(IFT_ETHER); 792 if (sca->ifp == NULL) { 793 free(sca, M_EPAIR); 794 ifc_free_unit(ifc, unit); 795 return (ENOSPC); 796 } 797 798 scb = malloc(sizeof(struct epair_softc), M_EPAIR, M_WAITOK | M_ZERO); 799 EPAIR_REFCOUNT_INIT(&scb->refcount, 1); 800 scb->ifp = if_alloc(IFT_ETHER); 801 if (scb->ifp == NULL) { 802 free(scb, M_EPAIR); 803 if_free(sca->ifp); 804 free(sca, M_EPAIR); 805 ifc_free_unit(ifc, unit); 806 return (ENOSPC); 807 } 808 809 /* 810 * Cross-reference the interfaces so we will be able to free both. 811 */ 812 sca->oifp = scb->ifp; 813 scb->oifp = sca->ifp; 814 815 /* 816 * Calculate the cpuid for netisr queueing based on the 817 * ifIndex of the interfaces. As long as we cannot configure 818 * this or use cpuset information easily we cannot guarantee 819 * cache locality but we can at least allow parallelism. 820 */ 821 sca->cpuid = 822 netisr_get_cpuid(sca->ifp->if_index); 823 scb->cpuid = 824 netisr_get_cpuid(scb->ifp->if_index); 825 826 /* Initialise pseudo media types. */ 827 ifmedia_init(&sca->media, 0, epair_media_change, epair_media_status); 828 ifmedia_add(&sca->media, IFM_ETHER | IFM_10G_T, 0, NULL); 829 ifmedia_set(&sca->media, IFM_ETHER | IFM_10G_T); 830 ifmedia_init(&scb->media, 0, epair_media_change, epair_media_status); 831 ifmedia_add(&scb->media, IFM_ETHER | IFM_10G_T, 0, NULL); 832 ifmedia_set(&scb->media, IFM_ETHER | IFM_10G_T); 833 834 /* Finish initialization of interface <n>a. */ 835 ifp = sca->ifp; 836 ifp->if_softc = sca; 837 strlcpy(ifp->if_xname, name, IFNAMSIZ); 838 ifp->if_dname = epairname; 839 ifp->if_dunit = unit; 840 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 841 ifp->if_capabilities = IFCAP_VLAN_MTU; 842 ifp->if_capenable = IFCAP_VLAN_MTU; 843 ifp->if_start = epair_start; 844 ifp->if_ioctl = epair_ioctl; 845 ifp->if_init = epair_init; 846 if_setsendqlen(ifp, ifqmaxlen); 847 if_setsendqready(ifp); 848 849 /* 850 * Calculate the etheraddr hashing the hostid and the 851 * interface index. The result would be hopefully unique. 852 * Note that the "a" component of an epair instance may get moved 853 * to a different VNET after creation. In that case its index 854 * will be freed and the index can get reused by new epair instance. 855 * Make sure we do not create same etheraddr again. 856 */ 857 getcredhostid(curthread->td_ucred, (unsigned long *)&hostid); 858 if (hostid == 0) 859 arc4rand(&hostid, sizeof(hostid), 0); 860 861 if (ifp->if_index > next_index) 862 next_index = ifp->if_index; 863 else 864 next_index++; 865 866 key[0] = (uint32_t)next_index; 867 key[1] = (uint32_t)(hostid & 0xffffffff); 868 key[2] = (uint32_t)((hostid >> 32) & 0xfffffffff); 869 hash = jenkins_hash32(key, 3, 0); 870 871 eaddr[0] = 0x02; 872 memcpy(&eaddr[1], &hash, 4); 873 eaddr[5] = 0x0a; 874 ether_ifattach(ifp, eaddr); 875 sca->if_qflush = ifp->if_qflush; 876 ifp->if_qflush = epair_qflush; 877 ifp->if_transmit = epair_transmit; 878 ifp->if_baudrate = IF_Gbps(10); /* arbitrary maximum */ 879 880 /* Swap the name and finish initialization of interface <n>b. */ 881 *dp = 'b'; 882 883 ifp = scb->ifp; 884 ifp->if_softc = scb; 885 strlcpy(ifp->if_xname, name, IFNAMSIZ); 886 ifp->if_dname = epairname; 887 ifp->if_dunit = unit; 888 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 889 ifp->if_capabilities = IFCAP_VLAN_MTU; 890 ifp->if_capenable = IFCAP_VLAN_MTU; 891 ifp->if_start = epair_start; 892 ifp->if_ioctl = epair_ioctl; 893 ifp->if_init = epair_init; 894 if_setsendqlen(ifp, ifqmaxlen); 895 if_setsendqready(ifp); 896 /* We need to play some tricks here for the second interface. */ 897 strlcpy(name, epairname, len); 898 899 /* Correctly set the name for the cloner list. */ 900 strlcpy(name, scb->ifp->if_xname, len); 901 epair_clone_add(ifc, scb); 902 903 scb->if_qflush = ifp->if_qflush; 904 ifp->if_qflush = epair_qflush; 905 ifp->if_transmit = epair_transmit; 906 ifp->if_baudrate = IF_Gbps(10); /* arbitrary maximum */ 907 908 /* 909 * Restore name to <n>a as the ifp for this will go into the 910 * cloner list for the initial call. 911 */ 912 strlcpy(name, sca->ifp->if_xname, len); 913 DPRINTF("name='%s/%db' created sca=%p scb=%p\n", name, unit, sca, scb); 914 915 /* Tell the world, that we are ready to rock. */ 916 sca->ifp->if_drv_flags |= IFF_DRV_RUNNING; 917 scb->ifp->if_drv_flags |= IFF_DRV_RUNNING; 918 if_link_state_change(sca->ifp, LINK_STATE_UP); 919 if_link_state_change(scb->ifp, LINK_STATE_UP); 920 921 return (0); 922 } 923 924 static int 925 epair_clone_destroy(struct if_clone *ifc, struct ifnet *ifp) 926 { 927 struct ifnet *oifp; 928 struct epair_softc *sca, *scb; 929 int unit, error; 930 931 DPRINTF("ifp=%p\n", ifp); 932 933 /* 934 * In case we called into if_clone_destroyif() ourselves 935 * again to remove the second interface, the softc will be 936 * NULL. In that case so not do anything but return success. 937 */ 938 if (ifp->if_softc == NULL) 939 return (0); 940 941 unit = ifp->if_dunit; 942 sca = ifp->if_softc; 943 oifp = sca->oifp; 944 scb = oifp->if_softc; 945 946 DPRINTF("ifp=%p oifp=%p\n", ifp, oifp); 947 if_link_state_change(ifp, LINK_STATE_DOWN); 948 if_link_state_change(oifp, LINK_STATE_DOWN); 949 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 950 oifp->if_drv_flags &= ~IFF_DRV_RUNNING; 951 952 /* 953 * Get rid of our second half. As the other of the two 954 * interfaces may reside in a different vnet, we need to 955 * switch before freeing them. 956 */ 957 CURVNET_SET_QUIET(oifp->if_vnet); 958 ether_ifdetach(oifp); 959 /* 960 * Wait for all packets to be dispatched to if_input. 961 * The numbers can only go down as the interface is 962 * detached so there is no need to use atomics. 963 */ 964 DPRINTF("scb refcnt=%u\n", scb->refcount); 965 EPAIR_REFCOUNT_ASSERT(scb->refcount == 1, 966 ("%s: ifp=%p scb->refcount!=1: %d", __func__, oifp, scb->refcount)); 967 oifp->if_softc = NULL; 968 error = if_clone_destroyif(ifc, oifp); 969 if (error) 970 panic("%s: if_clone_destroyif() for our 2nd iface failed: %d", 971 __func__, error); 972 if_free(oifp); 973 ifmedia_removeall(&scb->media); 974 free(scb, M_EPAIR); 975 CURVNET_RESTORE(); 976 977 ether_ifdetach(ifp); 978 /* 979 * Wait for all packets to be dispatched to if_input. 980 */ 981 DPRINTF("sca refcnt=%u\n", sca->refcount); 982 EPAIR_REFCOUNT_ASSERT(sca->refcount == 1, 983 ("%s: ifp=%p sca->refcount!=1: %d", __func__, ifp, sca->refcount)); 984 if_free(ifp); 985 ifmedia_removeall(&sca->media); 986 free(sca, M_EPAIR); 987 ifc_free_unit(ifc, unit); 988 989 return (0); 990 } 991 992 static void 993 vnet_epair_init(const void *unused __unused) 994 { 995 996 V_epair_cloner = if_clone_advanced(epairname, 0, 997 epair_clone_match, epair_clone_create, epair_clone_destroy); 998 #ifdef VIMAGE 999 netisr_register_vnet(&epair_nh); 1000 #endif 1001 } 1002 VNET_SYSINIT(vnet_epair_init, SI_SUB_PSEUDO, SI_ORDER_ANY, 1003 vnet_epair_init, NULL); 1004 1005 static void 1006 vnet_epair_uninit(const void *unused __unused) 1007 { 1008 1009 #ifdef VIMAGE 1010 netisr_unregister_vnet(&epair_nh); 1011 #endif 1012 if_clone_detach(V_epair_cloner); 1013 } 1014 VNET_SYSUNINIT(vnet_epair_uninit, SI_SUB_INIT_IF, SI_ORDER_ANY, 1015 vnet_epair_uninit, NULL); 1016 1017 static void 1018 epair_uninit(const void *unused __unused) 1019 { 1020 netisr_unregister(&epair_nh); 1021 epair_dpcpu_detach(); 1022 if (bootverbose) 1023 printf("%s unloaded.\n", epairname); 1024 } 1025 SYSUNINIT(epair_uninit, SI_SUB_INIT_IF, SI_ORDER_MIDDLE, 1026 epair_uninit, NULL); 1027 1028 static int 1029 epair_modevent(module_t mod, int type, void *data) 1030 { 1031 int qlimit; 1032 1033 switch (type) { 1034 case MOD_LOAD: 1035 /* For now limit us to one global mutex and one inq. */ 1036 epair_dpcpu_init(); 1037 epair_nh.nh_qlimit = 42 * ifqmaxlen; /* 42 shall be the number. */ 1038 if (TUNABLE_INT_FETCH("net.link.epair.netisr_maxqlen", &qlimit)) 1039 epair_nh.nh_qlimit = qlimit; 1040 netisr_register(&epair_nh); 1041 if (bootverbose) 1042 printf("%s initialized.\n", epairname); 1043 break; 1044 case MOD_UNLOAD: 1045 /* Handled in epair_uninit() */ 1046 break; 1047 default: 1048 return (EOPNOTSUPP); 1049 } 1050 return (0); 1051 } 1052 1053 static moduledata_t epair_mod = { 1054 "if_epair", 1055 epair_modevent, 1056 0 1057 }; 1058 1059 DECLARE_MODULE(if_epair, epair_mod, SI_SUB_PSEUDO, SI_ORDER_MIDDLE); 1060 MODULE_VERSION(if_epair, 1); 1061