xref: /freebsd/sys/net/if_ovpn.c (revision 0e8011faf58b743cc652e3b2ad0f7671227610df)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2021-2022 Rubicon Communications, LLC (Netgate)
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  *
27  */
28 #include "opt_inet.h"
29 #include "opt_inet6.h"
30 
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/buf_ring.h>
34 #include <sys/epoch.h>
35 #include <sys/file.h>
36 #include <sys/filedesc.h>
37 #include <sys/kernel.h>
38 #include <sys/malloc.h>
39 #include <sys/mbuf.h>
40 #include <sys/module.h>
41 #include <sys/nv.h>
42 #include <sys/priv.h>
43 #include <sys/protosw.h>
44 #include <sys/rmlock.h>
45 #include <sys/sdt.h>
46 #include <sys/smp.h>
47 #include <sys/socket.h>
48 #include <sys/socketvar.h>
49 #include <sys/sockio.h>
50 #include <sys/sysctl.h>
51 #include <sys/time.h>
52 
53 #include <machine/atomic.h>
54 
55 #include <net/bpf.h>
56 #include <net/if.h>
57 #include <net/if_clone.h>
58 #include <net/if_types.h>
59 #include <net/if_var.h>
60 #include <net/if_private.h>
61 #include <net/netisr.h>
62 #include <net/route/nhop.h>
63 
64 #include <netinet/in.h>
65 #include <netinet/in_fib.h>
66 #include <netinet/ip.h>
67 #include <netinet/ip6.h>
68 #include <netinet/ip_var.h>
69 #include <netinet/udp.h>
70 #include <netinet/udp_var.h>
71 
72 #include <netinet6/ip6_var.h>
73 #include <netinet6/in6_fib.h>
74 
75 #include <machine/in_cksum.h>
76 
77 #include <opencrypto/cryptodev.h>
78 
79 #include "if_ovpn.h"
80 
81 struct ovpn_kkey_dir {
82 	int			refcount;
83 	uint8_t			key[32];
84 	uint8_t			keylen;
85 	uint8_t			nonce[8];
86 	uint8_t			noncelen;
87 	enum ovpn_key_cipher	cipher;
88 	crypto_session_t	cryptoid;
89 
90 	struct mtx		replay_mtx;
91 	/*
92 	 * Last seen gapless sequence number. New rx seq numbers must be
93 	 * strictly higher than this.
94 	 */
95 	uint32_t		rx_seq;
96 	uint64_t		tx_seq;
97 
98 	/* Seen packets, relative to rx_seq. bit(0) will always be 0. */
99 	uint64_t		rx_window;
100 };
101 
102 struct ovpn_kkey {
103 	struct ovpn_kkey_dir	*encrypt;
104 	struct ovpn_kkey_dir	*decrypt;
105 	uint8_t			 keyid;
106 	uint32_t		 peerid;
107 };
108 
109 struct ovpn_keepalive {
110 	uint32_t	interval;
111 	uint32_t	timeout;
112 };
113 
114 struct ovpn_wire_header {
115 	uint32_t	 opcode; /* opcode, key id, peer id */
116 	uint32_t	 seq;
117 	uint8_t		 auth_tag[16];
118 };
119 
120 struct ovpn_peer_counters {
121 	uint64_t	pkt_in;
122 	uint64_t	pkt_out;
123 	uint64_t	bytes_in;
124 	uint64_t	bytes_out;
125 };
126 #define OVPN_PEER_COUNTER_SIZE (sizeof(struct ovpn_peer_counters)/sizeof(uint64_t))
127 
128 struct ovpn_notification {
129 	enum ovpn_notif_type	type;
130 	uint32_t		peerid;
131 
132 	/* Delete notification */
133 	enum ovpn_del_reason	del_reason;
134 	struct ovpn_peer_counters	counters;
135 };
136 
137 struct ovpn_softc;
138 
139 struct ovpn_kpeer {
140 	RB_ENTRY(ovpn_kpeer)	 tree;
141 	int			 refcount;
142 	uint32_t		 peerid;
143 
144 	struct ovpn_softc	*sc;
145 	struct sockaddr_storage	 local;
146 	struct sockaddr_storage	 remote;
147 
148 	struct in_addr		 vpn4;
149 	struct in6_addr		 vpn6;
150 
151 	struct ovpn_kkey	 keys[2];
152 
153 	enum ovpn_del_reason	 del_reason;
154 	struct ovpn_keepalive	 keepalive;
155 	uint32_t		*last_active;
156 	struct callout		 ping_send;
157 	struct callout		 ping_rcv;
158 
159 	counter_u64_t		 counters[OVPN_PEER_COUNTER_SIZE];
160 };
161 
162 struct ovpn_counters {
163 	uint64_t	lost_ctrl_pkts_in;
164 	uint64_t	lost_ctrl_pkts_out;
165 	uint64_t	lost_data_pkts_in;
166 	uint64_t	lost_data_pkts_out;
167 	uint64_t	nomem_data_pkts_in;
168 	uint64_t	nomem_data_pkts_out;
169 	uint64_t	received_ctrl_pkts;
170 	uint64_t	received_data_pkts;
171 	uint64_t	sent_ctrl_pkts;
172 	uint64_t	sent_data_pkts;
173 
174 	uint64_t	transport_bytes_sent;
175 	uint64_t	transport_bytes_received;
176 	uint64_t	tunnel_bytes_sent;
177 	uint64_t	tunnel_bytes_received;
178 };
179 #define OVPN_COUNTER_SIZE (sizeof(struct ovpn_counters)/sizeof(uint64_t))
180 
181 RB_HEAD(ovpn_kpeers, ovpn_kpeer);
182 
183 struct ovpn_softc {
184 	int			 refcount;
185 	struct rmlock		 lock;
186 	struct ifnet		*ifp;
187 	struct socket		*so;
188 	int			 peercount;
189 	struct ovpn_kpeers	 peers;
190 
191 	/* Pending notification */
192 	struct buf_ring		*notifring;
193 
194 	counter_u64_t 		 counters[OVPN_COUNTER_SIZE];
195 
196 	struct epoch_context	 epoch_ctx;
197 };
198 
199 static struct ovpn_kpeer *ovpn_find_peer(struct ovpn_softc *, uint32_t);
200 static bool ovpn_udp_input(struct mbuf *, int, struct inpcb *,
201     const struct sockaddr *, void *);
202 static int ovpn_transmit_to_peer(struct ifnet *, struct mbuf *,
203     struct ovpn_kpeer *, struct rm_priotracker *);
204 static int ovpn_encap(struct ovpn_softc *, uint32_t, struct mbuf *);
205 static int ovpn_get_af(struct mbuf *);
206 static void ovpn_free_kkey_dir(struct ovpn_kkey_dir *);
207 static bool ovpn_check_replay(struct ovpn_kkey_dir *, uint32_t);
208 static int ovpn_peer_compare(struct ovpn_kpeer *, struct ovpn_kpeer *);
209 
210 static RB_PROTOTYPE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare);
211 static RB_GENERATE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare);
212 
213 #define OVPN_MTU_MIN		576
214 #define OVPN_MTU_MAX		(IP_MAXPACKET - sizeof(struct ip) - \
215     sizeof(struct udphdr) - sizeof(struct ovpn_wire_header))
216 
217 #define OVPN_OP_DATA_V2		0x09
218 #define OVPN_OP_SHIFT		3
219 #define OVPN_SEQ_ROTATE		0x80000000
220 
221 VNET_DEFINE_STATIC(struct if_clone *, ovpn_cloner);
222 #define	V_ovpn_cloner	VNET(ovpn_cloner)
223 
224 #define OVPN_RLOCK_TRACKER	struct rm_priotracker _ovpn_lock_tracker; \
225     struct rm_priotracker *_ovpn_lock_trackerp = &_ovpn_lock_tracker
226 #define OVPN_RLOCK(sc)		rm_rlock(&(sc)->lock, _ovpn_lock_trackerp)
227 #define OVPN_RUNLOCK(sc)	rm_runlock(&(sc)->lock, _ovpn_lock_trackerp)
228 #define OVPN_WLOCK(sc)		rm_wlock(&(sc)->lock)
229 #define OVPN_WUNLOCK(sc)	rm_wunlock(&(sc)->lock)
230 #define OVPN_ASSERT(sc)		rm_assert(&(sc)->lock, RA_LOCKED)
231 #define OVPN_RASSERT(sc)	rm_assert(&(sc)->lock, RA_RLOCKED)
232 #define OVPN_WASSERT(sc)	rm_assert(&(sc)->lock, RA_WLOCKED)
233 #define OVPN_UNLOCK_ASSERT(sc)	rm_assert(&(sc)->lock, RA_UNLOCKED)
234 
235 #define OVPN_COUNTER(sc, name) \
236 	((sc)->counters[offsetof(struct ovpn_counters, name)/sizeof(uint64_t)])
237 #define OVPN_PEER_COUNTER(peer, name) \
238 	((peer)->counters[offsetof(struct ovpn_peer_counters, name) / \
239 	 sizeof(uint64_t)])
240 
241 #define OVPN_COUNTER_ADD(sc, name, val)	\
242 	counter_u64_add(OVPN_COUNTER(sc, name), val)
243 #define OVPN_PEER_COUNTER_ADD(p, name, val)	\
244 	counter_u64_add(OVPN_PEER_COUNTER(p, name), val)
245 
246 #define TO_IN(x)		((struct sockaddr_in *)(x))
247 #define TO_IN6(x)		((struct sockaddr_in6 *)(x))
248 
249 SDT_PROVIDER_DEFINE(if_ovpn);
250 SDT_PROBE_DEFINE1(if_ovpn, tx, transmit, start, "struct mbuf *");
251 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip4, "struct in_addr *", "struct ovpn_kpeer *");
252 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip6, "struct in6_addr *", "struct ovpn_kpeer *");
253 
254 static const char ovpnname[] = "ovpn";
255 static const char ovpngroupname[] = "openvpn";
256 
257 static MALLOC_DEFINE(M_OVPN, ovpnname, "OpenVPN DCO Interface");
258 #define	MTAG_OVPN_LOOP		0x6f76706e /* ovpn */
259 
260 SYSCTL_DECL(_net_link);
261 static SYSCTL_NODE(_net_link, IFT_OTHER, openvpn, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
262     "OpenVPN DCO Interface");
263 VNET_DEFINE_STATIC(int, replay_protection) = 0;
264 #define	V_replay_protection	VNET(replay_protection)
265 SYSCTL_INT(_net_link_openvpn, OID_AUTO, replay_protection, CTLFLAG_VNET | CTLFLAG_RW,
266     &VNET_NAME(replay_protection), 0, "Validate sequence numbers");
267 
268 VNET_DEFINE_STATIC(int, async_crypto);
269 #define	V_async_crypto		VNET(async_crypto)
270 SYSCTL_INT(_net_link_openvpn, OID_AUTO, async_crypto,
271 	CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_crypto), 0,
272 	"Use asynchronous mode to parallelize crypto jobs.");
273 
274 VNET_DEFINE_STATIC(int, async_netisr_queue);
275 #define	V_async_netisr_queue		VNET(async_netisr_queue)
276 SYSCTL_INT(_net_link_openvpn, OID_AUTO, netisr_queue,
277 	CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_netisr_queue), 0,
278 	"Use netisr_queue() rather than netisr_dispatch().");
279 
280 static int
281 ovpn_peer_compare(struct ovpn_kpeer *a, struct ovpn_kpeer *b)
282 {
283 	return (a->peerid - b->peerid);
284 }
285 
286 static struct ovpn_kpeer *
287 ovpn_find_peer(struct ovpn_softc *sc, uint32_t peerid)
288 {
289 	struct ovpn_kpeer p;
290 
291 	OVPN_ASSERT(sc);
292 
293 	p.peerid = peerid;
294 
295 	return (RB_FIND(ovpn_kpeers, &sc->peers, &p));
296 }
297 
298 static struct ovpn_kpeer *
299 ovpn_find_only_peer(struct ovpn_softc *sc)
300 {
301 	OVPN_ASSERT(sc);
302 
303 	return (RB_ROOT(&sc->peers));
304 }
305 
306 static uint16_t
307 ovpn_get_port(struct sockaddr_storage *s)
308 {
309 	switch (s->ss_family) {
310 	case AF_INET: {
311 		struct sockaddr_in *in = (struct sockaddr_in *)s;
312 		return (in->sin_port);
313 	}
314 	case AF_INET6: {
315 		struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)s;
316 		return (in6->sin6_port);
317 	}
318 	default:
319 		panic("Unsupported address family %d", s->ss_family);
320 	}
321 }
322 
323 static int
324 ovpn_nvlist_to_sockaddr(const nvlist_t *nvl, struct sockaddr_storage *sa)
325 {
326 	int af;
327 
328 	if (! nvlist_exists_number(nvl, "af"))
329 		return (EINVAL);
330 	if (! nvlist_exists_binary(nvl, "address"))
331 		return (EINVAL);
332 	if (! nvlist_exists_number(nvl, "port"))
333 		return (EINVAL);
334 
335 	af = nvlist_get_number(nvl, "af");
336 
337 	switch (af) {
338 #ifdef INET
339 	case AF_INET: {
340 		struct sockaddr_in *in = (struct sockaddr_in *)sa;
341 		size_t len;
342 		const void *addr = nvlist_get_binary(nvl, "address", &len);
343 		in->sin_family = af;
344 		if (len != sizeof(in->sin_addr))
345 			return (EINVAL);
346 
347 		memcpy(&in->sin_addr, addr, sizeof(in->sin_addr));
348 		in->sin_port = nvlist_get_number(nvl, "port");
349 		break;
350 	}
351 #endif
352 #ifdef INET6
353 	case AF_INET6: {
354 		struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)sa;
355 		size_t len;
356 		const void *addr = nvlist_get_binary(nvl, "address", &len);
357 		in6->sin6_family = af;
358 		if (len != sizeof(in6->sin6_addr))
359 			return (EINVAL);
360 
361 		memcpy(&in6->sin6_addr, addr, sizeof(in6->sin6_addr));
362 		in6->sin6_port = nvlist_get_number(nvl, "port");
363 		break;
364 	}
365 #endif
366 	default:
367 		return (EINVAL);
368 	}
369 
370 	return (0);
371 }
372 
373 static bool
374 ovpn_has_peers(struct ovpn_softc *sc)
375 {
376 	OVPN_ASSERT(sc);
377 
378 	return (sc->peercount > 0);
379 }
380 
381 static void
382 ovpn_rele_so(struct ovpn_softc *sc)
383 {
384 	bool has_peers;
385 
386 	OVPN_WASSERT(sc);
387 
388 	if (sc->so == NULL)
389 		return;
390 
391 	has_peers = ovpn_has_peers(sc);
392 
393 	if (! has_peers) {
394 		MPASS(sc->peercount == 0);
395 	} else {
396 		MPASS(sc->peercount > 0);
397 	}
398 }
399 
400 static void
401 ovpn_notify_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer)
402 {
403 	struct ovpn_notification *n;
404 
405 	OVPN_WASSERT(sc);
406 
407 	n = malloc(sizeof(*n), M_OVPN, M_NOWAIT);
408 	if (n == NULL)
409 		return;
410 
411 	n->peerid = peer->peerid;
412 	n->type = OVPN_NOTIF_DEL_PEER;
413 	n->del_reason = peer->del_reason;
414 
415 	n->counters.pkt_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_in));
416 	n->counters.pkt_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_out));
417 	n->counters.bytes_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_in));
418 	n->counters.bytes_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_out));
419 
420 	if (buf_ring_enqueue(sc->notifring, n) != 0) {
421 		free(n, M_OVPN);
422 	} else if (sc->so != NULL) {
423 		/* Wake up userspace */
424 		sc->so->so_error = EAGAIN;
425 		sorwakeup(sc->so);
426 		sowwakeup(sc->so);
427 	}
428 }
429 
430 static void
431 ovpn_notify_key_rotation(struct ovpn_softc *sc, struct ovpn_kpeer *peer)
432 {
433 	struct ovpn_notification *n;
434 
435 	n = malloc(sizeof(*n), M_OVPN, M_NOWAIT | M_ZERO);
436 	if (n == NULL)
437 		return;
438 
439 	n->peerid = peer->peerid;
440 	n->type = OVPN_NOTIF_ROTATE_KEY;
441 
442 	if (buf_ring_enqueue(sc->notifring, n) != 0) {
443 		free(n, M_OVPN);
444 	} else if (sc->so != NULL) {
445 		/* Wake up userspace */
446 		sc->so->so_error = EAGAIN;
447 		sorwakeup(sc->so);
448 		sowwakeup(sc->so);
449 	}
450 }
451 
452 static void
453 ovpn_peer_release_ref(struct ovpn_kpeer *peer, bool locked)
454 {
455 	struct ovpn_softc *sc;
456 
457 	CURVNET_ASSERT_SET();
458 
459 	atomic_add_int(&peer->refcount, -1);
460 
461 	if (atomic_load_int(&peer->refcount) > 0)
462 		return;
463 
464 	sc = peer->sc;
465 
466 	if (! locked) {
467 		OVPN_WLOCK(sc);
468 
469 		/* Might have changed before we acquired the lock. */
470 		if (atomic_load_int(&peer->refcount) > 0) {
471 			OVPN_WUNLOCK(sc);
472 			return;
473 		}
474 	}
475 
476 	OVPN_ASSERT(sc);
477 
478 	/* The peer should have been removed from the list already. */
479 	MPASS(ovpn_find_peer(sc, peer->peerid) == NULL);
480 
481 	ovpn_notify_del_peer(sc, peer);
482 
483 	for (int i = 0; i < 2; i++) {
484 		ovpn_free_kkey_dir(peer->keys[i].encrypt);
485 		ovpn_free_kkey_dir(peer->keys[i].decrypt);
486 	}
487 
488 	ovpn_rele_so(sc);
489 
490 	callout_stop(&peer->ping_send);
491 	callout_stop(&peer->ping_rcv);
492 	uma_zfree_pcpu(pcpu_zone_4, peer->last_active);
493 	free(peer, M_OVPN);
494 
495 	if (! locked)
496 		OVPN_WUNLOCK(sc);
497 }
498 
499 static int
500 ovpn_new_peer(struct ifnet *ifp, const nvlist_t *nvl)
501 {
502 #ifdef INET6
503 	struct epoch_tracker et;
504 #endif
505 	struct sockaddr_storage remote;
506 	struct ovpn_kpeer *peer = NULL;
507 	struct file *fp = NULL;
508 	struct ovpn_softc *sc = ifp->if_softc;
509 	struct thread *td = curthread;
510 	struct socket *so = NULL;
511 	int fd;
512 	uint32_t peerid;
513 	int ret = 0;
514 	bool setcb = false;
515 
516 	if (nvl == NULL)
517 		return (EINVAL);
518 
519 	if (! nvlist_exists_number(nvl, "peerid"))
520 		return (EINVAL);
521 
522 	if (! nvlist_exists_number(nvl, "fd"))
523 		return (EINVAL);
524 
525 	if (! nvlist_exists_nvlist(nvl, "remote"))
526 		return (EINVAL);
527 
528 	peerid = nvlist_get_number(nvl, "peerid");
529 
530 	ret = ovpn_nvlist_to_sockaddr(nvlist_get_nvlist(nvl, "remote"),
531 	    &remote);
532 	if (ret != 0)
533 		return (ret);
534 
535 	fd = nvlist_get_number(nvl, "fd");
536 
537 	/* Look up the userspace process and use the fd to find the socket. */
538 	ret = getsock(td, fd, &cap_connect_rights, &fp);
539 	if (ret != 0)
540 		return (ret);
541 
542 	so = fp->f_data;
543 
544 	peer = malloc(sizeof(*peer), M_OVPN, M_WAITOK | M_ZERO);
545 	peer->peerid = peerid;
546 	peer->sc = sc;
547 	peer->refcount = 1;
548 	peer->last_active = uma_zalloc_pcpu(pcpu_zone_4, M_WAITOK | M_ZERO);
549 	COUNTER_ARRAY_ALLOC(peer->counters, OVPN_PEER_COUNTER_SIZE, M_WAITOK);
550 
551 	if (nvlist_exists_binary(nvl, "vpn_ipv4")) {
552 		size_t len;
553 		const void *addr = nvlist_get_binary(nvl, "vpn_ipv4", &len);
554 		if (len != sizeof(peer->vpn4)) {
555 			ret = EINVAL;
556 			goto error;
557 		}
558 		memcpy(&peer->vpn4, addr, len);
559 	}
560 
561 	if (nvlist_exists_binary(nvl, "vpn_ipv6")) {
562 		size_t len;
563 		const void *addr = nvlist_get_binary(nvl, "vpn_ipv6", &len);
564 		if (len != sizeof(peer->vpn6)) {
565 			ret = EINVAL;
566 			goto error;
567 		}
568 		memcpy(&peer->vpn6, addr, len);
569 	}
570 
571 	callout_init_rm(&peer->ping_send, &sc->lock, CALLOUT_SHAREDLOCK);
572 	callout_init_rm(&peer->ping_rcv, &sc->lock, 0);
573 
574 	peer->local.ss_len = sizeof(peer->local);
575 	ret = sosockaddr(so, (struct sockaddr *)&peer->local);
576 	if (ret)
577 		goto error;
578 
579 	if (ovpn_get_port(&peer->local) == 0) {
580 		ret = EINVAL;
581 		goto error;
582 	}
583 	if (peer->local.ss_family != remote.ss_family) {
584 		ret = EINVAL;
585 		goto error;
586 	}
587 
588 	memcpy(&peer->remote, &remote, sizeof(remote));
589 
590 	if (peer->local.ss_family == AF_INET6 &&
591 	    IN6_IS_ADDR_V4MAPPED(&TO_IN6(&peer->remote)->sin6_addr)) {
592 		/* V4 mapped address, so treat this as v4, not v6. */
593 		in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->local);
594 		in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->remote);
595 	}
596 
597 #ifdef INET6
598 	if (peer->local.ss_family == AF_INET6 &&
599 	    IN6_IS_ADDR_UNSPECIFIED(&TO_IN6(&peer->local)->sin6_addr)) {
600 		NET_EPOCH_ENTER(et);
601 		ret = in6_selectsrc_addr(curthread->td_proc->p_fibnum,
602 		    &TO_IN6(&peer->remote)->sin6_addr,
603 		    0, NULL, &TO_IN6(&peer->local)->sin6_addr, NULL);
604 		NET_EPOCH_EXIT(et);
605 		if (ret != 0) {
606 			goto error;
607 		}
608 	}
609 #endif
610 	OVPN_WLOCK(sc);
611 
612 	/* Disallow peer id re-use. */
613 	if (ovpn_find_peer(sc, peerid) != NULL) {
614 		ret = EEXIST;
615 		goto error_locked;
616 	}
617 
618 	/* Make sure this is really a UDP socket. */
619 	if (so->so_type != SOCK_DGRAM || so->so_proto->pr_type != SOCK_DGRAM) {
620 		ret = EPROTOTYPE;
621 		goto error_locked;
622 	}
623 
624 	/* Must be the same socket as for other peers on this interface. */
625 	if (sc->so != NULL && so != sc->so) {
626 		if (! RB_EMPTY(&sc->peers)) {
627 			ret = EBUSY;
628 			goto error_locked;
629 		}
630 
631 		/*
632 		 * If we have no peers we can safely release the socket and accept
633 		 * a new one.
634 		 */
635 		ret = udp_set_kernel_tunneling(sc->so, NULL, NULL, NULL);
636 		sorele(sc->so);
637 		sc->so = NULL;
638 	}
639 
640 	if (sc->so == NULL) {
641 		sc->so = so;
642 		/*
643 		 * Maintain one extra ref so the socket doesn't go away until
644 		 * we're destroying the ifp.
645 		 */
646 		soref(sc->so);
647 		setcb = true;
648 	}
649 
650 	/* Insert the peer into the list. */
651 	RB_INSERT(ovpn_kpeers, &sc->peers, peer);
652 	sc->peercount++;
653 
654 	OVPN_WUNLOCK(sc);
655 
656 	if (setcb) {
657 		ret = udp_set_kernel_tunneling(sc->so, ovpn_udp_input, NULL, sc);
658 		MPASS(ret == 0);
659 	}
660 
661 	goto done;
662 
663 error_locked:
664 	OVPN_WUNLOCK(sc);
665 error:
666 	COUNTER_ARRAY_FREE(peer->counters, OVPN_PEER_COUNTER_SIZE);
667 	uma_zfree_pcpu(pcpu_zone_4, peer->last_active);
668 	free(peer, M_OVPN);
669 done:
670 	if (fp != NULL)
671 		fdrop(fp, td);
672 
673 	return (ret);
674 }
675 
676 static int
677 _ovpn_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer)
678 {
679 	struct ovpn_kpeer *tmp __diagused;
680 
681 	OVPN_WASSERT(sc);
682 	CURVNET_ASSERT_SET();
683 
684 	MPASS(RB_FIND(ovpn_kpeers, &sc->peers, peer) == peer);
685 
686 	tmp = RB_REMOVE(ovpn_kpeers, &sc->peers, peer);
687 	MPASS(tmp != NULL);
688 
689 	sc->peercount--;
690 
691 	ovpn_peer_release_ref(peer, true);
692 
693 	return (0);
694 }
695 
696 static int
697 ovpn_del_peer(struct ifnet *ifp, nvlist_t *nvl)
698 {
699 	struct ovpn_softc *sc = ifp->if_softc;
700 	struct ovpn_kpeer *peer;
701 	uint32_t peerid;
702 	int ret;
703 
704 	OVPN_WASSERT(sc);
705 
706 	if (nvl == NULL)
707 		return (EINVAL);
708 
709 	if (! nvlist_exists_number(nvl, "peerid"))
710 		return (EINVAL);
711 
712 	peerid = nvlist_get_number(nvl, "peerid");
713 
714 	peer = ovpn_find_peer(sc, peerid);
715 	if (peer == NULL)
716 		return (ENOENT);
717 
718 	peer->del_reason = OVPN_DEL_REASON_REQUESTED;
719 	ret = _ovpn_del_peer(sc, peer);
720 
721 	return (ret);
722 }
723 
724 static int
725 ovpn_create_kkey_dir(struct ovpn_kkey_dir **kdirp,
726     const nvlist_t *nvl)
727 {
728 	struct crypto_session_params csp;
729 	struct ovpn_kkey_dir *kdir;
730 	const char *ciphername;
731 	enum ovpn_key_cipher cipher;
732 	const void *key, *iv;
733 	size_t keylen = 0, ivlen = 0;
734 	int error;
735 
736 	if (! nvlist_exists_string(nvl, "cipher"))
737 		return (EINVAL);
738 	ciphername = nvlist_get_string(nvl, "cipher");
739 
740 	if (strcmp(ciphername, "none") == 0)
741 		cipher = OVPN_CIPHER_ALG_NONE;
742 	else if (strcmp(ciphername, "AES-256-GCM") == 0 ||
743 	    strcmp(ciphername, "AES-192-GCM") == 0 ||
744 	    strcmp(ciphername, "AES-128-GCM") == 0)
745 		cipher = OVPN_CIPHER_ALG_AES_GCM;
746 	else if (strcmp(ciphername, "CHACHA20-POLY1305") == 0)
747 		cipher = OVPN_CIPHER_ALG_CHACHA20_POLY1305;
748 	else
749 		return (EINVAL);
750 
751 	if (cipher != OVPN_CIPHER_ALG_NONE) {
752 		if (! nvlist_exists_binary(nvl, "key"))
753 			return (EINVAL);
754 		key = nvlist_get_binary(nvl, "key", &keylen);
755 		if (keylen > sizeof(kdir->key))
756 			return (E2BIG);
757 
758 		if (! nvlist_exists_binary(nvl, "iv"))
759 			return (EINVAL);
760 		iv = nvlist_get_binary(nvl, "iv", &ivlen);
761 		if (ivlen != 8)
762 			return (E2BIG);
763 	}
764 
765 	kdir = malloc(sizeof(struct ovpn_kkey_dir), M_OVPN,
766 	    M_WAITOK | M_ZERO);
767 
768 	kdir->cipher = cipher;
769 	kdir->keylen = keylen;
770 	kdir->tx_seq = 1;
771 	memcpy(kdir->key, key, keylen);
772 	kdir->noncelen = ivlen;
773 	memcpy(kdir->nonce, iv, ivlen);
774 
775 	if (kdir->cipher != OVPN_CIPHER_ALG_NONE) {
776 		/* Crypto init */
777 		bzero(&csp, sizeof(csp));
778 		csp.csp_mode = CSP_MODE_AEAD;
779 
780 		if (kdir->cipher == OVPN_CIPHER_ALG_CHACHA20_POLY1305)
781 			csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305;
782 		else
783 			csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
784 
785 		csp.csp_flags |= CSP_F_SEPARATE_AAD;
786 
787 		csp.csp_cipher_klen = kdir->keylen;
788 		csp.csp_cipher_key = kdir->key;
789 		csp.csp_ivlen = 96 / 8;
790 
791 		error = crypto_newsession(&kdir->cryptoid, &csp,
792 		    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE);
793 		if (error) {
794 			free(kdir, M_OVPN);
795 			return (error);
796 		}
797 	}
798 
799 	mtx_init(&kdir->replay_mtx, "if_ovpn rx replay", NULL, MTX_DEF);
800 	*kdirp = kdir;
801 
802 	return (0);
803 }
804 
805 static void
806 ovpn_free_kkey_dir(struct ovpn_kkey_dir *kdir)
807 {
808 	if (kdir == NULL)
809 		return;
810 
811 	mtx_destroy(&kdir->replay_mtx);
812 
813 	crypto_freesession(kdir->cryptoid);
814 	free(kdir, M_OVPN);
815 }
816 
817 static int
818 ovpn_set_key(struct ifnet *ifp, const nvlist_t *nvl)
819 {
820 	struct ovpn_softc *sc = ifp->if_softc;
821 	struct ovpn_kkey_dir *enc, *dec;
822 	struct ovpn_kpeer *peer;
823 	int slot, keyid, peerid;
824 	int error;
825 
826 	if (nvl == NULL)
827 		return (EINVAL);
828 
829 	if (! nvlist_exists_number(nvl, "slot"))
830 		return (EINVAL);
831 	slot = nvlist_get_number(nvl, "slot");
832 
833 	if (! nvlist_exists_number(nvl, "keyid"))
834 		return (EINVAL);
835 	keyid = nvlist_get_number(nvl, "keyid");
836 
837 	if (! nvlist_exists_number(nvl, "peerid"))
838 		return (EINVAL);
839 	peerid = nvlist_get_number(nvl, "peerid");
840 
841 	if (slot != OVPN_KEY_SLOT_PRIMARY &&
842 	    slot != OVPN_KEY_SLOT_SECONDARY)
843 		return (EINVAL);
844 
845 	if (! nvlist_exists_nvlist(nvl, "encrypt") ||
846 	    ! nvlist_exists_nvlist(nvl, "decrypt"))
847 		return (EINVAL);
848 
849 	error = ovpn_create_kkey_dir(&enc, nvlist_get_nvlist(nvl, "encrypt"));
850 	if (error)
851 		return (error);
852 
853 	error = ovpn_create_kkey_dir(&dec, nvlist_get_nvlist(nvl, "decrypt"));
854 	if (error) {
855 		ovpn_free_kkey_dir(enc);
856 		return (error);
857 	}
858 
859 	OVPN_WLOCK(sc);
860 
861 	peer = ovpn_find_peer(sc, peerid);
862 	if (peer == NULL) {
863 		ovpn_free_kkey_dir(dec);
864 		ovpn_free_kkey_dir(enc);
865 		OVPN_WUNLOCK(sc);
866 		return (ENOENT);
867 	}
868 
869 	ovpn_free_kkey_dir(peer->keys[slot].encrypt);
870 	ovpn_free_kkey_dir(peer->keys[slot].decrypt);
871 
872 	peer->keys[slot].encrypt = enc;
873 	peer->keys[slot].decrypt = dec;
874 
875 	peer->keys[slot].keyid = keyid;
876 	peer->keys[slot].peerid = peerid;
877 
878 	OVPN_WUNLOCK(sc);
879 
880 	return (0);
881 }
882 
883 static int
884 ovpn_check_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, enum ovpn_key_slot slot)
885 {
886 	OVPN_ASSERT(sc);
887 
888 	if (peer->keys[slot].encrypt == NULL)
889 		return (ENOLINK);
890 
891 	if (peer->keys[slot].decrypt == NULL)
892 		return (ENOLINK);
893 
894 	return (0);
895 }
896 
897 static int
898 ovpn_start(struct ifnet *ifp)
899 {
900 	struct ovpn_softc *sc = ifp->if_softc;
901 
902 	OVPN_WLOCK(sc);
903 
904 	ifp->if_flags |= IFF_UP;
905 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
906 	if_link_state_change(ifp, LINK_STATE_UP);
907 
908 	OVPN_WUNLOCK(sc);
909 
910 	return (0);
911 }
912 
913 static int
914 ovpn_swap_keys(struct ifnet *ifp, nvlist_t *nvl)
915 {
916 	struct ovpn_softc *sc = ifp->if_softc;
917 	struct ovpn_kpeer *peer;
918 	struct ovpn_kkey tmpkey;
919 	int error;
920 
921 	if (nvl == NULL)
922 		return (EINVAL);
923 
924 	if (! nvlist_exists_number(nvl, "peerid"))
925 		return (EINVAL);
926 
927 	OVPN_WLOCK(sc);
928 
929 	peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid"));
930 	if (peer == NULL) {
931 		OVPN_WUNLOCK(sc);
932 		return (ENOENT);
933 	}
934 
935 	/* Check that we have a second key to swap to. */
936 	error = ovpn_check_key(sc, peer, OVPN_KEY_SLOT_SECONDARY);
937 	if (error) {
938 		OVPN_WUNLOCK(sc);
939 		return (error);
940 	}
941 
942 	tmpkey = peer->keys[0];
943 	peer->keys[0] = peer->keys[1];
944 	peer->keys[1] = tmpkey;
945 
946 	OVPN_WUNLOCK(sc);
947 
948 	return (0);
949 }
950 
951 static int
952 ovpn_del_key(struct ifnet *ifp, const nvlist_t *nvl)
953 {
954 	enum ovpn_key_slot slot;
955 	struct ovpn_kpeer *peer;
956 	struct ovpn_softc *sc = ifp->if_softc;
957 
958 	if (nvl == NULL)
959 		return (EINVAL);
960 
961 	if (! nvlist_exists_number(nvl, "peerid"))
962 		return (EINVAL);
963 
964 	if (! nvlist_exists_number(nvl, "slot"))
965 		return (EINVAL);
966 	slot = nvlist_get_number(nvl, "slot");
967 
968 	if (slot != OVPN_KEY_SLOT_PRIMARY &&
969 	    slot != OVPN_KEY_SLOT_SECONDARY)
970 		return (EINVAL);
971 
972 	OVPN_WLOCK(sc);
973 
974 	peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid"));
975 	if (peer == NULL) {
976 		OVPN_WUNLOCK(sc);
977 		return (ENOENT);
978 	}
979 
980 	ovpn_free_kkey_dir(peer->keys[slot].encrypt);
981 	ovpn_free_kkey_dir(peer->keys[slot].decrypt);
982 
983 	peer->keys[slot].encrypt = NULL;
984 	peer->keys[slot].decrypt = NULL;
985 
986 	peer->keys[slot].keyid = 0;
987 	peer->keys[slot].peerid = 0;
988 
989 	OVPN_WUNLOCK(sc);
990 
991 	return (0);
992 }
993 
994 static void
995 ovpn_send_ping(void *arg)
996 {
997 	static const uint8_t ping_str[] = {
998 		0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb,
999 		0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48
1000 	};
1001 
1002 	struct epoch_tracker et;
1003 	struct ovpn_kpeer *peer = arg;
1004 	struct ovpn_softc *sc = peer->sc;
1005 	struct mbuf *m;
1006 
1007 	OVPN_RASSERT(sc);
1008 
1009 	/* Ensure we repeat! */
1010 	callout_reset(&peer->ping_send, peer->keepalive.interval * hz,
1011 	    ovpn_send_ping, peer);
1012 
1013 	m = m_get2(sizeof(ping_str), M_NOWAIT, MT_DATA, M_PKTHDR);
1014 	if (m == NULL)
1015 		return;
1016 
1017 	m_copyback(m, 0, sizeof(ping_str), ping_str);
1018 	m->m_len = m->m_pkthdr.len = sizeof(ping_str);
1019 
1020 	CURVNET_SET(sc->ifp->if_vnet);
1021 	NET_EPOCH_ENTER(et);
1022 	(void)ovpn_transmit_to_peer(sc->ifp, m, peer, NULL);
1023 	NET_EPOCH_EXIT(et);
1024 	CURVNET_RESTORE();
1025 }
1026 
1027 static void
1028 ovpn_timeout(void *arg)
1029 {
1030 	struct ovpn_kpeer *peer = arg;
1031 	struct ovpn_softc *sc = peer->sc;
1032 	uint32_t last, _last_active;
1033 	int ret __diagused;
1034 	int cpu;
1035 
1036 	OVPN_WASSERT(sc);
1037 
1038 	last = 0;
1039 	CPU_FOREACH(cpu) {
1040 		_last_active = *zpcpu_get_cpu(peer->last_active, cpu);
1041 		if (_last_active > last)
1042 			last = _last_active;
1043 	}
1044 
1045 	if (last + peer->keepalive.timeout > time_uptime) {
1046 		callout_reset(&peer->ping_rcv,
1047 		    (peer->keepalive.timeout - (time_uptime - last)) * hz,
1048 		    ovpn_timeout, peer);
1049 		return;
1050 	}
1051 
1052 	CURVNET_SET(sc->ifp->if_vnet);
1053 	peer->del_reason = OVPN_DEL_REASON_TIMEOUT;
1054 	ret = _ovpn_del_peer(sc, peer);
1055 	MPASS(ret == 0);
1056 	CURVNET_RESTORE();
1057 }
1058 
1059 static int
1060 ovpn_set_peer(struct ifnet *ifp, const nvlist_t *nvl)
1061 {
1062 	struct ovpn_softc *sc = ifp->if_softc;
1063 	struct ovpn_kpeer *peer;
1064 
1065 	if (nvl == NULL)
1066 		return (EINVAL);
1067 
1068 	if (! nvlist_exists_number(nvl, "interval") ||
1069 	    ! nvlist_exists_number(nvl, "timeout") ||
1070 	    ! nvlist_exists_number(nvl, "peerid"))
1071 		return (EINVAL);
1072 
1073 	OVPN_WLOCK(sc);
1074 
1075 	peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid"));
1076 	if (peer == NULL) {
1077 		OVPN_WUNLOCK(sc);
1078 		return (ENOENT);
1079 	}
1080 
1081 	peer->keepalive.interval = nvlist_get_number(nvl, "interval");
1082 	peer->keepalive.timeout = nvlist_get_number(nvl, "timeout");
1083 
1084 	if (peer->keepalive.interval > 0)
1085 		callout_reset(&peer->ping_send, peer->keepalive.interval * hz,
1086 		    ovpn_send_ping, peer);
1087 	if (peer->keepalive.timeout > 0)
1088 		callout_reset(&peer->ping_rcv, peer->keepalive.timeout * hz,
1089 		    ovpn_timeout, peer);
1090 
1091 	OVPN_WUNLOCK(sc);
1092 
1093 	return (0);
1094 }
1095 
1096 static int
1097 ovpn_set_ifmode(struct ifnet *ifp, const nvlist_t *nvl)
1098 {
1099 	struct ovpn_softc *sc = ifp->if_softc;
1100 	int ifmode;
1101 
1102 	if (nvl == NULL)
1103 		return (EINVAL);
1104 
1105 	if (! nvlist_exists_number(nvl, "ifmode") )
1106 		return (EINVAL);
1107 
1108 	ifmode = nvlist_get_number(nvl, "ifmode");
1109 
1110 	OVPN_WLOCK(sc);
1111 
1112 	/* deny this if UP */
1113 	if (ifp->if_flags & IFF_UP) {
1114 		OVPN_WUNLOCK(sc);
1115 		return (EBUSY);
1116 	}
1117 
1118 	switch (ifmode & ~IFF_MULTICAST) {
1119 	case IFF_POINTOPOINT:
1120 	case IFF_BROADCAST:
1121 		ifp->if_flags &=
1122 		    ~(IFF_BROADCAST|IFF_POINTOPOINT|IFF_MULTICAST);
1123 		ifp->if_flags |= ifmode;
1124 		break;
1125 	default:
1126 		OVPN_WUNLOCK(sc);
1127 		return (EINVAL);
1128 	}
1129 
1130 	OVPN_WUNLOCK(sc);
1131 
1132 	return (0);
1133 }
1134 
1135 static int
1136 ovpn_ioctl_set(struct ifnet *ifp, struct ifdrv *ifd)
1137 {
1138 	struct ovpn_softc *sc = ifp->if_softc;
1139 	uint8_t *buf = NULL;
1140 	nvlist_t *nvl = NULL;
1141 	int ret;
1142 
1143 	if (ifd->ifd_len != 0) {
1144 		if (ifd->ifd_len > OVPN_MAX_REQUEST_SIZE)
1145 			return (E2BIG);
1146 
1147 		buf = malloc(ifd->ifd_len, M_OVPN, M_WAITOK);
1148 
1149 		ret = copyin(ifd->ifd_data, buf, ifd->ifd_len);
1150 		if (ret != 0) {
1151 			free(buf, M_OVPN);
1152 			return (ret);
1153 		}
1154 
1155 		nvl = nvlist_unpack(buf, ifd->ifd_len, 0);
1156 		free(buf, M_OVPN);
1157 		if (nvl == NULL) {
1158 			return (EINVAL);
1159 		}
1160 	}
1161 
1162 	switch (ifd->ifd_cmd) {
1163 	case OVPN_NEW_PEER:
1164 		ret = ovpn_new_peer(ifp, nvl);
1165 		break;
1166 	case OVPN_DEL_PEER:
1167 		OVPN_WLOCK(sc);
1168 		ret = ovpn_del_peer(ifp, nvl);
1169 		OVPN_WUNLOCK(sc);
1170 		break;
1171 	case OVPN_NEW_KEY:
1172 		ret = ovpn_set_key(ifp, nvl);
1173 		break;
1174 	case OVPN_START_VPN:
1175 		ret = ovpn_start(ifp);
1176 		break;
1177 	case OVPN_SWAP_KEYS:
1178 		ret = ovpn_swap_keys(ifp, nvl);
1179 		break;
1180 	case OVPN_DEL_KEY:
1181 		ret = ovpn_del_key(ifp, nvl);
1182 		break;
1183 	case OVPN_SET_PEER:
1184 		ret = ovpn_set_peer(ifp, nvl);
1185 		break;
1186 	case OVPN_SET_IFMODE:
1187 		ret = ovpn_set_ifmode(ifp, nvl);
1188 		break;
1189 	default:
1190 		ret = ENOTSUP;
1191 	}
1192 
1193 	nvlist_destroy(nvl);
1194 	return (ret);
1195 }
1196 
1197 static int
1198 ovpn_add_counters(nvlist_t *parent, const char *name, counter_u64_t in,
1199     counter_u64_t out)
1200 {
1201 	nvlist_t *nvl;
1202 
1203 	nvl = nvlist_create(0);
1204 	if (nvl == NULL)
1205 		return (ENOMEM);
1206 
1207 	nvlist_add_number(nvl, "in", counter_u64_fetch(in));
1208 	nvlist_add_number(nvl, "out", counter_u64_fetch(out));
1209 
1210 	nvlist_add_nvlist(parent, name, nvl);
1211 
1212 	nvlist_destroy(nvl);
1213 
1214 	return (0);
1215 }
1216 
1217 static int
1218 ovpn_get_stats(struct ovpn_softc *sc, nvlist_t **onvl)
1219 {
1220 	nvlist_t *nvl;
1221 	int ret;
1222 
1223 	nvl = nvlist_create(0);
1224 	if (nvl == NULL)
1225 		return (ENOMEM);
1226 
1227 #define OVPN_COUNTER_OUT(name, in, out) \
1228 	do { \
1229 		ret = ovpn_add_counters(nvl, name, OVPN_COUNTER(sc, in), \
1230 		    OVPN_COUNTER(sc, out)); \
1231 		if (ret != 0) \
1232 			goto error; \
1233 	} while(0)
1234 
1235 	OVPN_COUNTER_OUT("lost_ctrl", lost_ctrl_pkts_in, lost_ctrl_pkts_out);
1236 	OVPN_COUNTER_OUT("lost_data", lost_data_pkts_in, lost_data_pkts_out);
1237 	OVPN_COUNTER_OUT("nomem_data", nomem_data_pkts_in,
1238 	    nomem_data_pkts_out);
1239 	OVPN_COUNTER_OUT("data", received_data_pkts, sent_data_pkts);
1240 	OVPN_COUNTER_OUT("ctrl", received_ctrl_pkts, sent_ctrl_pkts);
1241 	OVPN_COUNTER_OUT("tunnel", tunnel_bytes_received,
1242 	    tunnel_bytes_received);
1243 	OVPN_COUNTER_OUT("transport", transport_bytes_received,
1244 	    transport_bytes_received);
1245 #undef OVPN_COUNTER_OUT
1246 
1247 	*onvl = nvl;
1248 
1249 	return (0);
1250 
1251 error:
1252 	nvlist_destroy(nvl);
1253 	return (ret);
1254 }
1255 
1256 static int
1257 ovpn_get_peer_stats(struct ovpn_softc *sc, nvlist_t **nvl)
1258 {
1259 	struct ovpn_kpeer *peer;
1260 	nvlist_t *nvpeer = NULL;
1261 	int ret;
1262 
1263 	OVPN_RLOCK_TRACKER;
1264 
1265 	*nvl = nvlist_create(0);
1266 	if (*nvl == NULL)
1267 		return (ENOMEM);
1268 
1269 #define OVPN_PEER_COUNTER_OUT(name, in, out) \
1270 	do { \
1271 		ret = ovpn_add_counters(nvpeer, name, \
1272 		    OVPN_PEER_COUNTER(peer, in), OVPN_PEER_COUNTER(peer, out)); \
1273 		if (ret != 0) \
1274 			goto error; \
1275 	} while(0)
1276 
1277 	OVPN_RLOCK(sc);
1278 	RB_FOREACH(peer, ovpn_kpeers, &sc->peers) {
1279 		nvpeer = nvlist_create(0);
1280 		if (nvpeer == NULL) {
1281 			OVPN_RUNLOCK(sc);
1282 			nvlist_destroy(*nvl);
1283 			*nvl = NULL;
1284 			return (ENOMEM);
1285 		}
1286 
1287 		nvlist_add_number(nvpeer, "peerid", peer->peerid);
1288 
1289 		OVPN_PEER_COUNTER_OUT("packets", pkt_in, pkt_out);
1290 		OVPN_PEER_COUNTER_OUT("bytes", bytes_in, bytes_out);
1291 
1292 		nvlist_append_nvlist_array(*nvl, "peers", nvpeer);
1293 		nvlist_destroy(nvpeer);
1294 	}
1295 #undef OVPN_PEER_COUNTER_OUT
1296 	OVPN_RUNLOCK(sc);
1297 
1298 	return (0);
1299 
1300 error:
1301 	nvlist_destroy(nvpeer);
1302 	nvlist_destroy(*nvl);
1303 	*nvl = NULL;
1304 	return (ret);
1305 }
1306 
1307 static int
1308 ovpn_poll_pkt(struct ovpn_softc *sc, nvlist_t **onvl)
1309 {
1310 	nvlist_t *nvl;
1311 
1312 	nvl = nvlist_create(0);
1313 	if (nvl == NULL)
1314 		return (ENOMEM);
1315 
1316 	nvlist_add_number(nvl, "pending", buf_ring_count(sc->notifring));
1317 
1318 	*onvl = nvl;
1319 
1320 	return (0);
1321 }
1322 
1323 static void
1324 ovpn_notif_add_counters(nvlist_t *parent, struct ovpn_notification *n)
1325 {
1326 	nvlist_t *nvl;
1327 
1328 	nvl = nvlist_create(0);
1329 	if (nvl == NULL)
1330 		return;
1331 
1332 	nvlist_add_number(nvl, "in", n->counters.pkt_in);
1333 	nvlist_add_number(nvl, "out", n->counters.pkt_out);
1334 
1335 	nvlist_add_nvlist(parent, "packets", nvl);
1336 	nvlist_destroy(nvl);
1337 
1338 	nvl = nvlist_create(0);
1339 	if (nvl == NULL)
1340 		return;
1341 
1342 	nvlist_add_number(nvl, "in", n->counters.bytes_in);
1343 	nvlist_add_number(nvl, "out", n->counters.bytes_out);
1344 
1345 	nvlist_add_nvlist(parent, "bytes", nvl);
1346 	nvlist_destroy(nvl);
1347 }
1348 
1349 static int
1350 opvn_get_pkt(struct ovpn_softc *sc, nvlist_t **onvl)
1351 {
1352 	struct ovpn_notification *n;
1353 	nvlist_t *nvl;
1354 
1355 	/* Check if we have notifications pending. */
1356 	n = buf_ring_dequeue_mc(sc->notifring);
1357 	if (n == NULL)
1358 		return (ENOENT);
1359 
1360 	nvl = nvlist_create(0);
1361 	if (nvl == NULL) {
1362 		free(n, M_OVPN);
1363 		return (ENOMEM);
1364 	}
1365 	nvlist_add_number(nvl, "peerid", n->peerid);
1366 	nvlist_add_number(nvl, "notification", n->type);
1367 	if (n->type == OVPN_NOTIF_DEL_PEER) {
1368 		nvlist_add_number(nvl, "del_reason", n->del_reason);
1369 
1370 		/* No error handling, because we want to send the notification
1371 		 * even if we can't attach the counters. */
1372 		ovpn_notif_add_counters(nvl, n);
1373 	}
1374 	free(n, M_OVPN);
1375 
1376 	*onvl = nvl;
1377 
1378 	return (0);
1379 }
1380 
1381 static int
1382 ovpn_ioctl_get(struct ifnet *ifp, struct ifdrv *ifd)
1383 {
1384 	struct ovpn_softc *sc = ifp->if_softc;
1385 	nvlist_t *nvl = NULL;
1386 	int error;
1387 
1388 	switch (ifd->ifd_cmd) {
1389 	case OVPN_GET_STATS:
1390 		error = ovpn_get_stats(sc, &nvl);
1391 		break;
1392 	case OVPN_GET_PEER_STATS:
1393 		error = ovpn_get_peer_stats(sc, &nvl);
1394 		break;
1395 	case OVPN_POLL_PKT:
1396 		error = ovpn_poll_pkt(sc, &nvl);
1397 		break;
1398 	case OVPN_GET_PKT:
1399 		error = opvn_get_pkt(sc, &nvl);
1400 		break;
1401 	default:
1402 		error = ENOTSUP;
1403 		break;
1404 	}
1405 
1406 	if (error == 0) {
1407 		void *packed = NULL;
1408 		size_t len;
1409 
1410 		MPASS(nvl != NULL);
1411 
1412 		packed = nvlist_pack(nvl, &len);
1413 		if (! packed) {
1414 			nvlist_destroy(nvl);
1415 			return (ENOMEM);
1416 		}
1417 
1418 		if (len > ifd->ifd_len) {
1419 			free(packed, M_NVLIST);
1420 			nvlist_destroy(nvl);
1421 			return (ENOSPC);
1422 		}
1423 
1424 		error = copyout(packed, ifd->ifd_data, len);
1425 		ifd->ifd_len = len;
1426 
1427 		free(packed, M_NVLIST);
1428 		nvlist_destroy(nvl);
1429 	}
1430 
1431 	return (error);
1432 }
1433 
1434 static int
1435 ovpn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
1436 {
1437 	struct ifdrv *ifd;
1438 	int error;
1439 
1440 	CURVNET_ASSERT_SET();
1441 
1442 	switch (cmd) {
1443 	case SIOCSDRVSPEC:
1444 	case SIOCGDRVSPEC:
1445 		error = priv_check(curthread, PRIV_NET_OVPN);
1446 		if (error)
1447 			return (error);
1448 		break;
1449 	}
1450 
1451 	switch (cmd) {
1452 	case SIOCSDRVSPEC:
1453 		ifd = (struct ifdrv *)data;
1454 		error = ovpn_ioctl_set(ifp, ifd);
1455 		break;
1456 	case SIOCGDRVSPEC:
1457 		ifd = (struct ifdrv *)data;
1458 		error = ovpn_ioctl_get(ifp, ifd);
1459 		break;
1460 	case SIOCSIFMTU: {
1461 		struct ifreq *ifr = (struct ifreq *)data;
1462 		if (ifr->ifr_mtu < OVPN_MTU_MIN || ifr->ifr_mtu > OVPN_MTU_MAX)
1463 			return (EINVAL);
1464 
1465 		ifp->if_mtu = ifr->ifr_mtu;
1466 		return (0);
1467 	}
1468 	case SIOCSIFADDR:
1469 	case SIOCADDMULTI:
1470 	case SIOCDELMULTI:
1471 	case SIOCGIFMTU:
1472 	case SIOCSIFFLAGS:
1473 		return (0);
1474 	default:
1475 		error = EINVAL;
1476 	}
1477 
1478 	return (error);
1479 }
1480 
1481 static int
1482 ovpn_encrypt_tx_cb(struct cryptop *crp)
1483 {
1484 	struct epoch_tracker et;
1485 	struct ovpn_kpeer *peer = crp->crp_opaque;
1486 	struct ovpn_softc *sc = peer->sc;
1487 	struct mbuf *m = crp->crp_buf.cb_mbuf;
1488 	int tunnel_len;
1489 	int ret;
1490 
1491 	CURVNET_SET(sc->ifp->if_vnet);
1492 	NET_EPOCH_ENTER(et);
1493 
1494 	if (crp->crp_etype != 0) {
1495 		crypto_freereq(crp);
1496 		ovpn_peer_release_ref(peer, false);
1497 		NET_EPOCH_EXIT(et);
1498 		CURVNET_RESTORE();
1499 		OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
1500 		m_freem(m);
1501 		return (0);
1502 	}
1503 
1504 	MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF);
1505 
1506 	tunnel_len = m->m_pkthdr.len - sizeof(struct ovpn_wire_header);
1507 	ret = ovpn_encap(sc, peer->peerid, m);
1508 	if (ret == 0) {
1509 		OVPN_COUNTER_ADD(sc, sent_data_pkts, 1);
1510 		OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len);
1511 	}
1512 
1513 	crypto_freereq(crp);
1514 	ovpn_peer_release_ref(peer, false);
1515 
1516 	NET_EPOCH_EXIT(et);
1517 	CURVNET_RESTORE();
1518 
1519 	return (0);
1520 }
1521 
1522 static void
1523 ovpn_finish_rx(struct ovpn_softc *sc, struct mbuf *m,
1524     struct ovpn_kpeer *peer, struct ovpn_kkey *key, uint32_t seq,
1525     struct rm_priotracker *_ovpn_lock_trackerp)
1526 {
1527 	uint32_t af;
1528 
1529 	OVPN_RASSERT(sc);
1530 	NET_EPOCH_ASSERT();
1531 
1532 	/* Replay protection. */
1533 	if (V_replay_protection && ! ovpn_check_replay(key->decrypt, seq)) {
1534 		OVPN_RUNLOCK(sc);
1535 		OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1536 		m_freem(m);
1537 		return;
1538 	}
1539 
1540 	critical_enter();
1541 	*zpcpu_get(peer->last_active) = time_uptime;
1542 	critical_exit();
1543 
1544 	OVPN_RUNLOCK(sc);
1545 
1546 	OVPN_COUNTER_ADD(sc, received_data_pkts, 1);
1547 	OVPN_COUNTER_ADD(sc, tunnel_bytes_received, m->m_pkthdr.len);
1548 	OVPN_PEER_COUNTER_ADD(peer, pkt_in, 1);
1549 	OVPN_PEER_COUNTER_ADD(peer, bytes_in, m->m_pkthdr.len);
1550 
1551 	/* Receive the packet on our interface. */
1552 	m->m_pkthdr.rcvif = sc->ifp;
1553 
1554 	/* Clear checksum flags in case the real hardware set them. */
1555 	m->m_pkthdr.csum_flags = 0;
1556 
1557 	/* Clear mbuf tags & flags */
1558 	m_tag_delete_nonpersistent(m);
1559 	m_clrprotoflags(m);
1560 
1561 	/* Ensure we can read the first byte. */
1562 	m = m_pullup(m, 1);
1563 	if (m == NULL) {
1564 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1);
1565 		return;
1566 	}
1567 
1568 	/*
1569 	 * Check for address family, and disregard any control packets (e.g.
1570 	 * keepalive).
1571 	 */
1572 	af = ovpn_get_af(m);
1573 	if (af != 0) {
1574 		BPF_MTAP2(sc->ifp, &af, sizeof(af), m);
1575 		if (V_async_netisr_queue)
1576 			netisr_queue(af == AF_INET ? NETISR_IP : NETISR_IPV6, m);
1577 		else
1578 			netisr_dispatch(af == AF_INET ? NETISR_IP : NETISR_IPV6, m);
1579 	} else {
1580 		OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1581 		m_freem(m);
1582 	}
1583 }
1584 
1585 static struct ovpn_kkey *
1586 ovpn_find_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer,
1587     const struct ovpn_wire_header *ohdr)
1588 {
1589 	struct ovpn_kkey *key = NULL;
1590 	uint8_t keyid;
1591 
1592 	OVPN_RASSERT(sc);
1593 
1594 	keyid = (ntohl(ohdr->opcode) >> 24) & 0x07;
1595 
1596 	if (peer->keys[0].keyid == keyid)
1597 		key = &peer->keys[0];
1598 	else if (peer->keys[1].keyid == keyid)
1599 		key = &peer->keys[1];
1600 
1601 	return (key);
1602 }
1603 
1604 static int
1605 ovpn_decrypt_rx_cb(struct cryptop *crp)
1606 {
1607 	struct epoch_tracker et;
1608 	struct ovpn_softc *sc = crp->crp_opaque;
1609 	struct mbuf *m = crp->crp_buf.cb_mbuf;
1610 	struct ovpn_kkey *key;
1611 	struct ovpn_kpeer *peer;
1612 	struct ovpn_wire_header *ohdr;
1613 	uint32_t peerid;
1614 
1615 	OVPN_RLOCK_TRACKER;
1616 
1617 	OVPN_RLOCK(sc);
1618 
1619 	MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF);
1620 
1621 	if (crp->crp_etype != 0) {
1622 		crypto_freereq(crp);
1623 		atomic_add_int(&sc->refcount, -1);
1624 		OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1625 		OVPN_RUNLOCK(sc);
1626 		m_freem(m);
1627 		return (0);
1628 	}
1629 
1630 	CURVNET_SET(sc->ifp->if_vnet);
1631 
1632 	ohdr = mtodo(m, sizeof(struct udphdr));
1633 
1634 	peerid = ntohl(ohdr->opcode) & 0x00ffffff;
1635 	peer = ovpn_find_peer(sc, peerid);
1636 	if (peer == NULL) {
1637 		/* No such peer. Drop packet. */
1638 		crypto_freereq(crp);
1639 		atomic_add_int(&sc->refcount, -1);
1640 		OVPN_RUNLOCK(sc);
1641 		OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1642 		m_freem(m);
1643 		CURVNET_RESTORE();
1644 		return (0);
1645 	}
1646 
1647 	key = ovpn_find_key(sc, peer, ohdr);
1648 	if (key == NULL) {
1649 		crypto_freereq(crp);
1650 		atomic_add_int(&sc->refcount, -1);
1651 		/*
1652 		 * Has this key been removed between us starting the decrypt
1653 		 * and finishing it?
1654 		 */
1655 		OVPN_RUNLOCK(sc);
1656 		OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1657 		m_freem(m);
1658 		CURVNET_RESTORE();
1659 		return (0);
1660 	}
1661 
1662 	/* Now remove the outer headers */
1663 	m_adj_decap(m, sizeof(struct udphdr) +
1664 	    sizeof(struct ovpn_wire_header));
1665 
1666 	NET_EPOCH_ENTER(et);
1667 	ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), _ovpn_lock_trackerp);
1668 	NET_EPOCH_EXIT(et);
1669 	OVPN_UNLOCK_ASSERT(sc);
1670 
1671 	CURVNET_RESTORE();
1672 
1673 	crypto_freereq(crp);
1674 	atomic_add_int(&sc->refcount, -1);
1675 
1676 	return (0);
1677 }
1678 
1679 static int
1680 ovpn_get_af(struct mbuf *m)
1681 {
1682 	struct ip *ip;
1683 	struct ip6_hdr *ip6;
1684 
1685 	/*
1686 	 * We should pullup, but we're only interested in the first byte, so
1687 	 * that'll always be contiguous.
1688 	 */
1689 	ip = mtod(m, struct ip *);
1690 	if (ip->ip_v == IPVERSION)
1691 		return (AF_INET);
1692 
1693 	ip6 = mtod(m, struct ip6_hdr *);
1694 	if ((ip6->ip6_vfc & IPV6_VERSION_MASK) == IPV6_VERSION)
1695 		return (AF_INET6);
1696 
1697 	return (0);
1698 }
1699 
1700 #ifdef INET
1701 static struct ovpn_kpeer *
1702 ovpn_find_peer_by_ip(struct ovpn_softc *sc, const struct in_addr addr)
1703 {
1704 	struct ovpn_kpeer *peer = NULL;
1705 
1706 	OVPN_ASSERT(sc);
1707 
1708 	/* TODO: Add a second RB so we can look up by IP. */
1709 	RB_FOREACH(peer, ovpn_kpeers, &sc->peers) {
1710 		if (addr.s_addr == peer->vpn4.s_addr)
1711 			return (peer);
1712 	}
1713 
1714 	return (peer);
1715 }
1716 #endif
1717 
1718 #ifdef INET6
1719 static struct ovpn_kpeer *
1720 ovpn_find_peer_by_ip6(struct ovpn_softc *sc, const struct in6_addr *addr)
1721 {
1722 	struct ovpn_kpeer *peer = NULL;
1723 
1724 	OVPN_ASSERT(sc);
1725 
1726 	/* TODO: Add a third RB so we can look up by IPv6 address. */
1727 	RB_FOREACH(peer, ovpn_kpeers, &sc->peers) {
1728 		if (memcmp(addr, &peer->vpn6, sizeof(*addr)) == 0)
1729 			return (peer);
1730 	}
1731 
1732 	return (peer);
1733 }
1734 #endif
1735 
1736 static struct ovpn_kpeer *
1737 ovpn_route_peer(struct ovpn_softc *sc, struct mbuf **m0,
1738     const struct sockaddr *dst)
1739 {
1740 	struct ovpn_kpeer *peer = NULL;
1741 	int af;
1742 
1743 	NET_EPOCH_ASSERT();
1744 	OVPN_ASSERT(sc);
1745 
1746 	/* Shortcut if we're a client (or are a server and have only one client). */
1747 	if (sc->peercount == 1)
1748 		return (ovpn_find_only_peer(sc));
1749 
1750 	if (dst != NULL)
1751 		af = dst->sa_family;
1752 	else
1753 		af = ovpn_get_af(*m0);
1754 
1755 	switch (af) {
1756 #ifdef INET
1757 	case AF_INET: {
1758 		const struct sockaddr_in *sa = (const struct sockaddr_in *)dst;
1759 		struct nhop_object *nh;
1760 		const struct in_addr *ip_dst;
1761 
1762 		if (sa != NULL) {
1763 			ip_dst = &sa->sin_addr;
1764 		} else {
1765 			struct ip *ip;
1766 
1767 			*m0 = m_pullup(*m0, sizeof(struct ip));
1768 			if (*m0 == NULL)
1769 				return (NULL);
1770 			ip = mtod(*m0, struct ip *);
1771 			ip_dst = &ip->ip_dst;
1772 		}
1773 
1774 		peer = ovpn_find_peer_by_ip(sc, *ip_dst);
1775 		SDT_PROBE2(if_ovpn, tx, route, ip4, ip_dst, peer);
1776 		if (peer == NULL) {
1777 			nh = fib4_lookup(M_GETFIB(*m0), *ip_dst, 0,
1778 			    NHR_NONE, 0);
1779 			if (nh && (nh->nh_flags & NHF_GATEWAY)) {
1780 				peer = ovpn_find_peer_by_ip(sc,
1781 				    nh->gw4_sa.sin_addr);
1782 				SDT_PROBE2(if_ovpn, tx, route, ip4,
1783 				    &nh->gw4_sa.sin_addr, peer);
1784 			}
1785 		}
1786 		break;
1787 	}
1788 #endif
1789 #ifdef INET6
1790 	case AF_INET6: {
1791 		const struct sockaddr_in6 *sa6 =
1792 		    (const struct sockaddr_in6 *)dst;
1793 		struct nhop_object *nh;
1794 		const struct in6_addr *ip6_dst;
1795 
1796 		if (sa6 != NULL) {
1797 			ip6_dst = &sa6->sin6_addr;
1798 		} else {
1799 			struct ip6_hdr *ip6;
1800 
1801 			*m0 = m_pullup(*m0, sizeof(struct ip6_hdr));
1802 			if (*m0 == NULL)
1803 				return (NULL);
1804 			ip6 = mtod(*m0, struct ip6_hdr *);
1805 			ip6_dst = &ip6->ip6_dst;
1806 		}
1807 
1808 		peer = ovpn_find_peer_by_ip6(sc, ip6_dst);
1809 		SDT_PROBE2(if_ovpn, tx, route, ip6, ip6_dst, peer);
1810 		if (peer == NULL) {
1811 			nh = fib6_lookup(M_GETFIB(*m0), ip6_dst, 0,
1812 			    NHR_NONE, 0);
1813 			if (nh && (nh->nh_flags & NHF_GATEWAY)) {
1814 				peer = ovpn_find_peer_by_ip6(sc,
1815 				    &nh->gw6_sa.sin6_addr);
1816 				SDT_PROBE2(if_ovpn, tx, route, ip6,
1817 				    &nh->gw6_sa.sin6_addr, peer);
1818 			}
1819 		}
1820 		break;
1821 	}
1822 #endif
1823 	}
1824 
1825 	return (peer);
1826 }
1827 
1828 static int
1829 ovpn_transmit(struct ifnet *ifp, struct mbuf *m)
1830 {
1831 	return (ifp->if_output(ifp, m, NULL, NULL));
1832 }
1833 
1834 static int
1835 ovpn_transmit_to_peer(struct ifnet *ifp, struct mbuf *m,
1836     struct ovpn_kpeer *peer, struct rm_priotracker *_ovpn_lock_trackerp)
1837 {
1838 	struct ovpn_wire_header *ohdr;
1839 	struct ovpn_kkey *key;
1840 	struct ovpn_softc *sc;
1841 	struct cryptop *crp;
1842 	uint32_t af, seq;
1843 	uint64_t seq64;
1844 	size_t len, ovpn_hdr_len;
1845 	int tunnel_len;
1846 	int ret;
1847 
1848 	sc = ifp->if_softc;
1849 
1850 	OVPN_RASSERT(sc);
1851 
1852 	tunnel_len = m->m_pkthdr.len;
1853 
1854 	key = &peer->keys[OVPN_KEY_SLOT_PRIMARY];
1855 	if (key->encrypt == NULL) {
1856 		if (_ovpn_lock_trackerp != NULL)
1857 			OVPN_RUNLOCK(sc);
1858 		m_freem(m);
1859 		return (ENOLINK);
1860 	}
1861 
1862 	af = ovpn_get_af(m);
1863 	/* Don't capture control packets. */
1864 	if (af != 0)
1865 		BPF_MTAP2(ifp, &af, sizeof(af), m);
1866 
1867 	if (__predict_false(if_tunnel_check_nesting(ifp, m, MTAG_OVPN_LOOP, 3))) {
1868 		if (_ovpn_lock_trackerp != NULL)
1869 			OVPN_RUNLOCK(sc);
1870 		OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
1871 		m_freem(m);
1872 		return (ELOOP);
1873 	}
1874 
1875 	len = m->m_pkthdr.len;
1876 	MPASS(len <= ifp->if_mtu);
1877 
1878 	ovpn_hdr_len = sizeof(struct ovpn_wire_header);
1879 	if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE)
1880 		ovpn_hdr_len -= 16; /* No auth tag. */
1881 
1882 	M_PREPEND(m, ovpn_hdr_len, M_NOWAIT);
1883 	if (m == NULL) {
1884 		if (_ovpn_lock_trackerp != NULL)
1885 			OVPN_RUNLOCK(sc);
1886 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1887 		return (ENOBUFS);
1888 	}
1889 	ohdr = mtod(m, struct ovpn_wire_header *);
1890 	ohdr->opcode = (OVPN_OP_DATA_V2 << OVPN_OP_SHIFT) | key->keyid;
1891 	ohdr->opcode <<= 24;
1892 	ohdr->opcode |= key->peerid;
1893 	ohdr->opcode = htonl(ohdr->opcode);
1894 
1895 	seq64 = atomic_fetchadd_64(&peer->keys[OVPN_KEY_SLOT_PRIMARY].encrypt->tx_seq, 1);
1896 	if (seq64 == OVPN_SEQ_ROTATE) {
1897 		ovpn_notify_key_rotation(sc, peer);
1898 	} else if (seq64 > UINT32_MAX) {
1899 		/* We've wrapped, give up on this packet. */
1900 		if (_ovpn_lock_trackerp != NULL)
1901 			OVPN_RUNLOCK(sc);
1902 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1903 
1904 		/* Let's avoid (very unlikely, but still) wraparounds of the
1905 		 * 64-bit counter taking us back to 0. */
1906 		atomic_store_64(&peer->keys[OVPN_KEY_SLOT_PRIMARY].encrypt->tx_seq,
1907 		    UINT32_MAX);
1908 
1909 		return (ENOBUFS);
1910 	}
1911 
1912 	seq = htonl(seq64 & UINT32_MAX);
1913 	ohdr->seq = seq;
1914 
1915 	OVPN_PEER_COUNTER_ADD(peer, pkt_out, 1);
1916 	OVPN_PEER_COUNTER_ADD(peer, bytes_out, len);
1917 
1918 	if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) {
1919 		ret = ovpn_encap(sc, peer->peerid, m);
1920 		if (_ovpn_lock_trackerp != NULL)
1921 			OVPN_RUNLOCK(sc);
1922 		if (ret == 0) {
1923 			OVPN_COUNTER_ADD(sc, sent_data_pkts, 1);
1924 			OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len);
1925 		}
1926 		return (ret);
1927 	}
1928 
1929 	crp = crypto_getreq(key->encrypt->cryptoid, M_NOWAIT);
1930 	if (crp == NULL) {
1931 		if (_ovpn_lock_trackerp != NULL)
1932 			OVPN_RUNLOCK(sc);
1933 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1934 		m_freem(m);
1935 		return (ENOBUFS);
1936 	}
1937 
1938 	/* Encryption covers only the payload, not the header. */
1939 	crp->crp_payload_start = sizeof(*ohdr);
1940 	crp->crp_payload_length = len;
1941 	crp->crp_op = CRYPTO_OP_ENCRYPT;
1942 
1943 	/*
1944 	 * AAD data covers the ovpn_wire_header minus the auth
1945 	 * tag.
1946 	 */
1947 	crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag);
1948 	crp->crp_aad = ohdr;
1949 	crp->crp_aad_start = 0;
1950 	crp->crp_op |= CRYPTO_OP_COMPUTE_DIGEST;
1951 	crp->crp_digest_start = offsetof(struct ovpn_wire_header, auth_tag);
1952 
1953 	crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
1954 	memcpy(crp->crp_iv, &seq, sizeof(seq));
1955 	memcpy(crp->crp_iv + sizeof(seq), key->encrypt->nonce,
1956 	    key->encrypt->noncelen);
1957 
1958 	crypto_use_mbuf(crp, m);
1959 	crp->crp_flags |= CRYPTO_F_CBIFSYNC;
1960 	crp->crp_callback = ovpn_encrypt_tx_cb;
1961 	crp->crp_opaque = peer;
1962 
1963 	atomic_add_int(&peer->refcount, 1);
1964 	if (_ovpn_lock_trackerp != NULL)
1965 		OVPN_RUNLOCK(sc);
1966 	if (V_async_crypto)
1967 		ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED);
1968 	else
1969 		ret = crypto_dispatch(crp);
1970 	if (ret) {
1971 		OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
1972 	}
1973 
1974 	return (ret);
1975 }
1976 
1977 /*
1978  * Note: Expects to hold the read lock on entry, and will release it itself.
1979  */
1980 static int
1981 ovpn_encap(struct ovpn_softc *sc, uint32_t peerid, struct mbuf *m)
1982 {
1983 	struct udphdr *udp;
1984 	struct ovpn_kpeer *peer;
1985 	int len;
1986 
1987 	OVPN_RLOCK_TRACKER;
1988 
1989 	OVPN_RLOCK(sc);
1990 	NET_EPOCH_ASSERT();
1991 
1992 	peer = ovpn_find_peer(sc, peerid);
1993 	if (peer == NULL || sc->ifp->if_link_state != LINK_STATE_UP) {
1994 		OVPN_RUNLOCK(sc);
1995 		OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
1996 		m_freem(m);
1997 		return (ENETDOWN);
1998 	}
1999 
2000 	len = m->m_pkthdr.len;
2001 
2002 	M_PREPEND(m, sizeof(struct udphdr), M_NOWAIT);
2003 	if (m == NULL) {
2004 		OVPN_RUNLOCK(sc);
2005 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
2006 		m_freem(m);
2007 		return (ENOBUFS);
2008 	}
2009 	udp = mtod(m, struct udphdr *);
2010 
2011 	MPASS(peer->local.ss_family == peer->remote.ss_family);
2012 
2013 	udp->uh_sport = ovpn_get_port(&peer->local);
2014 	udp->uh_dport = ovpn_get_port(&peer->remote);
2015 	udp->uh_ulen = htons(sizeof(struct udphdr) + len);
2016 
2017 	switch (peer->remote.ss_family) {
2018 #ifdef INET
2019 	case AF_INET: {
2020 		struct sockaddr_in *in_local = TO_IN(&peer->local);
2021 		struct sockaddr_in *in_remote = TO_IN(&peer->remote);
2022 		struct ip *ip;
2023 
2024 		/*
2025 		 * This requires knowing the source IP, which we don't. Happily
2026 		 * we're allowed to keep this at 0, and the checksum won't do
2027 		 * anything the crypto won't already do.
2028 		 */
2029 		udp->uh_sum = 0;
2030 
2031 		/* Set the checksum flags so we recalculate checksums. */
2032 		m->m_pkthdr.csum_flags |= CSUM_IP;
2033 		m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
2034 
2035 		M_PREPEND(m, sizeof(struct ip), M_NOWAIT);
2036 		if (m == NULL) {
2037 			OVPN_RUNLOCK(sc);
2038 			OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
2039 			return (ENOBUFS);
2040 		}
2041 		ip = mtod(m, struct ip *);
2042 
2043 		ip->ip_tos = 0;
2044 		ip->ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) +
2045 		   len);
2046 		ip->ip_off = 0;
2047 		ip->ip_ttl = V_ip_defttl;
2048 		ip->ip_p = IPPROTO_UDP;
2049 		ip->ip_sum = 0;
2050 		if (in_local->sin_port != 0)
2051 			ip->ip_src = in_local->sin_addr;
2052 		else
2053 			ip->ip_src.s_addr = INADDR_ANY;
2054 		ip->ip_dst = in_remote->sin_addr;
2055 
2056 		OVPN_RUNLOCK(sc);
2057 		OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len);
2058 
2059 		return (ip_output(m, NULL, NULL, 0, NULL, NULL));
2060 	}
2061 #endif
2062 #ifdef INET6
2063 	case AF_INET6: {
2064 		struct sockaddr_in6 *in6_local = TO_IN6(&peer->local);
2065 		struct sockaddr_in6 *in6_remote = TO_IN6(&peer->remote);
2066 		struct ip6_hdr *ip6;
2067 
2068 		M_PREPEND(m, sizeof(struct ip6_hdr), M_NOWAIT);
2069 		if (m == NULL) {
2070 			OVPN_RUNLOCK(sc);
2071 			OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
2072 			return (ENOBUFS);
2073 		}
2074 		m = m_pullup(m, sizeof(*ip6) + sizeof(*udp));
2075 		if (m == NULL) {
2076 			OVPN_RUNLOCK(sc);
2077 			OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
2078 			return (ENOBUFS);
2079 		}
2080 
2081 		ip6 = mtod(m, struct ip6_hdr *);
2082 
2083 		ip6->ip6_vfc = IPV6_VERSION;
2084 		ip6->ip6_flow &= ~IPV6_FLOWINFO_MASK;
2085 		ip6->ip6_plen = htons(sizeof(*ip6) + sizeof(struct udphdr) +
2086 		    len);
2087 		ip6->ip6_nxt = IPPROTO_UDP;
2088 		ip6->ip6_hlim = V_ip6_defhlim;
2089 
2090 		memcpy(&ip6->ip6_src, &in6_local->sin6_addr,
2091 		    sizeof(ip6->ip6_src));
2092 		memcpy(&ip6->ip6_dst, &in6_remote->sin6_addr,
2093 		    sizeof(ip6->ip6_dst));
2094 
2095 		udp = mtodo(m, sizeof(*ip6));
2096 		udp->uh_sum = in6_cksum_pseudo(ip6,
2097 		    m->m_pkthdr.len - sizeof(struct ip6_hdr),
2098 		    IPPROTO_UDP, 0);
2099 
2100 		m->m_pkthdr.csum_flags |= CSUM_UDP_IPV6;
2101 		m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
2102 
2103 		OVPN_RUNLOCK(sc);
2104 		OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len);
2105 
2106 		return (ip6_output(m, NULL, NULL, IPV6_UNSPECSRC, NULL, NULL,
2107 		    NULL));
2108 	}
2109 #endif
2110 	default:
2111 		panic("Unsupported address family %d",
2112 		    peer->remote.ss_family);
2113 	}
2114 }
2115 
2116 static int
2117 ovpn_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
2118 	struct route *ro)
2119 {
2120 	struct ovpn_softc *sc;
2121 	struct ovpn_kpeer *peer;
2122 
2123 	OVPN_RLOCK_TRACKER;
2124 
2125 	sc = ifp->if_softc;
2126 
2127 	m = m_unshare(m, M_NOWAIT);
2128 	if (m == NULL) {
2129 		OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
2130 		return (ENOBUFS);
2131 	}
2132 
2133 	OVPN_RLOCK(sc);
2134 
2135 	SDT_PROBE1(if_ovpn, tx, transmit, start, m);
2136 
2137 	if (__predict_false(ifp->if_link_state != LINK_STATE_UP)) {
2138 		OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
2139 		OVPN_RUNLOCK(sc);
2140 		m_freem(m);
2141 		return (ENETDOWN);
2142 	}
2143 
2144 	/**
2145 	 * Only obey 'dst' (i.e. the gateway) if no route is supplied.
2146 	 * That's our indication that we're being called through pf's route-to,
2147 	 * and we should route according to 'dst' instead. We can't do so
2148 	 * consistently, because the usual openvpn configuration sets the first
2149 	 * non-server IP in the subnet as the gateway. If we always use that
2150 	 * one we'd end up routing all traffic to the first client.
2151 	 * tl;dr: 'ro == NULL' tells us pf is doing a route-to, and then but
2152 	 * only then, we should treat 'dst' as the destination. */
2153 	peer = ovpn_route_peer(sc, &m, ro == NULL ? dst : NULL);
2154 	if (peer == NULL) {
2155 		/* No destination. */
2156 		OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
2157 		OVPN_RUNLOCK(sc);
2158 		m_freem(m);
2159 		return (ENETDOWN);
2160 	}
2161 
2162 	return (ovpn_transmit_to_peer(ifp, m, peer, _ovpn_lock_trackerp));
2163 }
2164 
2165 static bool
2166 ovpn_check_replay(struct ovpn_kkey_dir *key, uint32_t seq)
2167 {
2168 	uint32_t d;
2169 
2170 	mtx_lock(&key->replay_mtx);
2171 
2172 	/* Sequence number must be strictly greater than rx_seq */
2173 	if (seq <= key->rx_seq) {
2174 		mtx_unlock(&key->replay_mtx);
2175 		return (false);
2176 	}
2177 
2178 	/* Large jump. The packet authenticated okay, so just accept that. */
2179 	if (seq > (key->rx_seq + (sizeof(key->rx_window) * 8))) {
2180 		key->rx_seq = seq;
2181 		key->rx_window = 0;
2182 		mtx_unlock(&key->replay_mtx);
2183 		return (true);
2184 	}
2185 
2186 	/* Happy case. */
2187 	if ((seq == key->rx_seq + 1) && key->rx_window == 0) {
2188 		key->rx_seq++;
2189 		mtx_unlock(&key->replay_mtx);
2190 		return (true);
2191 	}
2192 
2193 	d = seq - key->rx_seq - 1;
2194 
2195 	if (key->rx_window & ((uint64_t)1 << d)) {
2196 		/* Dupe! */
2197 		mtx_unlock(&key->replay_mtx);
2198 		return (false);
2199 	}
2200 
2201 	key->rx_window |= (uint64_t)1 << d;
2202 
2203 	while (key->rx_window & 1) {
2204 		key->rx_seq++;
2205 		key->rx_window >>= 1;
2206 	}
2207 
2208 	mtx_unlock(&key->replay_mtx);
2209 
2210 	return (true);
2211 }
2212 
2213 static struct ovpn_kpeer *
2214 ovpn_peer_from_mbuf(struct ovpn_softc *sc, struct mbuf *m, int off)
2215 {
2216 	struct ovpn_wire_header ohdr;
2217 	uint32_t peerid;
2218 	const size_t hdrlen = sizeof(ohdr) - sizeof(ohdr.auth_tag);
2219 
2220 	OVPN_RASSERT(sc);
2221 
2222 	if (m_length(m, NULL) < (off + sizeof(struct udphdr) + hdrlen))
2223 		return (NULL);
2224 
2225 	m_copydata(m, off + sizeof(struct udphdr), hdrlen, (caddr_t)&ohdr);
2226 
2227 	peerid = ntohl(ohdr.opcode) & 0x00ffffff;
2228 
2229 	return (ovpn_find_peer(sc, peerid));
2230 }
2231 
2232 static bool
2233 ovpn_udp_input(struct mbuf *m, int off, struct inpcb *inp,
2234     const struct sockaddr *sa, void *ctx)
2235 {
2236 	struct ovpn_softc *sc = ctx;
2237 	struct ovpn_wire_header tmphdr;
2238 	struct ovpn_wire_header *ohdr;
2239 	struct udphdr *uhdr;
2240 	struct ovpn_kkey *key;
2241 	struct cryptop *crp;
2242 	struct ovpn_kpeer *peer;
2243 	size_t ohdrlen;
2244 	int ret;
2245 	uint8_t op;
2246 
2247 	OVPN_RLOCK_TRACKER;
2248 
2249 	M_ASSERTPKTHDR(m);
2250 
2251 	m = m_unshare(m, M_NOWAIT);
2252 	if (m == NULL) {
2253 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1);
2254 		return (true);
2255 	}
2256 
2257 	OVPN_COUNTER_ADD(sc, transport_bytes_received, m->m_pkthdr.len - off);
2258 
2259 	ohdrlen = sizeof(*ohdr) - sizeof(ohdr->auth_tag);
2260 
2261 	OVPN_RLOCK(sc);
2262 
2263 	peer = ovpn_peer_from_mbuf(sc, m, off);
2264 	if (peer == NULL) {
2265 		OVPN_RUNLOCK(sc);
2266 		return (false);
2267 	}
2268 
2269 	if (m_length(m, NULL) < (off + sizeof(*uhdr) + ohdrlen)) {
2270 		/* Short packet. */
2271 		OVPN_RUNLOCK(sc);
2272 		return (false);
2273 	}
2274 
2275 	m_copydata(m, off + sizeof(*uhdr), ohdrlen, (caddr_t)&tmphdr);
2276 
2277 	op = ntohl(tmphdr.opcode) >> 24 >> OVPN_OP_SHIFT;
2278 	if (op != OVPN_OP_DATA_V2) {
2279 		/* Control packet? */
2280 		OVPN_RUNLOCK(sc);
2281 		return (false);
2282 	}
2283 
2284 	m = m_pullup(m, off + sizeof(*uhdr) + ohdrlen);
2285 	if (m == NULL) {
2286 		OVPN_RUNLOCK(sc);
2287 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1);
2288 		return (true);
2289 	}
2290 
2291 	/*
2292 	 * Simplify things by getting rid of the preceding headers, we don't
2293 	 * care about them.
2294 	 */
2295 	m_adj_decap(m, off);
2296 
2297 	uhdr = mtodo(m, 0);
2298 	ohdr = mtodo(m, sizeof(*uhdr));
2299 
2300 	key = ovpn_find_key(sc, peer, ohdr);
2301 	if (key == NULL || key->decrypt == NULL) {
2302 		OVPN_RUNLOCK(sc);
2303 		OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
2304 		m_freem(m);
2305 		return (true);
2306 	}
2307 
2308 	if (key->decrypt->cipher == OVPN_CIPHER_ALG_NONE) {
2309 		/* Now remove the outer headers */
2310 		m_adj_decap(m, sizeof(struct udphdr) + ohdrlen);
2311 
2312 		ohdr = mtodo(m, sizeof(*uhdr));
2313 
2314 		ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq),
2315 		    _ovpn_lock_trackerp);
2316 		OVPN_UNLOCK_ASSERT(sc);
2317 		return (true);
2318 	}
2319 
2320 	ohdrlen += sizeof(ohdr->auth_tag);
2321 
2322 	m = m_pullup(m, sizeof(*uhdr) + ohdrlen);
2323 	if (m == NULL) {
2324 		OVPN_RUNLOCK(sc);
2325 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1);
2326 		return (true);
2327 	}
2328 	uhdr = mtodo(m, 0);
2329 	ohdr = mtodo(m, sizeof(*uhdr));
2330 
2331 	/* Decrypt */
2332 	crp = crypto_getreq(key->decrypt->cryptoid, M_NOWAIT);
2333 	if (crp == NULL) {
2334 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1);
2335 		OVPN_RUNLOCK(sc);
2336 		m_freem(m);
2337 		return (true);
2338 	}
2339 
2340 	crp->crp_payload_start = sizeof(struct udphdr) + sizeof(*ohdr);
2341 	crp->crp_payload_length = ntohs(uhdr->uh_ulen) -
2342 	    sizeof(*uhdr) - sizeof(*ohdr);
2343 	crp->crp_op = CRYPTO_OP_DECRYPT;
2344 
2345 	/* AAD validation. */
2346 	crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag);
2347 	crp->crp_aad = ohdr;
2348 	crp->crp_aad_start = 0;
2349 	crp->crp_op |= CRYPTO_OP_VERIFY_DIGEST;
2350 	crp->crp_digest_start = sizeof(struct udphdr) +
2351 	    offsetof(struct ovpn_wire_header, auth_tag);
2352 
2353 	crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
2354 	memcpy(crp->crp_iv, &ohdr->seq, sizeof(ohdr->seq));
2355 	memcpy(crp->crp_iv + sizeof(ohdr->seq), key->decrypt->nonce,
2356 	    key->decrypt->noncelen);
2357 
2358 	crypto_use_mbuf(crp, m);
2359 	crp->crp_flags |= CRYPTO_F_CBIFSYNC;
2360 	crp->crp_callback = ovpn_decrypt_rx_cb;
2361 	crp->crp_opaque = sc;
2362 
2363 	atomic_add_int(&sc->refcount, 1);
2364 	OVPN_RUNLOCK(sc);
2365 	if (V_async_crypto)
2366 		ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED);
2367 	else
2368 		ret = crypto_dispatch(crp);
2369 	if (ret != 0) {
2370 		OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
2371 	}
2372 
2373 	return (true);
2374 }
2375 
2376 static void
2377 ovpn_qflush(struct ifnet *ifp __unused)
2378 {
2379 
2380 }
2381 
2382 static void
2383 ovpn_flush_rxring(struct ovpn_softc *sc)
2384 {
2385 	struct ovpn_notification *n;
2386 
2387 	OVPN_WASSERT(sc);
2388 
2389 	while (! buf_ring_empty(sc->notifring)) {
2390 		n = buf_ring_dequeue_sc(sc->notifring);
2391 		free(n, M_OVPN);
2392 	}
2393 }
2394 
2395 #ifdef VIMAGE
2396 static void
2397 ovpn_reassign(struct ifnet *ifp, struct vnet *new_vnet __unused,
2398     char *unused __unused)
2399 {
2400 	struct ovpn_softc *sc = ifp->if_softc;
2401 	struct ovpn_kpeer *peer, *tmppeer;
2402 	int ret __diagused;
2403 
2404 	OVPN_WLOCK(sc);
2405 
2406 	/* Flush keys & configuration. */
2407 	RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) {
2408 		peer->del_reason = OVPN_DEL_REASON_REQUESTED;
2409 		ret = _ovpn_del_peer(sc, peer);
2410 		MPASS(ret == 0);
2411 	}
2412 
2413 	ovpn_flush_rxring(sc);
2414 
2415 	OVPN_WUNLOCK(sc);
2416 }
2417 #endif
2418 
2419 static int
2420 ovpn_clone_match(struct if_clone *ifc, const char *name)
2421 {
2422 	/*
2423 	 * Allow all names that start with 'ovpn', specifically because pfSense
2424 	 * uses ovpnc1 / ovpns2
2425 	 */
2426 	return (strncmp(ovpnname, name, strlen(ovpnname)) == 0);
2427 }
2428 
2429 static int
2430 ovpn_clone_create(struct if_clone *ifc, char *name, size_t len,
2431     struct ifc_data *ifd, struct ifnet **ifpp)
2432 {
2433 	struct ovpn_softc *sc;
2434 	struct ifnet *ifp;
2435 	char *dp;
2436 	int error, unit, wildcard;
2437 
2438 	/* Try to see if a special unit was requested. */
2439 	error = ifc_name2unit(name, &unit);
2440 	if (error != 0)
2441 		return (error);
2442 	wildcard = (unit < 0);
2443 
2444 	error = ifc_alloc_unit(ifc, &unit);
2445 	if (error != 0)
2446 		return (error);
2447 
2448 	/*
2449 	 * If no unit had been given, we need to adjust the ifName.
2450 	 */
2451 	for (dp = name; *dp != '\0'; dp++);
2452 	if (wildcard) {
2453 		error = snprintf(dp, len - (dp - name), "%d", unit);
2454 		if (error > len - (dp - name)) {
2455 			/* ifName too long. */
2456 			ifc_free_unit(ifc, unit);
2457 			return (ENOSPC);
2458 		}
2459 		dp += error;
2460 	}
2461 
2462 	/* Make sure it doesn't already exist. */
2463 	if (ifunit(name) != NULL)
2464 		return (EEXIST);
2465 
2466 	sc = malloc(sizeof(struct ovpn_softc), M_OVPN, M_WAITOK | M_ZERO);
2467 	sc->ifp = if_alloc(IFT_ENC);
2468 	rm_init_flags(&sc->lock, "if_ovpn_lock", RM_RECURSE);
2469 	sc->refcount = 0;
2470 
2471 	sc->notifring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL);
2472 
2473 	COUNTER_ARRAY_ALLOC(sc->counters, OVPN_COUNTER_SIZE, M_WAITOK);
2474 
2475 	ifp = sc->ifp;
2476 	ifp->if_softc = sc;
2477 	strlcpy(ifp->if_xname, name, IFNAMSIZ);
2478 	ifp->if_dname = ovpngroupname;
2479 	ifp->if_dunit = unit;
2480 
2481 	ifp->if_addrlen = 0;
2482 	ifp->if_mtu = 1428;
2483 	ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST;
2484 	ifp->if_ioctl = ovpn_ioctl;
2485 	ifp->if_transmit = ovpn_transmit;
2486 	ifp->if_output = ovpn_output;
2487 	ifp->if_qflush = ovpn_qflush;
2488 #ifdef VIMAGE
2489 	ifp->if_reassign = ovpn_reassign;
2490 #endif
2491 	ifp->if_capabilities |= IFCAP_LINKSTATE;
2492 	ifp->if_capenable |= IFCAP_LINKSTATE;
2493 
2494 	if_attach(ifp);
2495 	bpfattach(ifp, DLT_NULL, sizeof(uint32_t));
2496 	*ifpp = ifp;
2497 
2498 	return (0);
2499 }
2500 
2501 static void
2502 ovpn_clone_destroy_cb(struct epoch_context *ctx)
2503 {
2504 	struct ovpn_softc *sc;
2505 	int ret __diagused;
2506 
2507 	sc = __containerof(ctx, struct ovpn_softc, epoch_ctx);
2508 
2509 	MPASS(sc->peercount == 0);
2510 	MPASS(RB_EMPTY(&sc->peers));
2511 
2512 	if (sc->so != NULL) {
2513 		CURVNET_SET(sc->ifp->if_vnet);
2514 		ret = udp_set_kernel_tunneling(sc->so, NULL, NULL, NULL);
2515 		MPASS(ret == 0);
2516 		sorele(sc->so);
2517 		CURVNET_RESTORE();
2518 	}
2519 
2520 	COUNTER_ARRAY_FREE(sc->counters, OVPN_COUNTER_SIZE);
2521 
2522 	if_free(sc->ifp);
2523 	free(sc, M_OVPN);
2524 }
2525 
2526 static int
2527 ovpn_clone_destroy(struct if_clone *ifc, struct ifnet *ifp, uint32_t flags)
2528 {
2529 	struct ovpn_softc *sc;
2530 	struct ovpn_kpeer *peer, *tmppeer;
2531 	int unit;
2532 	int ret __diagused;
2533 
2534 	sc = ifp->if_softc;
2535 	unit = ifp->if_dunit;
2536 
2537 	OVPN_WLOCK(sc);
2538 
2539 	if (atomic_load_int(&sc->refcount) > 0) {
2540 		OVPN_WUNLOCK(sc);
2541 		return (EBUSY);
2542 	}
2543 
2544 	RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) {
2545 		peer->del_reason = OVPN_DEL_REASON_REQUESTED;
2546 		ret = _ovpn_del_peer(sc, peer);
2547 		MPASS(ret == 0);
2548 	}
2549 
2550 	ovpn_flush_rxring(sc);
2551 	buf_ring_free(sc->notifring, M_OVPN);
2552 
2553 	OVPN_WUNLOCK(sc);
2554 
2555 	bpfdetach(ifp);
2556 	if_detach(ifp);
2557 	ifp->if_softc = NULL;
2558 
2559 	NET_EPOCH_CALL(ovpn_clone_destroy_cb, &sc->epoch_ctx);
2560 
2561 	if (unit != IF_DUNIT_NONE)
2562 		ifc_free_unit(ifc, unit);
2563 
2564 	NET_EPOCH_DRAIN_CALLBACKS();
2565 
2566 	return (0);
2567 }
2568 
2569 static void
2570 vnet_ovpn_init(const void *unused __unused)
2571 {
2572 	struct if_clone_addreq req = {
2573 		.match_f = ovpn_clone_match,
2574 		.create_f = ovpn_clone_create,
2575 		.destroy_f = ovpn_clone_destroy,
2576 	};
2577 	V_ovpn_cloner = ifc_attach_cloner(ovpngroupname, &req);
2578 }
2579 VNET_SYSINIT(vnet_ovpn_init, SI_SUB_PSEUDO, SI_ORDER_ANY,
2580     vnet_ovpn_init, NULL);
2581 
2582 static void
2583 vnet_ovpn_uninit(const void *unused __unused)
2584 {
2585 	if_clone_detach(V_ovpn_cloner);
2586 }
2587 VNET_SYSUNINIT(vnet_ovpn_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY,
2588     vnet_ovpn_uninit, NULL);
2589 
2590 static int
2591 ovpnmodevent(module_t mod, int type, void *data)
2592 {
2593 	switch (type) {
2594 	case MOD_LOAD:
2595 		/* Done in vnet_ovpn_init() */
2596 		break;
2597 	case MOD_UNLOAD:
2598 		/* Done in vnet_ovpn_uninit() */
2599 		break;
2600 	default:
2601 		return (EOPNOTSUPP);
2602 	}
2603 
2604 	return (0);
2605 }
2606 
2607 static moduledata_t ovpn_mod = {
2608 	"if_ovpn",
2609 	ovpnmodevent,
2610 	0
2611 };
2612 
2613 DECLARE_MODULE(if_ovpn, ovpn_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
2614 MODULE_VERSION(if_ovpn, 1);
2615 MODULE_DEPEND(if_ovpn, crypto, 1, 1, 1);
2616