xref: /freebsd/sys/net/if_ovpn.c (revision f374ba41f55c1a127303d92d830dd58eef2f5243)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2021-2022 Rubicon Communications, LLC (Netgate)
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  *
27  */
28 #include "opt_inet.h"
29 #include "opt_inet6.h"
30 
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/buf_ring.h>
34 #include <sys/epoch.h>
35 #include <sys/file.h>
36 #include <sys/filedesc.h>
37 #include <sys/kernel.h>
38 #include <sys/malloc.h>
39 #include <sys/mbuf.h>
40 #include <sys/module.h>
41 #include <sys/nv.h>
42 #include <sys/priv.h>
43 #include <sys/protosw.h>
44 #include <sys/rmlock.h>
45 #include <sys/sdt.h>
46 #include <sys/smp.h>
47 #include <sys/socket.h>
48 #include <sys/socketvar.h>
49 #include <sys/sockio.h>
50 #include <sys/sysctl.h>
51 #include <sys/time.h>
52 
53 #include <machine/atomic.h>
54 
55 #include <net/bpf.h>
56 #include <net/if.h>
57 #include <net/if_clone.h>
58 #include <net/if_types.h>
59 #include <net/if_var.h>
60 #include <net/if_private.h>
61 #include <net/netisr.h>
62 #include <net/route/nhop.h>
63 
64 #include <netinet/in.h>
65 #include <netinet/in_fib.h>
66 #include <netinet/ip.h>
67 #include <netinet/ip6.h>
68 #include <netinet/ip_var.h>
69 #include <netinet/udp.h>
70 #include <netinet/udp_var.h>
71 
72 #include <netinet6/ip6_var.h>
73 #include <netinet6/in6_fib.h>
74 
75 #include <machine/in_cksum.h>
76 
77 #include <opencrypto/cryptodev.h>
78 
79 #include "if_ovpn.h"
80 
81 struct ovpn_kkey_dir {
82 	int			refcount;
83 	uint8_t			key[32];
84 	uint8_t			keylen;
85 	uint8_t			nonce[8];
86 	uint8_t			noncelen;
87 	enum ovpn_key_cipher	cipher;
88 	crypto_session_t	cryptoid;
89 
90 	struct mtx		replay_mtx;
91 	/*
92 	 * Last seen gapless sequence number. New rx seq numbers must be
93 	 * strictly higher than this.
94 	 */
95 	uint32_t		rx_seq;
96 	/* Seen packets, relative to rx_seq. bit(0) will always be 0. */
97 	uint64_t		rx_window;
98 };
99 
100 struct ovpn_kkey {
101 	struct ovpn_kkey_dir	*encrypt;
102 	struct ovpn_kkey_dir	*decrypt;
103 	uint8_t			 keyid;
104 	uint32_t		 peerid;
105 };
106 
107 struct ovpn_keepalive {
108 	uint32_t	interval;
109 	uint32_t	timeout;
110 };
111 
112 struct ovpn_wire_header {
113 	uint32_t	 opcode; /* opcode, key id, peer id */
114 	uint32_t	 seq;
115 	uint8_t		 auth_tag[16];
116 };
117 
118 struct ovpn_peer_counters {
119 	uint64_t	pkt_in;
120 	uint64_t	pkt_out;
121 	uint64_t	bytes_in;
122 	uint64_t	bytes_out;
123 };
124 #define OVPN_PEER_COUNTER_SIZE (sizeof(struct ovpn_peer_counters)/sizeof(uint64_t))
125 
126 struct ovpn_notification {
127 	enum ovpn_notif_type	type;
128 	uint32_t		peerid;
129 
130 	/* Delete notification */
131 	enum ovpn_del_reason	del_reason;
132 	struct ovpn_peer_counters	counters;
133 };
134 
135 struct ovpn_softc;
136 
137 struct ovpn_kpeer {
138 	RB_ENTRY(ovpn_kpeer)	 tree;
139 	int			 refcount;
140 	uint32_t		 peerid;
141 
142 	struct ovpn_softc	*sc;
143 	struct sockaddr_storage	 local;
144 	struct sockaddr_storage	 remote;
145 
146 	struct in_addr		 vpn4;
147 	struct in6_addr		 vpn6;
148 
149 	struct ovpn_kkey	 keys[2];
150 	uint32_t		 tx_seq;
151 
152 	enum ovpn_del_reason	 del_reason;
153 	struct ovpn_keepalive	 keepalive;
154 	uint32_t		*last_active;
155 	struct callout		 ping_send;
156 	struct callout		 ping_rcv;
157 
158 	counter_u64_t		 counters[OVPN_PEER_COUNTER_SIZE];
159 };
160 
161 struct ovpn_counters {
162 	uint64_t	lost_ctrl_pkts_in;
163 	uint64_t	lost_ctrl_pkts_out;
164 	uint64_t	lost_data_pkts_in;
165 	uint64_t	lost_data_pkts_out;
166 	uint64_t	nomem_data_pkts_in;
167 	uint64_t	nomem_data_pkts_out;
168 	uint64_t	received_ctrl_pkts;
169 	uint64_t	received_data_pkts;
170 	uint64_t	sent_ctrl_pkts;
171 	uint64_t	sent_data_pkts;
172 
173 	uint64_t	transport_bytes_sent;
174 	uint64_t	transport_bytes_received;
175 	uint64_t	tunnel_bytes_sent;
176 	uint64_t	tunnel_bytes_received;
177 };
178 #define OVPN_COUNTER_SIZE (sizeof(struct ovpn_counters)/sizeof(uint64_t))
179 
180 RB_HEAD(ovpn_kpeers, ovpn_kpeer);
181 
182 struct ovpn_softc {
183 	int			 refcount;
184 	struct rmlock		 lock;
185 	struct ifnet		*ifp;
186 	struct socket		*so;
187 	int			 peercount;
188 	struct ovpn_kpeers	 peers;
189 
190 	/* Pending notification */
191 	struct buf_ring		*notifring;
192 
193 	counter_u64_t 		 counters[OVPN_COUNTER_SIZE];
194 
195 	struct epoch_context	 epoch_ctx;
196 };
197 
198 static struct ovpn_kpeer *ovpn_find_peer(struct ovpn_softc *, uint32_t);
199 static bool ovpn_udp_input(struct mbuf *, int, struct inpcb *,
200     const struct sockaddr *, void *);
201 static int ovpn_transmit_to_peer(struct ifnet *, struct mbuf *,
202     struct ovpn_kpeer *, struct rm_priotracker *);
203 static int ovpn_encap(struct ovpn_softc *, uint32_t, struct mbuf *);
204 static int ovpn_get_af(struct mbuf *);
205 static void ovpn_free_kkey_dir(struct ovpn_kkey_dir *);
206 static bool ovpn_check_replay(struct ovpn_kkey_dir *, uint32_t);
207 static int ovpn_peer_compare(struct ovpn_kpeer *, struct ovpn_kpeer *);
208 
209 static RB_PROTOTYPE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare);
210 static RB_GENERATE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare);
211 
212 #define OVPN_MTU_MIN		576
213 #define OVPN_MTU_MAX		(IP_MAXPACKET - sizeof(struct ip) - \
214     sizeof(struct udphdr) - sizeof(struct ovpn_wire_header))
215 
216 #define OVPN_OP_DATA_V2		0x09
217 #define OVPN_OP_SHIFT		3
218 
219 VNET_DEFINE_STATIC(struct if_clone *, ovpn_cloner);
220 #define	V_ovpn_cloner	VNET(ovpn_cloner)
221 
222 #define OVPN_RLOCK_TRACKER	struct rm_priotracker _ovpn_lock_tracker; \
223     struct rm_priotracker *_ovpn_lock_trackerp = &_ovpn_lock_tracker
224 #define OVPN_RLOCK(sc)		rm_rlock(&(sc)->lock, _ovpn_lock_trackerp)
225 #define OVPN_RUNLOCK(sc)	rm_runlock(&(sc)->lock, _ovpn_lock_trackerp)
226 #define OVPN_WLOCK(sc)		rm_wlock(&(sc)->lock)
227 #define OVPN_WUNLOCK(sc)	rm_wunlock(&(sc)->lock)
228 #define OVPN_ASSERT(sc)		rm_assert(&(sc)->lock, RA_LOCKED)
229 #define OVPN_RASSERT(sc)	rm_assert(&(sc)->lock, RA_RLOCKED)
230 #define OVPN_WASSERT(sc)	rm_assert(&(sc)->lock, RA_WLOCKED)
231 #define OVPN_UNLOCK_ASSERT(sc)	rm_assert(&(sc)->lock, RA_UNLOCKED)
232 
233 #define OVPN_COUNTER(sc, name) \
234 	((sc)->counters[offsetof(struct ovpn_counters, name)/sizeof(uint64_t)])
235 #define OVPN_PEER_COUNTER(peer, name) \
236 	((peer)->counters[offsetof(struct ovpn_peer_counters, name) / \
237 	 sizeof(uint64_t)])
238 
239 #define OVPN_COUNTER_ADD(sc, name, val)	\
240 	counter_u64_add(OVPN_COUNTER(sc, name), val)
241 #define OVPN_PEER_COUNTER_ADD(p, name, val)	\
242 	counter_u64_add(OVPN_PEER_COUNTER(p, name), val)
243 
244 #define TO_IN(x)		((struct sockaddr_in *)(x))
245 #define TO_IN6(x)		((struct sockaddr_in6 *)(x))
246 
247 SDT_PROVIDER_DEFINE(if_ovpn);
248 SDT_PROBE_DEFINE1(if_ovpn, tx, transmit, start, "struct mbuf *");
249 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip4, "struct in_addr *", "struct ovpn_kpeer *");
250 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip6, "struct in6_addr *", "struct ovpn_kpeer *");
251 
252 static const char ovpnname[] = "ovpn";
253 static const char ovpngroupname[] = "openvpn";
254 
255 static MALLOC_DEFINE(M_OVPN, ovpnname, "OpenVPN DCO Interface");
256 
257 SYSCTL_DECL(_net_link);
258 static SYSCTL_NODE(_net_link, IFT_OTHER, openvpn, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
259     "OpenVPN DCO Interface");
260 VNET_DEFINE_STATIC(int, replay_protection) = 0;
261 #define	V_replay_protection	VNET(replay_protection)
262 SYSCTL_INT(_net_link_openvpn, OID_AUTO, replay_protection, CTLFLAG_VNET | CTLFLAG_RW,
263     &VNET_NAME(replay_protection), 0, "Validate sequence numbers");
264 
265 VNET_DEFINE_STATIC(int, async_crypto);
266 #define	V_async_crypto		VNET(async_crypto)
267 SYSCTL_INT(_net_link_openvpn, OID_AUTO, async_crypto,
268 	CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_crypto), 0,
269 	"Use asynchronous mode to parallelize crypto jobs.");
270 
271 VNET_DEFINE_STATIC(int, async_netisr_queue);
272 #define	V_async_netisr_queue		VNET(async_netisr_queue)
273 SYSCTL_INT(_net_link_openvpn, OID_AUTO, netisr_queue,
274 	CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_netisr_queue), 0,
275 	"Use netisr_queue() rather than netisr_dispatch().");
276 
277 static int
278 ovpn_peer_compare(struct ovpn_kpeer *a, struct ovpn_kpeer *b)
279 {
280 	return (a->peerid - b->peerid);
281 }
282 
283 static struct ovpn_kpeer *
284 ovpn_find_peer(struct ovpn_softc *sc, uint32_t peerid)
285 {
286 	struct ovpn_kpeer p;
287 
288 	OVPN_ASSERT(sc);
289 
290 	p.peerid = peerid;
291 
292 	return (RB_FIND(ovpn_kpeers, &sc->peers, &p));
293 }
294 
295 static struct ovpn_kpeer *
296 ovpn_find_only_peer(struct ovpn_softc *sc)
297 {
298 	OVPN_ASSERT(sc);
299 
300 	return (RB_ROOT(&sc->peers));
301 }
302 
303 static uint16_t
304 ovpn_get_port(struct sockaddr_storage *s)
305 {
306 	switch (s->ss_family) {
307 	case AF_INET: {
308 		struct sockaddr_in *in = (struct sockaddr_in *)s;
309 		return (in->sin_port);
310 	}
311 	case AF_INET6: {
312 		struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)s;
313 		return (in6->sin6_port);
314 	}
315 	default:
316 		panic("Unsupported address family %d", s->ss_family);
317 	}
318 }
319 
320 static int
321 ovpn_nvlist_to_sockaddr(const nvlist_t *nvl, struct sockaddr_storage *sa)
322 {
323 	int af;
324 
325 	if (! nvlist_exists_number(nvl, "af"))
326 		return (EINVAL);
327 	if (! nvlist_exists_binary(nvl, "address"))
328 		return (EINVAL);
329 	if (! nvlist_exists_number(nvl, "port"))
330 		return (EINVAL);
331 
332 	af = nvlist_get_number(nvl, "af");
333 
334 	switch (af) {
335 #ifdef INET
336 	case AF_INET: {
337 		struct sockaddr_in *in = (struct sockaddr_in *)sa;
338 		size_t len;
339 		const void *addr = nvlist_get_binary(nvl, "address", &len);
340 		in->sin_family = af;
341 		if (len != sizeof(in->sin_addr))
342 			return (EINVAL);
343 
344 		memcpy(&in->sin_addr, addr, sizeof(in->sin_addr));
345 		in->sin_port = nvlist_get_number(nvl, "port");
346 		break;
347 	}
348 #endif
349 #ifdef INET6
350 	case AF_INET6: {
351 		struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)sa;
352 		size_t len;
353 		const void *addr = nvlist_get_binary(nvl, "address", &len);
354 		in6->sin6_family = af;
355 		if (len != sizeof(in6->sin6_addr))
356 			return (EINVAL);
357 
358 		memcpy(&in6->sin6_addr, addr, sizeof(in6->sin6_addr));
359 		in6->sin6_port = nvlist_get_number(nvl, "port");
360 		break;
361 	}
362 #endif
363 	default:
364 		return (EINVAL);
365 	}
366 
367 	return (0);
368 }
369 
370 static bool
371 ovpn_has_peers(struct ovpn_softc *sc)
372 {
373 	OVPN_ASSERT(sc);
374 
375 	return (sc->peercount > 0);
376 }
377 
378 static void
379 ovpn_rele_so(struct ovpn_softc *sc, struct ovpn_kpeer *peer)
380 {
381 	bool has_peers;
382 
383 	OVPN_WASSERT(sc);
384 
385 	if (sc->so == NULL)
386 		return;
387 
388 	has_peers = ovpn_has_peers(sc);
389 
390 	/* Only remove the tunnel function if we're releasing the socket for
391 	 * the last peer. */
392 	if (! has_peers)
393 		(void)udp_set_kernel_tunneling(sc->so, NULL, NULL, NULL);
394 
395 	sorele(sc->so);
396 
397 	if (! has_peers)
398 		sc->so = NULL;
399 }
400 
401 static void
402 ovpn_notify_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer)
403 {
404 	struct ovpn_notification *n;
405 
406 	OVPN_WASSERT(sc);
407 
408 	n = malloc(sizeof(*n), M_OVPN, M_NOWAIT);
409 	if (n == NULL)
410 		return;
411 
412 	n->peerid = peer->peerid;
413 	n->type = OVPN_NOTIF_DEL_PEER;
414 	n->del_reason = peer->del_reason;
415 
416 	n->counters.pkt_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_in));
417 	n->counters.pkt_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_out));
418 	n->counters.bytes_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_in));
419 	n->counters.bytes_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_out));
420 
421 	if (buf_ring_enqueue(sc->notifring, n) != 0) {
422 		free(n, M_OVPN);
423 	} else if (sc->so != NULL) {
424 		/* Wake up userspace */
425 		sc->so->so_error = EAGAIN;
426 		sorwakeup(sc->so);
427 		sowwakeup(sc->so);
428 	}
429 }
430 
431 static void
432 ovpn_peer_release_ref(struct ovpn_kpeer *peer, bool locked)
433 {
434 	struct ovpn_softc *sc;
435 
436 	CURVNET_ASSERT_SET();
437 
438 	atomic_add_int(&peer->refcount, -1);
439 
440 	if (atomic_load_int(&peer->refcount) > 0)
441 		return;
442 
443 	sc = peer->sc;
444 
445 	if (! locked) {
446 		OVPN_WLOCK(sc);
447 
448 		/* Might have changed before we acquired the lock. */
449 		if (atomic_load_int(&peer->refcount) > 0) {
450 			OVPN_WUNLOCK(sc);
451 			return;
452 		}
453 	}
454 
455 	OVPN_ASSERT(sc);
456 
457 	/* The peer should have been removed from the list already. */
458 	MPASS(ovpn_find_peer(sc, peer->peerid) == NULL);
459 
460 	ovpn_notify_del_peer(sc, peer);
461 
462 	for (int i = 0; i < 2; i++) {
463 		ovpn_free_kkey_dir(peer->keys[i].encrypt);
464 		ovpn_free_kkey_dir(peer->keys[i].decrypt);
465 	}
466 
467 	ovpn_rele_so(sc, peer);
468 
469 	callout_stop(&peer->ping_send);
470 	callout_stop(&peer->ping_rcv);
471 	uma_zfree_pcpu(pcpu_zone_4, peer->last_active);
472 	free(peer, M_OVPN);
473 
474 	if (! locked)
475 		OVPN_WUNLOCK(sc);
476 }
477 
478 static int
479 ovpn_new_peer(struct ifnet *ifp, const nvlist_t *nvl)
480 {
481 #ifdef INET6
482 	struct epoch_tracker et;
483 #endif
484 	struct sockaddr_storage remote;
485 	struct ovpn_kpeer *peer = NULL;
486 	struct file *fp = NULL;
487 	struct sockaddr *name = NULL;
488 	struct ovpn_softc *sc = ifp->if_softc;
489 	struct thread *td = curthread;
490 	struct socket *so = NULL;
491 	int fd;
492 	uint32_t peerid;
493 	int ret = 0;
494 
495 	if (nvl == NULL)
496 		return (EINVAL);
497 
498 	if (! nvlist_exists_number(nvl, "peerid"))
499 		return (EINVAL);
500 
501 	if (! nvlist_exists_number(nvl, "fd"))
502 		return (EINVAL);
503 
504 	if (! nvlist_exists_nvlist(nvl, "remote"))
505 		return (EINVAL);
506 
507 	peerid = nvlist_get_number(nvl, "peerid");
508 
509 	ret = ovpn_nvlist_to_sockaddr(nvlist_get_nvlist(nvl, "remote"),
510 	    &remote);
511 	if (ret != 0)
512 		return (ret);
513 
514 	fd = nvlist_get_number(nvl, "fd");
515 
516 	/* Look up the userspace process and use the fd to find the socket. */
517 	ret = getsock(td, fd, &cap_connect_rights, &fp);
518 	if (ret != 0)
519 		return (ret);
520 
521 	so = fp->f_data;
522 
523 	peer = malloc(sizeof(*peer), M_OVPN, M_WAITOK | M_ZERO);
524 	peer->peerid = peerid;
525 	peer->sc = sc;
526 	peer->tx_seq = 1;
527 	peer->refcount = 1;
528 	peer->last_active = uma_zalloc_pcpu(pcpu_zone_4, M_WAITOK | M_ZERO);
529 	COUNTER_ARRAY_ALLOC(peer->counters, OVPN_PEER_COUNTER_SIZE, M_WAITOK);
530 
531 	if (nvlist_exists_binary(nvl, "vpn_ipv4")) {
532 		size_t len;
533 		const void *addr = nvlist_get_binary(nvl, "vpn_ipv4", &len);
534 		if (len != sizeof(peer->vpn4)) {
535 			ret = EINVAL;
536 			goto error;
537 		}
538 		memcpy(&peer->vpn4, addr, len);
539 	}
540 
541 	if (nvlist_exists_binary(nvl, "vpn_ipv6")) {
542 		size_t len;
543 		const void *addr = nvlist_get_binary(nvl, "vpn_ipv6", &len);
544 		if (len != sizeof(peer->vpn6)) {
545 			ret = EINVAL;
546 			goto error;
547 		}
548 		memcpy(&peer->vpn6, addr, len);
549 	}
550 
551 	callout_init_rm(&peer->ping_send, &sc->lock, CALLOUT_SHAREDLOCK);
552 	callout_init_rm(&peer->ping_rcv, &sc->lock, 0);
553 
554 	ret = so->so_proto->pr_sockaddr(so, &name);
555 	if (ret)
556 		goto error;
557 
558 	if (ovpn_get_port((struct sockaddr_storage *)name) == 0) {
559 		ret = EINVAL;
560 		goto error;
561 	}
562 	if (name->sa_family != remote.ss_family) {
563 		ret = EINVAL;
564 		goto error;
565 	}
566 
567 	memcpy(&peer->local, name, name->sa_len);
568 	memcpy(&peer->remote, &remote, sizeof(remote));
569 	free(name, M_SONAME);
570 	name = NULL;
571 
572 	if (peer->local.ss_family == AF_INET6 &&
573 	    IN6_IS_ADDR_V4MAPPED(&TO_IN6(&peer->remote)->sin6_addr)) {
574 		/* V4 mapped address, so treat this as v4, not v6. */
575 		in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->local);
576 		in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->remote);
577 	}
578 
579 #ifdef INET6
580 	if (peer->local.ss_family == AF_INET6 &&
581 	    IN6_IS_ADDR_UNSPECIFIED(&TO_IN6(&peer->local)->sin6_addr)) {
582 		NET_EPOCH_ENTER(et);
583 		ret = in6_selectsrc_addr(curthread->td_proc->p_fibnum,
584 		    &TO_IN6(&peer->remote)->sin6_addr,
585 		    0, NULL, &TO_IN6(&peer->local)->sin6_addr, NULL);
586 		NET_EPOCH_EXIT(et);
587 		if (ret != 0) {
588 			goto error;
589 		}
590 	}
591 #endif
592 	OVPN_WLOCK(sc);
593 
594 	/* Disallow peer id re-use. */
595 	if (ovpn_find_peer(sc, peerid) != NULL) {
596 		ret = EEXIST;
597 		goto error_locked;
598 	}
599 
600 	/* Make sure this is really a UDP socket. */
601 	if (so->so_type != SOCK_DGRAM || so->so_proto->pr_type != SOCK_DGRAM) {
602 		ret = EPROTOTYPE;
603 		goto error_locked;
604 	}
605 
606 	/* Must be the same socket as for other peers on this interface. */
607 	if (sc->so != NULL && so != sc->so)
608 		goto error_locked;
609 
610 	if (sc->so == NULL)
611 		sc->so = so;
612 
613 	/* Insert the peer into the list. */
614 	RB_INSERT(ovpn_kpeers, &sc->peers, peer);
615 	sc->peercount++;
616 	soref(sc->so);
617 
618 	ret = udp_set_kernel_tunneling(sc->so, ovpn_udp_input, NULL, sc);
619 	if (ret == EBUSY) {
620 		/* Fine, another peer already set the input function. */
621 		ret = 0;
622 	}
623 	if (ret != 0) {
624 		RB_REMOVE(ovpn_kpeers, &sc->peers, peer);
625 		sc->peercount--;
626 		goto error_locked;
627 	}
628 
629 	OVPN_WUNLOCK(sc);
630 
631 	goto done;
632 
633 error_locked:
634 	OVPN_WUNLOCK(sc);
635 error:
636 	free(name, M_SONAME);
637 	COUNTER_ARRAY_FREE(peer->counters, OVPN_PEER_COUNTER_SIZE);
638 	uma_zfree_pcpu(pcpu_zone_4, peer->last_active);
639 	free(peer, M_OVPN);
640 done:
641 	if (fp != NULL)
642 		fdrop(fp, td);
643 
644 	return (ret);
645 }
646 
647 static int
648 _ovpn_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer)
649 {
650 	struct ovpn_kpeer *tmp __diagused;
651 
652 	OVPN_WASSERT(sc);
653 	CURVNET_ASSERT_SET();
654 
655 	MPASS(RB_FIND(ovpn_kpeers, &sc->peers, peer) == peer);
656 
657 	tmp = RB_REMOVE(ovpn_kpeers, &sc->peers, peer);
658 	MPASS(tmp != NULL);
659 
660 	sc->peercount--;
661 
662 	ovpn_peer_release_ref(peer, true);
663 
664 	return (0);
665 }
666 
667 static int
668 ovpn_del_peer(struct ifnet *ifp, nvlist_t *nvl)
669 {
670 	struct ovpn_softc *sc = ifp->if_softc;
671 	struct ovpn_kpeer *peer;
672 	uint32_t peerid;
673 	int ret;
674 
675 	OVPN_WASSERT(sc);
676 
677 	if (nvl == NULL)
678 		return (EINVAL);
679 
680 	if (! nvlist_exists_number(nvl, "peerid"))
681 		return (EINVAL);
682 
683 	peerid = nvlist_get_number(nvl, "peerid");
684 
685 	peer = ovpn_find_peer(sc, peerid);
686 	if (peer == NULL)
687 		return (ENOENT);
688 
689 	peer->del_reason = OVPN_DEL_REASON_REQUESTED;
690 	ret = _ovpn_del_peer(sc, peer);
691 
692 	return (ret);
693 }
694 
695 static int
696 ovpn_create_kkey_dir(struct ovpn_kkey_dir **kdirp,
697     const nvlist_t *nvl)
698 {
699 	struct crypto_session_params csp;
700 	struct ovpn_kkey_dir *kdir;
701 	const char *ciphername;
702 	enum ovpn_key_cipher cipher;
703 	const void *key, *iv;
704 	size_t keylen = 0, ivlen = 0;
705 	int error;
706 
707 	if (! nvlist_exists_string(nvl, "cipher"))
708 		return (EINVAL);
709 	ciphername = nvlist_get_string(nvl, "cipher");
710 
711 	if (strcmp(ciphername, "none") == 0)
712 		cipher = OVPN_CIPHER_ALG_NONE;
713 	else if (strcmp(ciphername, "AES-256-GCM") == 0 ||
714 	    strcmp(ciphername, "AES-192-GCM") == 0 ||
715 	    strcmp(ciphername, "AES-128-GCM") == 0)
716 		cipher = OVPN_CIPHER_ALG_AES_GCM;
717 	else if (strcmp(ciphername, "CHACHA20-POLY1305") == 0)
718 		cipher = OVPN_CIPHER_ALG_CHACHA20_POLY1305;
719 	else
720 		return (EINVAL);
721 
722 	if (cipher != OVPN_CIPHER_ALG_NONE) {
723 		if (! nvlist_exists_binary(nvl, "key"))
724 			return (EINVAL);
725 		key = nvlist_get_binary(nvl, "key", &keylen);
726 		if (keylen > sizeof(kdir->key))
727 			return (E2BIG);
728 
729 		if (! nvlist_exists_binary(nvl, "iv"))
730 			return (EINVAL);
731 		iv = nvlist_get_binary(nvl, "iv", &ivlen);
732 		if (ivlen != 8)
733 			return (E2BIG);
734 	}
735 
736 	kdir = malloc(sizeof(struct ovpn_kkey_dir), M_OVPN,
737 	    M_WAITOK | M_ZERO);
738 
739 	kdir->cipher = cipher;
740 	kdir->keylen = keylen;
741 	memcpy(kdir->key, key, keylen);
742 	kdir->noncelen = ivlen;
743 	memcpy(kdir->nonce, iv, ivlen);
744 
745 	if (kdir->cipher != OVPN_CIPHER_ALG_NONE) {
746 		/* Crypto init */
747 		bzero(&csp, sizeof(csp));
748 		csp.csp_mode = CSP_MODE_AEAD;
749 
750 		if (kdir->cipher == OVPN_CIPHER_ALG_CHACHA20_POLY1305)
751 			csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305;
752 		else
753 			csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
754 
755 		csp.csp_flags |= CSP_F_SEPARATE_AAD;
756 
757 		csp.csp_cipher_klen = kdir->keylen;
758 		csp.csp_cipher_key = kdir->key;
759 		csp.csp_ivlen = 96 / 8;
760 
761 		error = crypto_newsession(&kdir->cryptoid, &csp,
762 		    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE);
763 		if (error) {
764 			free(kdir, M_OVPN);
765 			return (error);
766 		}
767 	}
768 
769 	mtx_init(&kdir->replay_mtx, "if_ovpn rx replay", NULL, MTX_DEF);
770 	*kdirp = kdir;
771 
772 	return (0);
773 }
774 
775 static void
776 ovpn_free_kkey_dir(struct ovpn_kkey_dir *kdir)
777 {
778 	if (kdir == NULL)
779 		return;
780 
781 	mtx_destroy(&kdir->replay_mtx);
782 
783 	crypto_freesession(kdir->cryptoid);
784 	free(kdir, M_OVPN);
785 }
786 
787 static int
788 ovpn_set_key(struct ifnet *ifp, const nvlist_t *nvl)
789 {
790 	struct ovpn_softc *sc = ifp->if_softc;
791 	struct ovpn_kkey_dir *enc, *dec;
792 	struct ovpn_kpeer *peer;
793 	int slot, keyid, peerid;
794 	int error;
795 
796 	if (nvl == NULL)
797 		return (EINVAL);
798 
799 	if (! nvlist_exists_number(nvl, "slot"))
800 		return (EINVAL);
801 	slot = nvlist_get_number(nvl, "slot");
802 
803 	if (! nvlist_exists_number(nvl, "keyid"))
804 		return (EINVAL);
805 	keyid = nvlist_get_number(nvl, "keyid");
806 
807 	if (! nvlist_exists_number(nvl, "peerid"))
808 		return (EINVAL);
809 	peerid = nvlist_get_number(nvl, "peerid");
810 
811 	if (slot != OVPN_KEY_SLOT_PRIMARY &&
812 	    slot != OVPN_KEY_SLOT_SECONDARY)
813 		return (EINVAL);
814 
815 	if (! nvlist_exists_nvlist(nvl, "encrypt") ||
816 	    ! nvlist_exists_nvlist(nvl, "decrypt"))
817 		return (EINVAL);
818 
819 	error = ovpn_create_kkey_dir(&enc, nvlist_get_nvlist(nvl, "encrypt"));
820 	if (error)
821 		return (error);
822 
823 	error = ovpn_create_kkey_dir(&dec, nvlist_get_nvlist(nvl, "decrypt"));
824 	if (error) {
825 		ovpn_free_kkey_dir(enc);
826 		return (error);
827 	}
828 
829 	OVPN_WLOCK(sc);
830 
831 	peer = ovpn_find_peer(sc, peerid);
832 	if (peer == NULL) {
833 		ovpn_free_kkey_dir(dec);
834 		ovpn_free_kkey_dir(enc);
835 		OVPN_WUNLOCK(sc);
836 		return (ENOENT);
837 	}
838 
839 	ovpn_free_kkey_dir(peer->keys[slot].encrypt);
840 	ovpn_free_kkey_dir(peer->keys[slot].decrypt);
841 
842 	peer->keys[slot].encrypt = enc;
843 	peer->keys[slot].decrypt = dec;
844 
845 	peer->keys[slot].keyid = keyid;
846 	peer->keys[slot].peerid = peerid;
847 
848 	OVPN_WUNLOCK(sc);
849 
850 	return (0);
851 }
852 
853 static int
854 ovpn_check_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, enum ovpn_key_slot slot)
855 {
856 	OVPN_ASSERT(sc);
857 
858 	if (peer->keys[slot].encrypt == NULL)
859 		return (ENOLINK);
860 
861 	if (peer->keys[slot].decrypt == NULL)
862 		return (ENOLINK);
863 
864 	return (0);
865 }
866 
867 static int
868 ovpn_start(struct ifnet *ifp)
869 {
870 	struct ovpn_softc *sc = ifp->if_softc;
871 
872 	OVPN_WLOCK(sc);
873 
874 	ifp->if_flags |= IFF_UP;
875 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
876 	if_link_state_change(ifp, LINK_STATE_UP);
877 
878 	OVPN_WUNLOCK(sc);
879 
880 	return (0);
881 }
882 
883 static int
884 ovpn_swap_keys(struct ifnet *ifp, nvlist_t *nvl)
885 {
886 	struct ovpn_softc *sc = ifp->if_softc;
887 	struct ovpn_kpeer *peer;
888 	struct ovpn_kkey tmpkey;
889 	int error;
890 
891 	if (nvl == NULL)
892 		return (EINVAL);
893 
894 	if (! nvlist_exists_number(nvl, "peerid"))
895 		return (EINVAL);
896 
897 	OVPN_WLOCK(sc);
898 
899 	peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid"));
900 	if (peer == NULL) {
901 		OVPN_WUNLOCK(sc);
902 		return (ENOENT);
903 	}
904 
905 	/* Check that we have a second key to swap to. */
906 	error = ovpn_check_key(sc, peer, OVPN_KEY_SLOT_SECONDARY);
907 	if (error) {
908 		OVPN_WUNLOCK(sc);
909 		return (error);
910 	}
911 
912 	tmpkey = peer->keys[0];
913 	peer->keys[0] = peer->keys[1];
914 	peer->keys[1] = tmpkey;
915 
916 	OVPN_WUNLOCK(sc);
917 
918 	return (0);
919 }
920 
921 static int
922 ovpn_del_key(struct ifnet *ifp, const nvlist_t *nvl)
923 {
924 	enum ovpn_key_slot slot;
925 	struct ovpn_kpeer *peer;
926 	struct ovpn_softc *sc = ifp->if_softc;
927 
928 	if (nvl == NULL)
929 		return (EINVAL);
930 
931 	if (! nvlist_exists_number(nvl, "peerid"))
932 		return (EINVAL);
933 
934 	if (! nvlist_exists_number(nvl, "slot"))
935 		return (EINVAL);
936 	slot = nvlist_get_number(nvl, "slot");
937 
938 	if (slot != OVPN_KEY_SLOT_PRIMARY &&
939 	    slot != OVPN_KEY_SLOT_SECONDARY)
940 		return (EINVAL);
941 
942 	OVPN_WLOCK(sc);
943 
944 	peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid"));
945 	if (peer == NULL) {
946 		OVPN_WUNLOCK(sc);
947 		return (ENOENT);
948 	}
949 
950 	ovpn_free_kkey_dir(peer->keys[slot].encrypt);
951 	ovpn_free_kkey_dir(peer->keys[slot].decrypt);
952 
953 	peer->keys[slot].encrypt = NULL;
954 	peer->keys[slot].decrypt = NULL;
955 
956 	peer->keys[slot].keyid = 0;
957 	peer->keys[slot].peerid = 0;
958 
959 	OVPN_WUNLOCK(sc);
960 
961 	return (0);
962 }
963 
964 static void
965 ovpn_send_ping(void *arg)
966 {
967 	static const uint8_t ping_str[] = {
968 		0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb,
969 		0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48
970 	};
971 
972 	struct epoch_tracker et;
973 	struct ovpn_kpeer *peer = arg;
974 	struct ovpn_softc *sc = peer->sc;
975 	struct mbuf *m;
976 
977 	OVPN_RASSERT(sc);
978 
979 	/* Ensure we repeat! */
980 	callout_reset(&peer->ping_send, peer->keepalive.interval * hz,
981 	    ovpn_send_ping, peer);
982 
983 	m = m_get2(sizeof(ping_str), M_NOWAIT, MT_DATA, M_PKTHDR);
984 	if (m == NULL)
985 		return;
986 
987 	m_copyback(m, 0, sizeof(ping_str), ping_str);
988 	m->m_len = m->m_pkthdr.len = sizeof(ping_str);
989 
990 	CURVNET_SET(sc->ifp->if_vnet);
991 	NET_EPOCH_ENTER(et);
992 	(void)ovpn_transmit_to_peer(sc->ifp, m, peer, NULL);
993 	NET_EPOCH_EXIT(et);
994 	CURVNET_RESTORE();
995 }
996 
997 static void
998 ovpn_timeout(void *arg)
999 {
1000 	struct ovpn_kpeer *peer = arg;
1001 	struct ovpn_softc *sc = peer->sc;
1002 	uint32_t last, _last_active;
1003 	int ret __diagused;
1004 	int cpu;
1005 
1006 	OVPN_WASSERT(sc);
1007 
1008 	last = 0;
1009 	CPU_FOREACH(cpu) {
1010 		_last_active = *zpcpu_get_cpu(peer->last_active, cpu);
1011 		if (_last_active > last)
1012 			last = _last_active;
1013 	}
1014 
1015 	if (last + peer->keepalive.timeout > time_uptime) {
1016 		callout_reset(&peer->ping_rcv,
1017 		    (peer->keepalive.timeout - (time_uptime - last)) * hz,
1018 		    ovpn_timeout, peer);
1019 		return;
1020 	}
1021 
1022 	CURVNET_SET(sc->ifp->if_vnet);
1023 	peer->del_reason = OVPN_DEL_REASON_TIMEOUT;
1024 	ret = _ovpn_del_peer(sc, peer);
1025 	MPASS(ret == 0);
1026 	CURVNET_RESTORE();
1027 }
1028 
1029 static int
1030 ovpn_set_peer(struct ifnet *ifp, const nvlist_t *nvl)
1031 {
1032 	struct ovpn_softc *sc = ifp->if_softc;
1033 	struct ovpn_kpeer *peer;
1034 
1035 	if (nvl == NULL)
1036 		return (EINVAL);
1037 
1038 	if (! nvlist_exists_number(nvl, "interval") ||
1039 	    ! nvlist_exists_number(nvl, "timeout") ||
1040 	    ! nvlist_exists_number(nvl, "peerid"))
1041 		return (EINVAL);
1042 
1043 	OVPN_WLOCK(sc);
1044 
1045 	peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid"));
1046 	if (peer == NULL) {
1047 		OVPN_WUNLOCK(sc);
1048 		return (ENOENT);
1049 	}
1050 
1051 	peer->keepalive.interval = nvlist_get_number(nvl, "interval");
1052 	peer->keepalive.timeout = nvlist_get_number(nvl, "timeout");
1053 
1054 	if (peer->keepalive.interval > 0)
1055 		callout_reset(&peer->ping_send, peer->keepalive.interval * hz,
1056 		    ovpn_send_ping, peer);
1057 	if (peer->keepalive.timeout > 0)
1058 		callout_reset(&peer->ping_rcv, peer->keepalive.timeout * hz,
1059 		    ovpn_timeout, peer);
1060 
1061 	OVPN_WUNLOCK(sc);
1062 
1063 	return (0);
1064 }
1065 
1066 static int
1067 ovpn_set_ifmode(struct ifnet *ifp, const nvlist_t *nvl)
1068 {
1069 	struct ovpn_softc *sc = ifp->if_softc;
1070 	int ifmode;
1071 
1072 	if (nvl == NULL)
1073 		return (EINVAL);
1074 
1075 	if (! nvlist_exists_number(nvl, "ifmode") )
1076 		return (EINVAL);
1077 
1078 	ifmode = nvlist_get_number(nvl, "ifmode");
1079 
1080 	OVPN_WLOCK(sc);
1081 
1082 	/* deny this if UP */
1083 	if (ifp->if_flags & IFF_UP) {
1084 		OVPN_WUNLOCK(sc);
1085 		return (EBUSY);
1086 	}
1087 
1088 	switch (ifmode & ~IFF_MULTICAST) {
1089 	case IFF_POINTOPOINT:
1090 	case IFF_BROADCAST:
1091 		ifp->if_flags &=
1092 		    ~(IFF_BROADCAST|IFF_POINTOPOINT|IFF_MULTICAST);
1093 		ifp->if_flags |= ifmode;
1094 		break;
1095 	default:
1096 		OVPN_WUNLOCK(sc);
1097 		return (EINVAL);
1098 	}
1099 
1100 	OVPN_WUNLOCK(sc);
1101 
1102 	return (0);
1103 }
1104 
1105 static int
1106 ovpn_ioctl_set(struct ifnet *ifp, struct ifdrv *ifd)
1107 {
1108 	struct ovpn_softc *sc = ifp->if_softc;
1109 	uint8_t *buf = NULL;
1110 	nvlist_t *nvl = NULL;
1111 	int ret;
1112 
1113 	if (ifd->ifd_len != 0) {
1114 		if (ifd->ifd_len > OVPN_MAX_REQUEST_SIZE)
1115 			return (E2BIG);
1116 
1117 		buf = malloc(ifd->ifd_len, M_OVPN, M_WAITOK);
1118 
1119 		ret = copyin(ifd->ifd_data, buf, ifd->ifd_len);
1120 		if (ret != 0) {
1121 			free(buf, M_OVPN);
1122 			return (ret);
1123 		}
1124 
1125 		nvl = nvlist_unpack(buf, ifd->ifd_len, 0);
1126 		free(buf, M_OVPN);
1127 		if (nvl == NULL) {
1128 			return (EINVAL);
1129 		}
1130 	}
1131 
1132 	switch (ifd->ifd_cmd) {
1133 	case OVPN_NEW_PEER:
1134 		ret = ovpn_new_peer(ifp, nvl);
1135 		break;
1136 	case OVPN_DEL_PEER:
1137 		OVPN_WLOCK(sc);
1138 		ret = ovpn_del_peer(ifp, nvl);
1139 		OVPN_WUNLOCK(sc);
1140 		break;
1141 	case OVPN_NEW_KEY:
1142 		ret = ovpn_set_key(ifp, nvl);
1143 		break;
1144 	case OVPN_START_VPN:
1145 		ret = ovpn_start(ifp);
1146 		break;
1147 	case OVPN_SWAP_KEYS:
1148 		ret = ovpn_swap_keys(ifp, nvl);
1149 		break;
1150 	case OVPN_DEL_KEY:
1151 		ret = ovpn_del_key(ifp, nvl);
1152 		break;
1153 	case OVPN_SET_PEER:
1154 		ret = ovpn_set_peer(ifp, nvl);
1155 		break;
1156 	case OVPN_SET_IFMODE:
1157 		ret = ovpn_set_ifmode(ifp, nvl);
1158 		break;
1159 	default:
1160 		ret = ENOTSUP;
1161 	}
1162 
1163 	nvlist_destroy(nvl);
1164 	return (ret);
1165 }
1166 
1167 static int
1168 ovpn_add_counters(nvlist_t *parent, const char *name, counter_u64_t in,
1169     counter_u64_t out)
1170 {
1171 	nvlist_t *nvl;
1172 
1173 	nvl = nvlist_create(0);
1174 	if (nvl == NULL)
1175 		return (ENOMEM);
1176 
1177 	nvlist_add_number(nvl, "in", counter_u64_fetch(in));
1178 	nvlist_add_number(nvl, "out", counter_u64_fetch(out));
1179 
1180 	nvlist_add_nvlist(parent, name, nvl);
1181 
1182 	nvlist_destroy(nvl);
1183 
1184 	return (0);
1185 }
1186 
1187 static int
1188 ovpn_get_stats(struct ovpn_softc *sc, nvlist_t **onvl)
1189 {
1190 	nvlist_t *nvl;
1191 	int ret;
1192 
1193 	nvl = nvlist_create(0);
1194 	if (nvl == NULL)
1195 		return (ENOMEM);
1196 
1197 #define OVPN_COUNTER_OUT(name, in, out) \
1198 	do { \
1199 		ret = ovpn_add_counters(nvl, name, OVPN_COUNTER(sc, in), \
1200 		    OVPN_COUNTER(sc, out)); \
1201 		if (ret != 0) \
1202 			goto error; \
1203 	} while(0)
1204 
1205 	OVPN_COUNTER_OUT("lost_ctrl", lost_ctrl_pkts_in, lost_ctrl_pkts_out);
1206 	OVPN_COUNTER_OUT("lost_data", lost_data_pkts_in, lost_data_pkts_out);
1207 	OVPN_COUNTER_OUT("nomem_data", nomem_data_pkts_in,
1208 	    nomem_data_pkts_out);
1209 	OVPN_COUNTER_OUT("data", received_data_pkts, sent_data_pkts);
1210 	OVPN_COUNTER_OUT("ctrl", received_ctrl_pkts, sent_ctrl_pkts);
1211 	OVPN_COUNTER_OUT("tunnel", tunnel_bytes_received,
1212 	    tunnel_bytes_received);
1213 	OVPN_COUNTER_OUT("transport", transport_bytes_received,
1214 	    transport_bytes_received);
1215 #undef OVPN_COUNTER_OUT
1216 
1217 	*onvl = nvl;
1218 
1219 	return (0);
1220 
1221 error:
1222 	nvlist_destroy(nvl);
1223 	return (ret);
1224 }
1225 
1226 static int
1227 ovpn_get_peer_stats(struct ovpn_softc *sc, nvlist_t **nvl)
1228 {
1229 	struct ovpn_kpeer *peer;
1230 	nvlist_t *nvpeer = NULL;
1231 	int ret;
1232 
1233 	OVPN_RLOCK_TRACKER;
1234 
1235 	*nvl = nvlist_create(0);
1236 	if (*nvl == NULL)
1237 		return (ENOMEM);
1238 
1239 #define OVPN_PEER_COUNTER_OUT(name, in, out) \
1240 	do { \
1241 		ret = ovpn_add_counters(nvpeer, name, \
1242 		    OVPN_PEER_COUNTER(peer, in), OVPN_PEER_COUNTER(peer, out)); \
1243 		if (ret != 0) \
1244 			goto error; \
1245 	} while(0)
1246 
1247 	OVPN_RLOCK(sc);
1248 	RB_FOREACH(peer, ovpn_kpeers, &sc->peers) {
1249 		nvpeer = nvlist_create(0);
1250 		if (nvpeer == NULL) {
1251 			OVPN_RUNLOCK(sc);
1252 			nvlist_destroy(*nvl);
1253 			*nvl = NULL;
1254 			return (ENOMEM);
1255 		}
1256 
1257 		nvlist_add_number(nvpeer, "peerid", peer->peerid);
1258 
1259 		OVPN_PEER_COUNTER_OUT("packets", pkt_in, pkt_out);
1260 		OVPN_PEER_COUNTER_OUT("bytes", bytes_in, bytes_out);
1261 
1262 		nvlist_append_nvlist_array(*nvl, "peers", nvpeer);
1263 		nvlist_destroy(nvpeer);
1264 	}
1265 #undef OVPN_PEER_COUNTER_OUT
1266 	OVPN_RUNLOCK(sc);
1267 
1268 	return (0);
1269 
1270 error:
1271 	nvlist_destroy(nvpeer);
1272 	nvlist_destroy(*nvl);
1273 	*nvl = NULL;
1274 	return (ret);
1275 }
1276 
1277 static int
1278 ovpn_poll_pkt(struct ovpn_softc *sc, nvlist_t **onvl)
1279 {
1280 	nvlist_t *nvl;
1281 
1282 	nvl = nvlist_create(0);
1283 	if (nvl == NULL)
1284 		return (ENOMEM);
1285 
1286 	nvlist_add_number(nvl, "pending", buf_ring_count(sc->notifring));
1287 
1288 	*onvl = nvl;
1289 
1290 	return (0);
1291 }
1292 
1293 static void
1294 ovpn_notif_add_counters(nvlist_t *parent, struct ovpn_notification *n)
1295 {
1296 	nvlist_t *nvl;
1297 
1298 	nvl = nvlist_create(0);
1299 	if (nvl == NULL)
1300 		return;
1301 
1302 	nvlist_add_number(nvl, "in", n->counters.pkt_in);
1303 	nvlist_add_number(nvl, "out", n->counters.pkt_out);
1304 
1305 	nvlist_add_nvlist(parent, "packets", nvl);
1306 	nvlist_destroy(nvl);
1307 
1308 	nvl = nvlist_create(0);
1309 	if (nvl == NULL)
1310 		return;
1311 
1312 	nvlist_add_number(nvl, "in", n->counters.bytes_in);
1313 	nvlist_add_number(nvl, "out", n->counters.bytes_out);
1314 
1315 	nvlist_add_nvlist(parent, "bytes", nvl);
1316 	nvlist_destroy(nvl);
1317 }
1318 
1319 static int
1320 opvn_get_pkt(struct ovpn_softc *sc, nvlist_t **onvl)
1321 {
1322 	struct ovpn_notification *n;
1323 	nvlist_t *nvl;
1324 
1325 	/* Check if we have notifications pending. */
1326 	n = buf_ring_dequeue_mc(sc->notifring);
1327 	if (n == NULL)
1328 		return (ENOENT);
1329 
1330 	nvl = nvlist_create(0);
1331 	if (nvl == NULL) {
1332 		free(n, M_OVPN);
1333 		return (ENOMEM);
1334 	}
1335 	nvlist_add_number(nvl, "peerid", n->peerid);
1336 	nvlist_add_number(nvl, "notification", n->type);
1337 	if (n->type == OVPN_NOTIF_DEL_PEER) {
1338 		nvlist_add_number(nvl, "del_reason", n->del_reason);
1339 
1340 		/* No error handling, because we want to send the notification
1341 		 * even if we can't attach the counters. */
1342 		ovpn_notif_add_counters(nvl, n);
1343 	}
1344 	free(n, M_OVPN);
1345 
1346 	*onvl = nvl;
1347 
1348 	return (0);
1349 }
1350 
1351 static int
1352 ovpn_ioctl_get(struct ifnet *ifp, struct ifdrv *ifd)
1353 {
1354 	struct ovpn_softc *sc = ifp->if_softc;
1355 	nvlist_t *nvl = NULL;
1356 	int error;
1357 
1358 	switch (ifd->ifd_cmd) {
1359 	case OVPN_GET_STATS:
1360 		error = ovpn_get_stats(sc, &nvl);
1361 		break;
1362 	case OVPN_GET_PEER_STATS:
1363 		error = ovpn_get_peer_stats(sc, &nvl);
1364 		break;
1365 	case OVPN_POLL_PKT:
1366 		error = ovpn_poll_pkt(sc, &nvl);
1367 		break;
1368 	case OVPN_GET_PKT:
1369 		error = opvn_get_pkt(sc, &nvl);
1370 		break;
1371 	default:
1372 		error = ENOTSUP;
1373 		break;
1374 	}
1375 
1376 	if (error == 0) {
1377 		void *packed = NULL;
1378 		size_t len;
1379 
1380 		MPASS(nvl != NULL);
1381 
1382 		packed = nvlist_pack(nvl, &len);
1383 		if (! packed) {
1384 			nvlist_destroy(nvl);
1385 			return (ENOMEM);
1386 		}
1387 
1388 		if (len > ifd->ifd_len) {
1389 			free(packed, M_NVLIST);
1390 			nvlist_destroy(nvl);
1391 			return (ENOSPC);
1392 		}
1393 
1394 		error = copyout(packed, ifd->ifd_data, len);
1395 		ifd->ifd_len = len;
1396 
1397 		free(packed, M_NVLIST);
1398 		nvlist_destroy(nvl);
1399 	}
1400 
1401 	return (error);
1402 }
1403 
1404 static int
1405 ovpn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
1406 {
1407 	struct ifdrv *ifd;
1408 	int error;
1409 
1410 	CURVNET_ASSERT_SET();
1411 
1412 	switch (cmd) {
1413 	case SIOCSDRVSPEC:
1414 	case SIOCGDRVSPEC:
1415 		error = priv_check(curthread, PRIV_NET_OVPN);
1416 		if (error)
1417 			return (error);
1418 		break;
1419 	}
1420 
1421 	switch (cmd) {
1422 	case SIOCSDRVSPEC:
1423 		ifd = (struct ifdrv *)data;
1424 		error = ovpn_ioctl_set(ifp, ifd);
1425 		break;
1426 	case SIOCGDRVSPEC:
1427 		ifd = (struct ifdrv *)data;
1428 		error = ovpn_ioctl_get(ifp, ifd);
1429 		break;
1430 	case SIOCSIFMTU: {
1431 		struct ifreq *ifr = (struct ifreq *)data;
1432 		if (ifr->ifr_mtu < OVPN_MTU_MIN || ifr->ifr_mtu > OVPN_MTU_MAX)
1433 			return (EINVAL);
1434 
1435 		ifp->if_mtu = ifr->ifr_mtu;
1436 		return (0);
1437 	}
1438 	case SIOCSIFADDR:
1439 	case SIOCADDMULTI:
1440 	case SIOCDELMULTI:
1441 	case SIOCGIFMTU:
1442 	case SIOCSIFFLAGS:
1443 		return (0);
1444 	default:
1445 		error = EINVAL;
1446 	}
1447 
1448 	return (error);
1449 }
1450 
1451 static int
1452 ovpn_encrypt_tx_cb(struct cryptop *crp)
1453 {
1454 	struct epoch_tracker et;
1455 	struct ovpn_kpeer *peer = crp->crp_opaque;
1456 	struct ovpn_softc *sc = peer->sc;
1457 	struct mbuf *m = crp->crp_buf.cb_mbuf;
1458 	int tunnel_len;
1459 	int ret;
1460 
1461 	CURVNET_SET(sc->ifp->if_vnet);
1462 	NET_EPOCH_ENTER(et);
1463 
1464 	if (crp->crp_etype != 0) {
1465 		crypto_freereq(crp);
1466 		ovpn_peer_release_ref(peer, false);
1467 		NET_EPOCH_EXIT(et);
1468 		CURVNET_RESTORE();
1469 		OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
1470 		m_freem(m);
1471 		return (0);
1472 	}
1473 
1474 	MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF);
1475 
1476 	tunnel_len = m->m_pkthdr.len - sizeof(struct ovpn_wire_header);
1477 	ret = ovpn_encap(sc, peer->peerid, m);
1478 	if (ret == 0) {
1479 		OVPN_COUNTER_ADD(sc, sent_data_pkts, 1);
1480 		OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len);
1481 	}
1482 
1483 	crypto_freereq(crp);
1484 	ovpn_peer_release_ref(peer, false);
1485 
1486 	NET_EPOCH_EXIT(et);
1487 	CURVNET_RESTORE();
1488 
1489 	return (0);
1490 }
1491 
1492 static void
1493 ovpn_finish_rx(struct ovpn_softc *sc, struct mbuf *m,
1494     struct ovpn_kpeer *peer, struct ovpn_kkey *key, uint32_t seq,
1495     struct rm_priotracker *_ovpn_lock_trackerp)
1496 {
1497 	uint32_t af;
1498 
1499 	OVPN_RASSERT(sc);
1500 	NET_EPOCH_ASSERT();
1501 
1502 	/* Replay protection. */
1503 	if (V_replay_protection && ! ovpn_check_replay(key->decrypt, seq)) {
1504 		OVPN_RUNLOCK(sc);
1505 		OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1506 		m_freem(m);
1507 		return;
1508 	}
1509 
1510 	critical_enter();
1511 	*zpcpu_get(peer->last_active) = time_uptime;
1512 	critical_exit();
1513 
1514 	OVPN_RUNLOCK(sc);
1515 
1516 	OVPN_COUNTER_ADD(sc, received_data_pkts, 1);
1517 	OVPN_COUNTER_ADD(sc, tunnel_bytes_received, m->m_pkthdr.len);
1518 	OVPN_PEER_COUNTER_ADD(peer, pkt_in, 1);
1519 	OVPN_PEER_COUNTER_ADD(peer, bytes_in, m->m_pkthdr.len);
1520 
1521 	/* Receive the packet on our interface. */
1522 	m->m_pkthdr.rcvif = sc->ifp;
1523 
1524 	/* Clear checksum flags in case the real hardware set them. */
1525 	m->m_pkthdr.csum_flags = 0;
1526 
1527 	/* Ensure we can read the first byte. */
1528 	m = m_pullup(m, 1);
1529 	if (m == NULL) {
1530 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1);
1531 		return;
1532 	}
1533 
1534 	/*
1535 	 * Check for address family, and disregard any control packets (e.g.
1536 	 * keepalive).
1537 	 */
1538 	af = ovpn_get_af(m);
1539 	if (af != 0) {
1540 		BPF_MTAP2(sc->ifp, &af, sizeof(af), m);
1541 		if (V_async_netisr_queue)
1542 			netisr_queue(af == AF_INET ? NETISR_IP : NETISR_IPV6, m);
1543 		else
1544 			netisr_dispatch(af == AF_INET ? NETISR_IP : NETISR_IPV6, m);
1545 	} else {
1546 		OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1547 		m_freem(m);
1548 	}
1549 }
1550 
1551 static struct ovpn_kkey *
1552 ovpn_find_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer,
1553     const struct ovpn_wire_header *ohdr)
1554 {
1555 	struct ovpn_kkey *key = NULL;
1556 	uint8_t keyid;
1557 
1558 	OVPN_RASSERT(sc);
1559 
1560 	keyid = (ntohl(ohdr->opcode) >> 24) & 0x07;
1561 
1562 	if (peer->keys[0].keyid == keyid)
1563 		key = &peer->keys[0];
1564 	else if (peer->keys[1].keyid == keyid)
1565 		key = &peer->keys[1];
1566 
1567 	return (key);
1568 }
1569 
1570 static int
1571 ovpn_decrypt_rx_cb(struct cryptop *crp)
1572 {
1573 	struct epoch_tracker et;
1574 	struct ovpn_softc *sc = crp->crp_opaque;
1575 	struct mbuf *m = crp->crp_buf.cb_mbuf;
1576 	struct ovpn_kkey *key;
1577 	struct ovpn_kpeer *peer;
1578 	struct ovpn_wire_header *ohdr;
1579 	uint32_t peerid;
1580 
1581 	OVPN_RLOCK_TRACKER;
1582 
1583 	OVPN_RLOCK(sc);
1584 
1585 	MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF);
1586 
1587 	if (crp->crp_etype != 0) {
1588 		crypto_freereq(crp);
1589 		atomic_add_int(&sc->refcount, -1);
1590 		OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1591 		OVPN_RUNLOCK(sc);
1592 		m_freem(m);
1593 		return (0);
1594 	}
1595 
1596 	CURVNET_SET(sc->ifp->if_vnet);
1597 
1598 	ohdr = mtodo(m, sizeof(struct udphdr));
1599 
1600 	peerid = ntohl(ohdr->opcode) & 0x00ffffff;
1601 	peer = ovpn_find_peer(sc, peerid);
1602 	if (peer == NULL) {
1603 		/* No such peer. Drop packet. */
1604 		crypto_freereq(crp);
1605 		atomic_add_int(&sc->refcount, -1);
1606 		OVPN_RUNLOCK(sc);
1607 		OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1608 		m_freem(m);
1609 		CURVNET_RESTORE();
1610 		return (0);
1611 	}
1612 
1613 	key = ovpn_find_key(sc, peer, ohdr);
1614 	if (key == NULL) {
1615 		crypto_freereq(crp);
1616 		atomic_add_int(&sc->refcount, -1);
1617 		/*
1618 		 * Has this key been removed between us starting the decrypt
1619 		 * and finishing it?
1620 		 */
1621 		OVPN_RUNLOCK(sc);
1622 		OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1623 		m_freem(m);
1624 		CURVNET_RESTORE();
1625 		return (0);
1626 	}
1627 
1628 	/* Now remove the outer headers */
1629 	m_adj_decap(m, sizeof(struct udphdr) +
1630 	    sizeof(struct ovpn_wire_header));
1631 
1632 	NET_EPOCH_ENTER(et);
1633 	ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), _ovpn_lock_trackerp);
1634 	NET_EPOCH_EXIT(et);
1635 	OVPN_UNLOCK_ASSERT(sc);
1636 
1637 	CURVNET_RESTORE();
1638 
1639 	crypto_freereq(crp);
1640 	atomic_add_int(&sc->refcount, -1);
1641 
1642 	return (0);
1643 }
1644 
1645 static int
1646 ovpn_get_af(struct mbuf *m)
1647 {
1648 	struct ip *ip;
1649 	struct ip6_hdr *ip6;
1650 
1651 	/*
1652 	 * We should pullup, but we're only interested in the first byte, so
1653 	 * that'll always be contiguous.
1654 	 */
1655 	ip = mtod(m, struct ip *);
1656 	if (ip->ip_v == IPVERSION)
1657 		return (AF_INET);
1658 
1659 	ip6 = mtod(m, struct ip6_hdr *);
1660 	if ((ip6->ip6_vfc & IPV6_VERSION_MASK) == IPV6_VERSION)
1661 		return (AF_INET6);
1662 
1663 	return (0);
1664 }
1665 
1666 static struct ovpn_kpeer *
1667 ovpn_find_peer_by_ip(struct ovpn_softc *sc, const struct in_addr addr)
1668 {
1669 	struct ovpn_kpeer *peer = NULL;
1670 
1671 	OVPN_ASSERT(sc);
1672 
1673 	/* TODO: Add a second RB so we can look up by IP. */
1674 	RB_FOREACH(peer, ovpn_kpeers, &sc->peers) {
1675 		if (addr.s_addr == peer->vpn4.s_addr)
1676 			return (peer);
1677 	}
1678 
1679 	return (peer);
1680 }
1681 
1682 static struct ovpn_kpeer *
1683 ovpn_find_peer_by_ip6(struct ovpn_softc *sc, const struct in6_addr *addr)
1684 {
1685 	struct ovpn_kpeer *peer = NULL;
1686 
1687 	OVPN_ASSERT(sc);
1688 
1689 	/* TODO: Add a third RB so we can look up by IPv6 address. */
1690 	RB_FOREACH(peer, ovpn_kpeers, &sc->peers) {
1691 		if (memcmp(addr, &peer->vpn6, sizeof(*addr)) == 0)
1692 			return (peer);
1693 	}
1694 
1695 	return (peer);
1696 }
1697 
1698 static struct ovpn_kpeer *
1699 ovpn_route_peer(struct ovpn_softc *sc, struct mbuf **m0,
1700     const struct sockaddr *dst)
1701 {
1702 	struct ovpn_kpeer *peer = NULL;
1703 	int af;
1704 
1705 	NET_EPOCH_ASSERT();
1706 	OVPN_ASSERT(sc);
1707 
1708 	/* Shortcut if we're a client (or are a server and have only one client). */
1709 	if (sc->peercount == 1)
1710 		return (ovpn_find_only_peer(sc));
1711 
1712 	if (dst != NULL)
1713 		af = dst->sa_family;
1714 	else
1715 		af = ovpn_get_af(*m0);
1716 
1717 	switch (af) {
1718 #ifdef INET
1719 	case AF_INET: {
1720 		const struct sockaddr_in *sa = (const struct sockaddr_in *)dst;
1721 		struct nhop_object *nh;
1722 		const struct in_addr *ip_dst;
1723 
1724 		if (sa != NULL) {
1725 			ip_dst = &sa->sin_addr;
1726 		} else {
1727 			struct ip *ip;
1728 
1729 			*m0 = m_pullup(*m0, sizeof(struct ip));
1730 			if (*m0 == NULL)
1731 				return (NULL);
1732 			ip = mtod(*m0, struct ip *);
1733 			ip_dst = &ip->ip_dst;
1734 		}
1735 
1736 		peer = ovpn_find_peer_by_ip(sc, *ip_dst);
1737 		SDT_PROBE2(if_ovpn, tx, route, ip4, ip_dst, peer);
1738 		if (peer == NULL) {
1739 			nh = fib4_lookup(M_GETFIB(*m0), *ip_dst, 0,
1740 			    NHR_NONE, 0);
1741 			if (nh && (nh->nh_flags & NHF_GATEWAY)) {
1742 				peer = ovpn_find_peer_by_ip(sc,
1743 				    nh->gw4_sa.sin_addr);
1744 				SDT_PROBE2(if_ovpn, tx, route, ip4,
1745 				    &nh->gw4_sa.sin_addr, peer);
1746 			}
1747 		}
1748 		break;
1749 	}
1750 #endif
1751 #ifdef INET6
1752 	case AF_INET6: {
1753 		const struct sockaddr_in6 *sa6 =
1754 		    (const struct sockaddr_in6 *)dst;
1755 		struct nhop_object *nh;
1756 		const struct in6_addr *ip6_dst;
1757 
1758 		if (sa6 != NULL) {
1759 			ip6_dst = &sa6->sin6_addr;
1760 		} else {
1761 			struct ip6_hdr *ip6;
1762 
1763 			*m0 = m_pullup(*m0, sizeof(struct ip6_hdr));
1764 			if (*m0 == NULL)
1765 				return (NULL);
1766 			ip6 = mtod(*m0, struct ip6_hdr *);
1767 			ip6_dst = &ip6->ip6_dst;
1768 		}
1769 
1770 		peer = ovpn_find_peer_by_ip6(sc, ip6_dst);
1771 		SDT_PROBE2(if_ovpn, tx, route, ip6, ip6_dst, peer);
1772 		if (peer == NULL) {
1773 			nh = fib6_lookup(M_GETFIB(*m0), ip6_dst, 0,
1774 			    NHR_NONE, 0);
1775 			if (nh && (nh->nh_flags & NHF_GATEWAY)) {
1776 				peer = ovpn_find_peer_by_ip6(sc,
1777 				    &nh->gw6_sa.sin6_addr);
1778 				SDT_PROBE2(if_ovpn, tx, route, ip6,
1779 				    &nh->gw6_sa.sin6_addr, peer);
1780 			}
1781 		}
1782 		break;
1783 	}
1784 #endif
1785 	}
1786 
1787 	return (peer);
1788 }
1789 
1790 static int
1791 ovpn_transmit(struct ifnet *ifp, struct mbuf *m)
1792 {
1793 	return (ifp->if_output(ifp, m, NULL, NULL));
1794 }
1795 
1796 static int
1797 ovpn_transmit_to_peer(struct ifnet *ifp, struct mbuf *m,
1798     struct ovpn_kpeer *peer, struct rm_priotracker *_ovpn_lock_trackerp)
1799 {
1800 	struct ovpn_wire_header *ohdr;
1801 	struct ovpn_kkey *key;
1802 	struct ovpn_softc *sc;
1803 	struct cryptop *crp;
1804 	uint32_t af, seq;
1805 	size_t len, ovpn_hdr_len;
1806 	int tunnel_len;
1807 	int ret;
1808 
1809 	sc = ifp->if_softc;
1810 
1811 	OVPN_RASSERT(sc);
1812 
1813 	tunnel_len = m->m_pkthdr.len;
1814 
1815 	key = &peer->keys[OVPN_KEY_SLOT_PRIMARY];
1816 	if (key->encrypt == NULL) {
1817 		if (_ovpn_lock_trackerp != NULL)
1818 			OVPN_RUNLOCK(sc);
1819 		m_freem(m);
1820 		return (ENOLINK);
1821 	}
1822 
1823 	af = ovpn_get_af(m);
1824 	/* Don't capture control packets. */
1825 	if (af != 0)
1826 		BPF_MTAP2(ifp, &af, sizeof(af), m);
1827 
1828 	len = m->m_pkthdr.len;
1829 	MPASS(len <= ifp->if_mtu);
1830 
1831 	ovpn_hdr_len = sizeof(struct ovpn_wire_header);
1832 	if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE)
1833 		ovpn_hdr_len -= 16; /* No auth tag. */
1834 
1835 	M_PREPEND(m, ovpn_hdr_len, M_NOWAIT);
1836 	if (m == NULL) {
1837 		if (_ovpn_lock_trackerp != NULL)
1838 			OVPN_RUNLOCK(sc);
1839 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1840 		return (ENOBUFS);
1841 	}
1842 	ohdr = mtod(m, struct ovpn_wire_header *);
1843 	ohdr->opcode = (OVPN_OP_DATA_V2 << OVPN_OP_SHIFT) | key->keyid;
1844 	ohdr->opcode <<= 24;
1845 	ohdr->opcode |= key->peerid;
1846 	ohdr->opcode = htonl(ohdr->opcode);
1847 
1848 	seq = atomic_fetchadd_32(&peer->tx_seq, 1);
1849 	seq = htonl(seq);
1850 	ohdr->seq = seq;
1851 
1852 	OVPN_PEER_COUNTER_ADD(peer, pkt_out, 1);
1853 	OVPN_PEER_COUNTER_ADD(peer, bytes_out, len);
1854 
1855 	if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) {
1856 		ret = ovpn_encap(sc, peer->peerid, m);
1857 		if (_ovpn_lock_trackerp != NULL)
1858 			OVPN_RUNLOCK(sc);
1859 		if (ret == 0) {
1860 			OVPN_COUNTER_ADD(sc, sent_data_pkts, 1);
1861 			OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len);
1862 		}
1863 		return (ret);
1864 	}
1865 
1866 	crp = crypto_getreq(key->encrypt->cryptoid, M_NOWAIT);
1867 	if (crp == NULL) {
1868 		if (_ovpn_lock_trackerp != NULL)
1869 			OVPN_RUNLOCK(sc);
1870 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1871 		m_freem(m);
1872 		return (ENOBUFS);
1873 	}
1874 
1875 	/* Encryption covers only the payload, not the header. */
1876 	crp->crp_payload_start = sizeof(*ohdr);
1877 	crp->crp_payload_length = len;
1878 	crp->crp_op = CRYPTO_OP_ENCRYPT;
1879 
1880 	/*
1881 	 * AAD data covers the ovpn_wire_header minus the auth
1882 	 * tag.
1883 	 */
1884 	crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag);
1885 	crp->crp_aad = ohdr;
1886 	crp->crp_aad_start = 0;
1887 	crp->crp_op |= CRYPTO_OP_COMPUTE_DIGEST;
1888 	crp->crp_digest_start = offsetof(struct ovpn_wire_header, auth_tag);
1889 
1890 	crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
1891 	memcpy(crp->crp_iv, &seq, sizeof(seq));
1892 	memcpy(crp->crp_iv + sizeof(seq), key->encrypt->nonce,
1893 	    key->encrypt->noncelen);
1894 
1895 	crypto_use_mbuf(crp, m);
1896 	crp->crp_flags |= CRYPTO_F_CBIFSYNC;
1897 	crp->crp_callback = ovpn_encrypt_tx_cb;
1898 	crp->crp_opaque = peer;
1899 
1900 	atomic_add_int(&peer->refcount, 1);
1901 	if (_ovpn_lock_trackerp != NULL)
1902 		OVPN_RUNLOCK(sc);
1903 	if (V_async_crypto)
1904 		ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED);
1905 	else
1906 		ret = crypto_dispatch(crp);
1907 	if (ret) {
1908 		OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
1909 	}
1910 
1911 	return (ret);
1912 }
1913 
1914 /*
1915  * Note: Expects to hold the read lock on entry, and will release it itself.
1916  */
1917 static int
1918 ovpn_encap(struct ovpn_softc *sc, uint32_t peerid, struct mbuf *m)
1919 {
1920 	struct udphdr *udp;
1921 	struct ovpn_kpeer *peer;
1922 	int len;
1923 
1924 	OVPN_RLOCK_TRACKER;
1925 
1926 	OVPN_RLOCK(sc);
1927 	NET_EPOCH_ASSERT();
1928 
1929 	peer = ovpn_find_peer(sc, peerid);
1930 	if (peer == NULL || sc->ifp->if_link_state != LINK_STATE_UP) {
1931 		OVPN_RUNLOCK(sc);
1932 		OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
1933 		m_freem(m);
1934 		return (ENETDOWN);
1935 	}
1936 
1937 	len = m->m_pkthdr.len;
1938 
1939 	M_PREPEND(m, sizeof(struct udphdr), M_NOWAIT);
1940 	if (m == NULL) {
1941 		OVPN_RUNLOCK(sc);
1942 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1943 		m_freem(m);
1944 		return (ENOBUFS);
1945 	}
1946 	udp = mtod(m, struct udphdr *);
1947 
1948 	MPASS(peer->local.ss_family == peer->remote.ss_family);
1949 
1950 	udp->uh_sport = ovpn_get_port(&peer->local);
1951 	udp->uh_dport = ovpn_get_port(&peer->remote);
1952 	udp->uh_ulen = htons(sizeof(struct udphdr) + len);
1953 
1954 	switch (peer->remote.ss_family) {
1955 #ifdef INET
1956 	case AF_INET: {
1957 		struct sockaddr_in *in_local = TO_IN(&peer->local);
1958 		struct sockaddr_in *in_remote = TO_IN(&peer->remote);
1959 		struct ip *ip;
1960 
1961 		/*
1962 		 * This requires knowing the source IP, which we don't. Happily
1963 		 * we're allowed to keep this at 0, and the checksum won't do
1964 		 * anything the crypto won't already do.
1965 		 */
1966 		udp->uh_sum = 0;
1967 
1968 		/* Set the checksum flags so we recalculate checksums. */
1969 		m->m_pkthdr.csum_flags |= CSUM_IP;
1970 		m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
1971 
1972 		M_PREPEND(m, sizeof(struct ip), M_NOWAIT);
1973 		if (m == NULL) {
1974 			OVPN_RUNLOCK(sc);
1975 			OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1976 			return (ENOBUFS);
1977 		}
1978 		ip = mtod(m, struct ip *);
1979 
1980 		ip->ip_tos = 0;
1981 		ip->ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) +
1982 		   len);
1983 		ip->ip_off = 0;
1984 		ip->ip_ttl = V_ip_defttl;
1985 		ip->ip_p = IPPROTO_UDP;
1986 		ip->ip_sum = 0;
1987 		if (in_local->sin_port != 0)
1988 			ip->ip_src = in_local->sin_addr;
1989 		else
1990 			ip->ip_src.s_addr = INADDR_ANY;
1991 		ip->ip_dst = in_remote->sin_addr;
1992 
1993 		OVPN_RUNLOCK(sc);
1994 		OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len);
1995 
1996 		return (ip_output(m, NULL, NULL, 0, NULL, NULL));
1997 	}
1998 #endif
1999 #ifdef INET6
2000 	case AF_INET6: {
2001 		struct sockaddr_in6 *in6_local = TO_IN6(&peer->local);
2002 		struct sockaddr_in6 *in6_remote = TO_IN6(&peer->remote);
2003 		struct ip6_hdr *ip6;
2004 
2005 		M_PREPEND(m, sizeof(struct ip6_hdr), M_NOWAIT);
2006 		if (m == NULL) {
2007 			OVPN_RUNLOCK(sc);
2008 			OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
2009 			return (ENOBUFS);
2010 		}
2011 		m = m_pullup(m, sizeof(*ip6) + sizeof(*udp));
2012 		if (m == NULL) {
2013 			OVPN_RUNLOCK(sc);
2014 			OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
2015 			return (ENOBUFS);
2016 		}
2017 
2018 		ip6 = mtod(m, struct ip6_hdr *);
2019 
2020 		ip6->ip6_vfc = IPV6_VERSION;
2021 		ip6->ip6_flow &= ~IPV6_FLOWINFO_MASK;
2022 		ip6->ip6_plen = htons(sizeof(*ip6) + sizeof(struct udphdr) +
2023 		    len);
2024 		ip6->ip6_nxt = IPPROTO_UDP;
2025 		ip6->ip6_hlim = V_ip6_defhlim;
2026 
2027 		memcpy(&ip6->ip6_src, &in6_local->sin6_addr,
2028 		    sizeof(ip6->ip6_src));
2029 		memcpy(&ip6->ip6_dst, &in6_remote->sin6_addr,
2030 		    sizeof(ip6->ip6_dst));
2031 
2032 		udp = mtodo(m, sizeof(*ip6));
2033 		udp->uh_sum = in6_cksum_pseudo(ip6,
2034 		    m->m_pkthdr.len - sizeof(struct ip6_hdr),
2035 		    IPPROTO_UDP, 0);
2036 
2037 		m->m_pkthdr.csum_flags |= CSUM_UDP_IPV6;
2038 		m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
2039 
2040 		OVPN_RUNLOCK(sc);
2041 		OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len);
2042 
2043 		return (ip6_output(m, NULL, NULL, IPV6_UNSPECSRC, NULL, NULL,
2044 		    NULL));
2045 	}
2046 #endif
2047 	default:
2048 		panic("Unsupported address family %d",
2049 		    peer->remote.ss_family);
2050 	}
2051 }
2052 
2053 static int
2054 ovpn_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
2055 	struct route *ro)
2056 {
2057 	struct ovpn_softc *sc;
2058 	struct ovpn_kpeer *peer;
2059 
2060 	OVPN_RLOCK_TRACKER;
2061 
2062 	sc = ifp->if_softc;
2063 
2064 	OVPN_RLOCK(sc);
2065 
2066 	SDT_PROBE1(if_ovpn, tx, transmit, start, m);
2067 
2068 	if (__predict_false(ifp->if_link_state != LINK_STATE_UP)) {
2069 		OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
2070 		OVPN_RUNLOCK(sc);
2071 		m_freem(m);
2072 		return (ENETDOWN);
2073 	}
2074 
2075 	/**
2076 	 * Only obey 'dst' (i.e. the gateway) if no route is supplied.
2077 	 * That's our indication that we're being called through pf's route-to,
2078 	 * and we should route according to 'dst' instead. We can't do so
2079 	 * consistently, because the usual openvpn configuration sets the first
2080 	 * non-server IP in the subnet as the gateway. If we always use that
2081 	 * one we'd end up routing all traffic to the first client.
2082 	 * tl;dr: 'ro == NULL' tells us pf is doing a route-to, and then but
2083 	 * only then, we should treat 'dst' as the destination. */
2084 	peer = ovpn_route_peer(sc, &m, ro == NULL ? dst : NULL);
2085 	if (peer == NULL) {
2086 		/* No destination. */
2087 		OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
2088 		OVPN_RUNLOCK(sc);
2089 		m_freem(m);
2090 		return (ENETDOWN);
2091 	}
2092 
2093 	return (ovpn_transmit_to_peer(ifp, m, peer, _ovpn_lock_trackerp));
2094 }
2095 
2096 static bool
2097 ovpn_check_replay(struct ovpn_kkey_dir *key, uint32_t seq)
2098 {
2099 	uint32_t d;
2100 
2101 	mtx_lock(&key->replay_mtx);
2102 
2103 	/* Sequence number must be strictly greater than rx_seq */
2104 	if (seq <= key->rx_seq) {
2105 		mtx_unlock(&key->replay_mtx);
2106 		return (false);
2107 	}
2108 
2109 	/* Large jump. The packet authenticated okay, so just accept that. */
2110 	if (seq > (key->rx_seq + (sizeof(key->rx_window) * 8))) {
2111 		key->rx_seq = seq;
2112 		key->rx_window = 0;
2113 		mtx_unlock(&key->replay_mtx);
2114 		return (true);
2115 	}
2116 
2117 	/* Happy case. */
2118 	if ((seq == key->rx_seq + 1) && key->rx_window == 0) {
2119 		key->rx_seq++;
2120 		mtx_unlock(&key->replay_mtx);
2121 		return (true);
2122 	}
2123 
2124 	d = seq - key->rx_seq - 1;
2125 
2126 	if (key->rx_window & ((uint64_t)1 << d)) {
2127 		/* Dupe! */
2128 		mtx_unlock(&key->replay_mtx);
2129 		return (false);
2130 	}
2131 
2132 	key->rx_window |= (uint64_t)1 << d;
2133 
2134 	while (key->rx_window & 1) {
2135 		key->rx_seq++;
2136 		key->rx_window >>= 1;
2137 	}
2138 
2139 	mtx_unlock(&key->replay_mtx);
2140 
2141 	return (true);
2142 }
2143 
2144 static struct ovpn_kpeer *
2145 ovpn_peer_from_mbuf(struct ovpn_softc *sc, struct mbuf *m, int off)
2146 {
2147 	struct ovpn_wire_header ohdr;
2148 	uint32_t peerid;
2149 	const size_t hdrlen = sizeof(ohdr) - sizeof(ohdr.auth_tag);
2150 
2151 	OVPN_RASSERT(sc);
2152 
2153 	if (m_length(m, NULL) < (off + sizeof(struct udphdr) + hdrlen))
2154 		return (NULL);
2155 
2156 	m_copydata(m, off + sizeof(struct udphdr), hdrlen, (caddr_t)&ohdr);
2157 
2158 	peerid = ntohl(ohdr.opcode) & 0x00ffffff;
2159 
2160 	return (ovpn_find_peer(sc, peerid));
2161 }
2162 
2163 static bool
2164 ovpn_udp_input(struct mbuf *m, int off, struct inpcb *inp,
2165     const struct sockaddr *sa, void *ctx)
2166 {
2167 	struct ovpn_softc *sc = ctx;
2168 	struct ovpn_wire_header tmphdr;
2169 	struct ovpn_wire_header *ohdr;
2170 	struct udphdr *uhdr;
2171 	struct ovpn_kkey *key;
2172 	struct cryptop *crp;
2173 	struct ovpn_kpeer *peer;
2174 	size_t ohdrlen;
2175 	int ret;
2176 	uint8_t op;
2177 
2178 	OVPN_RLOCK_TRACKER;
2179 
2180 	M_ASSERTPKTHDR(m);
2181 
2182 	OVPN_COUNTER_ADD(sc, transport_bytes_received, m->m_pkthdr.len - off);
2183 
2184 	ohdrlen = sizeof(*ohdr) - sizeof(ohdr->auth_tag);
2185 
2186 	OVPN_RLOCK(sc);
2187 
2188 	peer = ovpn_peer_from_mbuf(sc, m, off);
2189 	if (peer == NULL) {
2190 		OVPN_RUNLOCK(sc);
2191 		return (false);
2192 	}
2193 
2194 	if (m_length(m, NULL) < (off + sizeof(*uhdr) + ohdrlen)) {
2195 		/* Short packet. */
2196 		OVPN_RUNLOCK(sc);
2197 		return (false);
2198 	}
2199 
2200 	m_copydata(m, off + sizeof(*uhdr), ohdrlen, (caddr_t)&tmphdr);
2201 
2202 	op = ntohl(tmphdr.opcode) >> 24 >> OVPN_OP_SHIFT;
2203 	if (op != OVPN_OP_DATA_V2) {
2204 		/* Control packet? */
2205 		OVPN_RUNLOCK(sc);
2206 		return (false);
2207 	}
2208 
2209 	m = m_pullup(m, off + sizeof(*uhdr) + ohdrlen);
2210 	if (m == NULL) {
2211 		OVPN_RUNLOCK(sc);
2212 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1);
2213 		return (true);
2214 	}
2215 
2216 	/*
2217 	 * Simplify things by getting rid of the preceding headers, we don't
2218 	 * care about them.
2219 	 */
2220 	m_adj_decap(m, off);
2221 
2222 	uhdr = mtodo(m, 0);
2223 	ohdr = mtodo(m, sizeof(*uhdr));
2224 
2225 	key = ovpn_find_key(sc, peer, ohdr);
2226 	if (key == NULL || key->decrypt == NULL) {
2227 		OVPN_RUNLOCK(sc);
2228 		OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
2229 		m_freem(m);
2230 		return (true);
2231 	}
2232 
2233 	if (key->decrypt->cipher == OVPN_CIPHER_ALG_NONE) {
2234 		/* Now remove the outer headers */
2235 		m_adj_decap(m, sizeof(struct udphdr) + ohdrlen);
2236 
2237 		ohdr = mtodo(m, sizeof(*uhdr));
2238 
2239 		ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq),
2240 		    _ovpn_lock_trackerp);
2241 		OVPN_UNLOCK_ASSERT(sc);
2242 		return (true);
2243 	}
2244 
2245 	ohdrlen += sizeof(ohdr->auth_tag);
2246 
2247 	m = m_pullup(m, sizeof(*uhdr) + ohdrlen);
2248 	if (m == NULL) {
2249 		OVPN_RUNLOCK(sc);
2250 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1);
2251 		return (true);
2252 	}
2253 	uhdr = mtodo(m, 0);
2254 	ohdr = mtodo(m, sizeof(*uhdr));
2255 
2256 	/* Decrypt */
2257 	crp = crypto_getreq(key->decrypt->cryptoid, M_NOWAIT);
2258 	if (crp == NULL) {
2259 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1);
2260 		OVPN_RUNLOCK(sc);
2261 		m_freem(m);
2262 		return (true);
2263 	}
2264 
2265 	crp->crp_payload_start = sizeof(struct udphdr) + sizeof(*ohdr);
2266 	crp->crp_payload_length = ntohs(uhdr->uh_ulen) -
2267 	    sizeof(*uhdr) - sizeof(*ohdr);
2268 	crp->crp_op = CRYPTO_OP_DECRYPT;
2269 
2270 	/* AAD validation. */
2271 	crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag);
2272 	crp->crp_aad = ohdr;
2273 	crp->crp_aad_start = 0;
2274 	crp->crp_op |= CRYPTO_OP_VERIFY_DIGEST;
2275 	crp->crp_digest_start = sizeof(struct udphdr) +
2276 	    offsetof(struct ovpn_wire_header, auth_tag);
2277 
2278 	crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
2279 	memcpy(crp->crp_iv, &ohdr->seq, sizeof(ohdr->seq));
2280 	memcpy(crp->crp_iv + sizeof(ohdr->seq), key->decrypt->nonce,
2281 	    key->decrypt->noncelen);
2282 
2283 	crypto_use_mbuf(crp, m);
2284 	crp->crp_flags |= CRYPTO_F_CBIFSYNC;
2285 	crp->crp_callback = ovpn_decrypt_rx_cb;
2286 	crp->crp_opaque = sc;
2287 
2288 	atomic_add_int(&sc->refcount, 1);
2289 	OVPN_RUNLOCK(sc);
2290 	if (V_async_crypto)
2291 		ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED);
2292 	else
2293 		ret = crypto_dispatch(crp);
2294 	if (ret != 0) {
2295 		OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
2296 	}
2297 
2298 	return (true);
2299 }
2300 
2301 static void
2302 ovpn_qflush(struct ifnet *ifp __unused)
2303 {
2304 
2305 }
2306 
2307 static void
2308 ovpn_flush_rxring(struct ovpn_softc *sc)
2309 {
2310 	struct ovpn_notification *n;
2311 
2312 	OVPN_WASSERT(sc);
2313 
2314 	while (! buf_ring_empty(sc->notifring)) {
2315 		n = buf_ring_dequeue_sc(sc->notifring);
2316 		free(n, M_OVPN);
2317 	}
2318 }
2319 
2320 #ifdef VIMAGE
2321 static void
2322 ovpn_reassign(struct ifnet *ifp, struct vnet *new_vnet __unused,
2323     char *unused __unused)
2324 {
2325 	struct ovpn_softc *sc = ifp->if_softc;
2326 	struct ovpn_kpeer *peer, *tmppeer;
2327 	int ret __diagused;
2328 
2329 	OVPN_WLOCK(sc);
2330 
2331 	/* Flush keys & configuration. */
2332 	RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) {
2333 		peer->del_reason = OVPN_DEL_REASON_REQUESTED;
2334 		ret = _ovpn_del_peer(sc, peer);
2335 		MPASS(ret == 0);
2336 	}
2337 
2338 	ovpn_flush_rxring(sc);
2339 
2340 	OVPN_WUNLOCK(sc);
2341 }
2342 #endif
2343 
2344 static int
2345 ovpn_clone_match(struct if_clone *ifc, const char *name)
2346 {
2347 	/*
2348 	 * Allow all names that start with 'ovpn', specifically because pfSense
2349 	 * uses ovpnc1 / ovpns2
2350 	 */
2351 	return (strncmp(ovpnname, name, strlen(ovpnname)) == 0);
2352 }
2353 
2354 static int
2355 ovpn_clone_create(struct if_clone *ifc, char *name, size_t len,
2356     struct ifc_data *ifd, struct ifnet **ifpp)
2357 {
2358 	struct ovpn_softc *sc;
2359 	struct ifnet *ifp;
2360 	char *dp;
2361 	int error, unit, wildcard;
2362 
2363 	/* Try to see if a special unit was requested. */
2364 	error = ifc_name2unit(name, &unit);
2365 	if (error != 0)
2366 		return (error);
2367 	wildcard = (unit < 0);
2368 
2369 	error = ifc_alloc_unit(ifc, &unit);
2370 	if (error != 0)
2371 		return (error);
2372 
2373 	/*
2374 	 * If no unit had been given, we need to adjust the ifName.
2375 	 */
2376 	for (dp = name; *dp != '\0'; dp++);
2377 	if (wildcard) {
2378 		error = snprintf(dp, len - (dp - name), "%d", unit);
2379 		if (error > len - (dp - name)) {
2380 			/* ifName too long. */
2381 			ifc_free_unit(ifc, unit);
2382 			return (ENOSPC);
2383 		}
2384 		dp += error;
2385 	}
2386 
2387 	/* Make sure it doesn't already exist. */
2388 	if (ifunit(name) != NULL)
2389 		return (EEXIST);
2390 
2391 	sc = malloc(sizeof(struct ovpn_softc), M_OVPN, M_WAITOK | M_ZERO);
2392 	sc->ifp = if_alloc(IFT_ENC);
2393 	rm_init_flags(&sc->lock, "if_ovpn_lock", RM_RECURSE);
2394 	sc->refcount = 0;
2395 
2396 	sc->notifring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL);
2397 
2398 	COUNTER_ARRAY_ALLOC(sc->counters, OVPN_COUNTER_SIZE, M_WAITOK);
2399 
2400 	ifp = sc->ifp;
2401 	ifp->if_softc = sc;
2402 	strlcpy(ifp->if_xname, name, IFNAMSIZ);
2403 	ifp->if_dname = ovpngroupname;
2404 	ifp->if_dunit = unit;
2405 
2406 	ifp->if_addrlen = 0;
2407 	ifp->if_mtu = 1428;
2408 	ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST;
2409 	ifp->if_ioctl = ovpn_ioctl;
2410 	ifp->if_transmit = ovpn_transmit;
2411 	ifp->if_output = ovpn_output;
2412 	ifp->if_qflush = ovpn_qflush;
2413 #ifdef VIMAGE
2414 	ifp->if_reassign = ovpn_reassign;
2415 #endif
2416 	ifp->if_capabilities |= IFCAP_LINKSTATE;
2417 	ifp->if_capenable |= IFCAP_LINKSTATE;
2418 
2419 	if_attach(ifp);
2420 	bpfattach(ifp, DLT_NULL, sizeof(uint32_t));
2421 	*ifpp = ifp;
2422 
2423 	return (0);
2424 }
2425 
2426 static void
2427 ovpn_clone_destroy_cb(struct epoch_context *ctx)
2428 {
2429 	struct ovpn_softc *sc;
2430 
2431 	sc = __containerof(ctx, struct ovpn_softc, epoch_ctx);
2432 
2433 	MPASS(sc->peercount == 0);
2434 	MPASS(RB_EMPTY(&sc->peers));
2435 
2436 	COUNTER_ARRAY_FREE(sc->counters, OVPN_COUNTER_SIZE);
2437 
2438 	if_free(sc->ifp);
2439 	free(sc, M_OVPN);
2440 }
2441 
2442 static int
2443 ovpn_clone_destroy(struct if_clone *ifc, struct ifnet *ifp, uint32_t flags)
2444 {
2445 	struct ovpn_softc *sc;
2446 	struct ovpn_kpeer *peer, *tmppeer;
2447 	int unit;
2448 	int ret __diagused;
2449 
2450 	sc = ifp->if_softc;
2451 	unit = ifp->if_dunit;
2452 
2453 	OVPN_WLOCK(sc);
2454 
2455 	if (atomic_load_int(&sc->refcount) > 0) {
2456 		OVPN_WUNLOCK(sc);
2457 		return (EBUSY);
2458 	}
2459 
2460 	RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) {
2461 		peer->del_reason = OVPN_DEL_REASON_REQUESTED;
2462 		ret = _ovpn_del_peer(sc, peer);
2463 		MPASS(ret == 0);
2464 	}
2465 
2466 	ovpn_flush_rxring(sc);
2467 	buf_ring_free(sc->notifring, M_OVPN);
2468 
2469 	OVPN_WUNLOCK(sc);
2470 
2471 	bpfdetach(ifp);
2472 	if_detach(ifp);
2473 	ifp->if_softc = NULL;
2474 
2475 	NET_EPOCH_CALL(ovpn_clone_destroy_cb, &sc->epoch_ctx);
2476 
2477 	if (unit != IF_DUNIT_NONE)
2478 		ifc_free_unit(ifc, unit);
2479 
2480 	NET_EPOCH_DRAIN_CALLBACKS();
2481 
2482 	return (0);
2483 }
2484 
2485 static void
2486 vnet_ovpn_init(const void *unused __unused)
2487 {
2488 	struct if_clone_addreq req = {
2489 		.match_f = ovpn_clone_match,
2490 		.create_f = ovpn_clone_create,
2491 		.destroy_f = ovpn_clone_destroy,
2492 	};
2493 	V_ovpn_cloner = ifc_attach_cloner(ovpngroupname, &req);
2494 }
2495 VNET_SYSINIT(vnet_ovpn_init, SI_SUB_PSEUDO, SI_ORDER_ANY,
2496     vnet_ovpn_init, NULL);
2497 
2498 static void
2499 vnet_ovpn_uninit(const void *unused __unused)
2500 {
2501 	if_clone_detach(V_ovpn_cloner);
2502 }
2503 VNET_SYSUNINIT(vnet_ovpn_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY,
2504     vnet_ovpn_uninit, NULL);
2505 
2506 static int
2507 ovpnmodevent(module_t mod, int type, void *data)
2508 {
2509 	switch (type) {
2510 	case MOD_LOAD:
2511 		/* Done in vnet_ovpn_init() */
2512 		break;
2513 	case MOD_UNLOAD:
2514 		/* Done in vnet_ovpn_uninit() */
2515 		break;
2516 	default:
2517 		return (EOPNOTSUPP);
2518 	}
2519 
2520 	return (0);
2521 }
2522 
2523 static moduledata_t ovpn_mod = {
2524 	"if_ovpn",
2525 	ovpnmodevent,
2526 	0
2527 };
2528 
2529 DECLARE_MODULE(if_ovpn, ovpn_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
2530 MODULE_VERSION(if_ovpn, 1);
2531