xref: /freebsd/sys/net/if_ovpn.c (revision 23c73dbae7b2f64014b275643efbe92f449feaa2)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2021-2022 Rubicon Communications, LLC (Netgate)
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  *
27  */
28 #include "opt_inet.h"
29 #include "opt_inet6.h"
30 
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/buf_ring.h>
34 #include <sys/epoch.h>
35 #include <sys/file.h>
36 #include <sys/filedesc.h>
37 #include <sys/kernel.h>
38 #include <sys/malloc.h>
39 #include <sys/mbuf.h>
40 #include <sys/module.h>
41 #include <sys/nv.h>
42 #include <sys/priv.h>
43 #include <sys/protosw.h>
44 #include <sys/rmlock.h>
45 #include <sys/sdt.h>
46 #include <sys/smp.h>
47 #include <sys/socket.h>
48 #include <sys/socketvar.h>
49 #include <sys/sockio.h>
50 #include <sys/sysctl.h>
51 #include <sys/time.h>
52 
53 #include <machine/atomic.h>
54 
55 #include <net/bpf.h>
56 #include <net/if.h>
57 #include <net/if_clone.h>
58 #include <net/if_types.h>
59 #include <net/if_var.h>
60 #include <net/netisr.h>
61 #include <net/route/nhop.h>
62 
63 #include <netinet/in.h>
64 #include <netinet/in_fib.h>
65 #include <netinet/ip.h>
66 #include <netinet/ip6.h>
67 #include <netinet/ip_var.h>
68 #include <netinet/udp.h>
69 #include <netinet/udp_var.h>
70 
71 #include <netinet6/ip6_var.h>
72 #include <netinet6/in6_fib.h>
73 
74 #include <machine/in_cksum.h>
75 
76 #include <opencrypto/cryptodev.h>
77 
78 #include "if_ovpn.h"
79 
80 struct ovpn_kkey_dir {
81 	int			refcount;
82 	uint8_t			key[32];
83 	uint8_t			keylen;
84 	uint8_t			nonce[8];
85 	uint8_t			noncelen;
86 	enum ovpn_key_cipher	cipher;
87 	crypto_session_t	cryptoid;
88 
89 	struct mtx		replay_mtx;
90 	/*
91 	 * Last seen gapless sequence number. New rx seq numbers must be
92 	 * strictly higher than this.
93 	 */
94 	uint32_t		rx_seq;
95 	/* Seen packets, relative to rx_seq. bit(0) will always be 0. */
96 	uint64_t		rx_window;
97 };
98 
99 struct ovpn_kkey {
100 	struct ovpn_kkey_dir	*encrypt;
101 	struct ovpn_kkey_dir	*decrypt;
102 	uint8_t			 keyid;
103 	uint32_t		 peerid;
104 };
105 
106 struct ovpn_keepalive {
107 	uint32_t	interval;
108 	uint32_t	timeout;
109 };
110 
111 struct ovpn_wire_header {
112 	uint32_t	 opcode; /* opcode, key id, peer id */
113 	uint32_t	 seq;
114 	uint8_t		 auth_tag[16];
115 };
116 
117 struct ovpn_notification {
118 	enum ovpn_notif_type	type;
119 	uint32_t		peerid;
120 };
121 
122 struct ovpn_softc;
123 
124 struct ovpn_kpeer {
125 	int			 refcount;
126 	uint32_t		 peerid;
127 
128 	struct ovpn_softc	*sc;
129 	struct sockaddr_storage	 local;
130 	struct sockaddr_storage	 remote;
131 
132 	struct in_addr		 vpn4;
133 	struct in6_addr		 vpn6;
134 
135 	struct ovpn_kkey	 keys[2];
136 	uint32_t		 tx_seq;
137 
138 	struct ovpn_keepalive	 keepalive;
139 	uint32_t		*last_active;
140 	struct callout		 ping_send;
141 	struct callout		 ping_rcv;
142 };
143 
144 #define OVPN_MAX_PEERS	128
145 
146 struct ovpn_counters {
147 	uint64_t	lost_ctrl_pkts_in;
148 	uint64_t	lost_ctrl_pkts_out;
149 	uint64_t	lost_data_pkts_in;
150 	uint64_t	lost_data_pkts_out;
151 	uint64_t	nomem_data_pkts_in;
152 	uint64_t	nomem_data_pkts_out;
153 	uint64_t	received_ctrl_pkts;
154 	uint64_t	received_data_pkts;
155 	uint64_t	sent_ctrl_pkts;
156 	uint64_t	sent_data_pkts;
157 
158 	uint64_t	transport_bytes_sent;
159 	uint64_t	transport_bytes_received;
160 	uint64_t	tunnel_bytes_sent;
161 	uint64_t	tunnel_bytes_received;
162 };
163 #define OVPN_COUNTER_SIZE (sizeof(struct ovpn_counters)/sizeof(uint64_t))
164 
165 struct ovpn_softc {
166 	int			 refcount;
167 	struct rmlock		 lock;
168 	struct ifnet		*ifp;
169 	struct socket		*so;
170 	int			 peercount;
171 	struct ovpn_kpeer	*peers[OVPN_MAX_PEERS]; /* XXX Hard limit for now? */
172 
173 	/* Pending notification */
174 	struct buf_ring		*notifring;
175 
176 	counter_u64_t 		 counters[OVPN_COUNTER_SIZE];
177 
178 	struct epoch_context	 epoch_ctx;
179 };
180 
181 static struct ovpn_kpeer *ovpn_find_peer(struct ovpn_softc *, uint32_t);
182 static bool ovpn_udp_input(struct mbuf *, int, struct inpcb *,
183     const struct sockaddr *, void *);
184 static int ovpn_transmit_to_peer(struct ifnet *, struct mbuf *,
185     struct ovpn_kpeer *, struct rm_priotracker *);
186 static int ovpn_encap(struct ovpn_softc *, uint32_t, struct mbuf *);
187 static int ovpn_get_af(struct mbuf *);
188 static void ovpn_free_kkey_dir(struct ovpn_kkey_dir *);
189 static bool ovpn_check_replay(struct ovpn_kkey_dir *, uint32_t);
190 
191 #define OVPN_MTU_MIN		576
192 #define OVPN_MTU_MAX		(IP_MAXPACKET - sizeof(struct ip) - \
193     sizeof(struct udphdr) - sizeof(struct ovpn_wire_header))
194 
195 #define OVPN_OP_DATA_V2		0x09
196 #define OVPN_OP_SHIFT		3
197 
198 VNET_DEFINE_STATIC(struct if_clone *, ovpn_cloner);
199 #define	V_ovpn_cloner	VNET(ovpn_cloner)
200 
201 #define OVPN_RLOCK_TRACKER	struct rm_priotracker _ovpn_lock_tracker; \
202     struct rm_priotracker *_ovpn_lock_trackerp = &_ovpn_lock_tracker
203 #define OVPN_RLOCK(sc)		rm_rlock(&(sc)->lock, _ovpn_lock_trackerp)
204 #define OVPN_RUNLOCK(sc)	rm_runlock(&(sc)->lock, _ovpn_lock_trackerp)
205 #define OVPN_WLOCK(sc)		rm_wlock(&(sc)->lock)
206 #define OVPN_WUNLOCK(sc)	rm_wunlock(&(sc)->lock)
207 #define OVPN_ASSERT(sc)		rm_assert(&(sc)->lock, RA_LOCKED)
208 #define OVPN_RASSERT(sc)	rm_assert(&(sc)->lock, RA_RLOCKED)
209 #define OVPN_WASSERT(sc)	rm_assert(&(sc)->lock, RA_WLOCKED)
210 #define OVPN_UNLOCK_ASSERT(sc)	rm_assert(&(sc)->lock, RA_UNLOCKED)
211 
212 #define OVPN_COUNTER_ADD(sc, name, val)	\
213 	counter_u64_add(sc->counters[offsetof(struct ovpn_counters, name) / \
214 	    sizeof(uint64_t)], val)
215 
216 #define TO_IN(x)		((struct sockaddr_in *)(x))
217 #define TO_IN6(x)		((struct sockaddr_in6 *)(x))
218 
219 SDT_PROVIDER_DEFINE(if_ovpn);
220 SDT_PROBE_DEFINE1(if_ovpn, tx, transmit, start, "struct mbuf *");
221 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip4, "struct in_addr *", "struct ovpn_kpeer *");
222 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip6, "struct in6_addr *", "struct ovpn_kpeer *");
223 
224 static const char ovpnname[] = "ovpn";
225 static const char ovpngroupname[] = "openvpn";
226 
227 static MALLOC_DEFINE(M_OVPN, ovpnname, "OpenVPN DCO Interface");
228 
229 SYSCTL_DECL(_net_link);
230 static SYSCTL_NODE(_net_link, IFT_OTHER, openvpn, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
231     "OpenVPN DCO Interface");
232 VNET_DEFINE_STATIC(int, replay_protection) = 0;
233 #define	V_replay_protection	VNET(replay_protection)
234 SYSCTL_INT(_net_link_openvpn, OID_AUTO, replay_protection, CTLFLAG_VNET | CTLFLAG_RW,
235     &VNET_NAME(replay_protection), 0, "Validate sequence numbers");
236 
237 VNET_DEFINE_STATIC(int, async_crypto);
238 #define	V_async_crypto		VNET(async_crypto)
239 SYSCTL_INT(_net_link_openvpn, OID_AUTO, async_crypto,
240 	CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_crypto), 0,
241 	"Use asynchronous mode to parallelize crypto jobs.");
242 
243 VNET_DEFINE_STATIC(int, async_netisr_queue);
244 #define	V_async_netisr_queue		VNET(async_netisr_queue)
245 SYSCTL_INT(_net_link_openvpn, OID_AUTO, netisr_queue,
246 	CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_netisr_queue), 0,
247 	"Use netisr_queue() rather than netisr_dispatch().");
248 
249 static struct ovpn_kpeer *
250 ovpn_find_peer(struct ovpn_softc *sc, uint32_t peerid)
251 {
252 	struct ovpn_kpeer *p = NULL;
253 
254 	OVPN_ASSERT(sc);
255 
256 	for (int i = 0; i < OVPN_MAX_PEERS; i++) {
257 		p = sc->peers[i];
258 		if (p == NULL)
259 			continue;
260 
261 		if (p->peerid == peerid) {
262 			MPASS(p->sc == sc);
263 			break;
264 		}
265 	}
266 
267 	return (p);
268 }
269 
270 static struct ovpn_kpeer *
271 ovpn_find_only_peer(struct ovpn_softc *sc)
272 {
273 	OVPN_ASSERT(sc);
274 
275 	for (int i = 0; i < OVPN_MAX_PEERS; i++) {
276 		if (sc->peers[i] == NULL)
277 			continue;
278 		return (sc->peers[i]);
279 	}
280 
281 	MPASS(false);
282 
283 	return (NULL);
284 }
285 
286 static uint16_t
287 ovpn_get_port(struct sockaddr_storage *s)
288 {
289 	switch (s->ss_family) {
290 	case AF_INET: {
291 		struct sockaddr_in *in = (struct sockaddr_in *)s;
292 		return (in->sin_port);
293 	}
294 	case AF_INET6: {
295 		struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)s;
296 		return (in6->sin6_port);
297 	}
298 	default:
299 		panic("Unsupported address family %d", s->ss_family);
300 	}
301 }
302 
303 static int
304 ovpn_nvlist_to_sockaddr(const nvlist_t *nvl, struct sockaddr_storage *sa)
305 {
306 	int af;
307 
308 	if (! nvlist_exists_number(nvl, "af"))
309 		return (EINVAL);
310 	if (! nvlist_exists_binary(nvl, "address"))
311 		return (EINVAL);
312 	if (! nvlist_exists_number(nvl, "port"))
313 		return (EINVAL);
314 
315 	af = nvlist_get_number(nvl, "af");
316 
317 	switch (af) {
318 #ifdef INET
319 	case AF_INET: {
320 		struct sockaddr_in *in = (struct sockaddr_in *)sa;
321 		size_t len;
322 		const void *addr = nvlist_get_binary(nvl, "address", &len);
323 		in->sin_family = af;
324 		if (len != sizeof(in->sin_addr))
325 			return (EINVAL);
326 
327 		memcpy(&in->sin_addr, addr, sizeof(in->sin_addr));
328 		in->sin_port = nvlist_get_number(nvl, "port");
329 		break;
330 	}
331 #endif
332 #ifdef INET6
333 	case AF_INET6: {
334 		struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)sa;
335 		size_t len;
336 		const void *addr = nvlist_get_binary(nvl, "address", &len);
337 		in6->sin6_family = af;
338 		if (len != sizeof(in6->sin6_addr))
339 			return (EINVAL);
340 
341 		memcpy(&in6->sin6_addr, addr, sizeof(in6->sin6_addr));
342 		in6->sin6_port = nvlist_get_number(nvl, "port");
343 		break;
344 	}
345 #endif
346 	default:
347 		return (EINVAL);
348 	}
349 
350 	return (0);
351 }
352 
353 static bool
354 ovpn_has_peers(struct ovpn_softc *sc)
355 {
356 	OVPN_ASSERT(sc);
357 
358 	return (sc->peercount > 0);
359 }
360 
361 static void
362 ovpn_rele_so(struct ovpn_softc *sc, struct ovpn_kpeer *peer)
363 {
364 	bool has_peers;
365 
366 	OVPN_WASSERT(sc);
367 
368 	if (sc->so == NULL)
369 		return;
370 
371 	has_peers = ovpn_has_peers(sc);
372 
373 	/* Only remove the tunnel function if we're releasing the socket for
374 	 * the last peer. */
375 	if (! has_peers)
376 		(void)udp_set_kernel_tunneling(sc->so, NULL, NULL, NULL);
377 
378 	sorele(sc->so);
379 
380 	if (! has_peers)
381 		sc->so = NULL;
382 }
383 
384 static void
385 ovpn_notify_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer)
386 {
387 	struct ovpn_notification *n;
388 
389 	OVPN_WASSERT(sc);
390 
391 	n = malloc(sizeof(*n), M_OVPN, M_NOWAIT);
392 	if (n == NULL)
393 		return;
394 
395 	n->peerid = peer->peerid;
396 	n->type = OVPN_NOTIF_DEL_PEER;
397 	if (buf_ring_enqueue(sc->notifring, n) != 0) {
398 		free(n, M_OVPN);
399 	} else if (sc->so != NULL) {
400 		/* Wake up userspace */
401 		sc->so->so_error = EAGAIN;
402 		sorwakeup(sc->so);
403 		sowwakeup(sc->so);
404 	}
405 }
406 
407 static void
408 ovpn_peer_release_ref(struct ovpn_kpeer *peer, bool locked)
409 {
410 	struct ovpn_softc *sc;
411 
412 	CURVNET_ASSERT_SET();
413 
414 	atomic_add_int(&peer->refcount, -1);
415 
416 	if (atomic_load_int(&peer->refcount) > 0)
417 		return;
418 
419 	sc = peer->sc;
420 
421 	if (! locked) {
422 		OVPN_WLOCK(sc);
423 
424 		/* Might have changed before we acquired the lock. */
425 		if (atomic_load_int(&peer->refcount) > 0) {
426 			OVPN_WUNLOCK(sc);
427 			return;
428 		}
429 	}
430 
431 	OVPN_ASSERT(sc);
432 
433 	/* The peer should have been removed from the list already. */
434 	MPASS(ovpn_find_peer(sc, peer->peerid) == NULL);
435 
436 	ovpn_notify_del_peer(sc, peer);
437 
438 	for (int i = 0; i < 2; i++) {
439 		ovpn_free_kkey_dir(peer->keys[i].encrypt);
440 		ovpn_free_kkey_dir(peer->keys[i].decrypt);
441 	}
442 
443 	ovpn_rele_so(sc, peer);
444 
445 	callout_stop(&peer->ping_send);
446 	callout_stop(&peer->ping_rcv);
447 	uma_zfree_pcpu(pcpu_zone_4, peer->last_active);
448 	free(peer, M_OVPN);
449 
450 	if (! locked)
451 		OVPN_WUNLOCK(sc);
452 }
453 
454 static int
455 ovpn_new_peer(struct ifnet *ifp, const nvlist_t *nvl)
456 {
457 #ifdef INET6
458 	struct epoch_tracker et;
459 #endif
460 	struct sockaddr_storage remote;
461 	struct ovpn_kpeer *peer = NULL;
462 	struct file *fp = NULL;
463 	struct sockaddr *name = NULL;
464 	struct ovpn_softc *sc = ifp->if_softc;
465 	struct thread *td = curthread;
466 	struct socket *so = NULL;
467 	int fd;
468 	uint32_t peerid;
469 	int ret = 0, i;
470 
471 	if (nvl == NULL)
472 		return (EINVAL);
473 
474 	if (! nvlist_exists_number(nvl, "peerid"))
475 		return (EINVAL);
476 
477 	if (! nvlist_exists_number(nvl, "fd"))
478 		return (EINVAL);
479 
480 	if (! nvlist_exists_nvlist(nvl, "remote"))
481 		return (EINVAL);
482 
483 	peerid = nvlist_get_number(nvl, "peerid");
484 
485 	ret = ovpn_nvlist_to_sockaddr(nvlist_get_nvlist(nvl, "remote"),
486 	    &remote);
487 	if (ret != 0)
488 		return (ret);
489 
490 	fd = nvlist_get_number(nvl, "fd");
491 
492 	/* Look up the userspace process and use the fd to find the socket. */
493 	ret = getsock(td, fd, &cap_connect_rights, &fp);
494 	if (ret != 0)
495 		return (ret);
496 
497 	so = fp->f_data;
498 
499 	peer = malloc(sizeof(*peer), M_OVPN, M_WAITOK | M_ZERO);
500 	peer->peerid = peerid;
501 	peer->sc = sc;
502 	peer->tx_seq = 1;
503 	peer->refcount = 1;
504 	peer->last_active = uma_zalloc_pcpu(pcpu_zone_4, M_WAITOK | M_ZERO);
505 
506 	if (nvlist_exists_binary(nvl, "vpn_ipv4")) {
507 		size_t len;
508 		const void *addr = nvlist_get_binary(nvl, "vpn_ipv4", &len);
509 		if (len != sizeof(peer->vpn4)) {
510 			ret = EINVAL;
511 			goto error;
512 		}
513 		memcpy(&peer->vpn4, addr, len);
514 	}
515 
516 	if (nvlist_exists_binary(nvl, "vpn_ipv6")) {
517 		size_t len;
518 		const void *addr = nvlist_get_binary(nvl, "vpn_ipv6", &len);
519 		if (len != sizeof(peer->vpn6)) {
520 			ret = EINVAL;
521 			goto error;
522 		}
523 		memcpy(&peer->vpn6, addr, len);
524 	}
525 
526 	callout_init_rm(&peer->ping_send, &sc->lock, CALLOUT_SHAREDLOCK);
527 	callout_init_rm(&peer->ping_rcv, &sc->lock, 0);
528 
529 	ret = so->so_proto->pr_sockaddr(so, &name);
530 	if (ret)
531 		goto error;
532 
533 	if (ovpn_get_port((struct sockaddr_storage *)name) == 0) {
534 		ret = EINVAL;
535 		goto error;
536 	}
537 	if (name->sa_family != remote.ss_family) {
538 		ret = EINVAL;
539 		goto error;
540 	}
541 
542 	memcpy(&peer->local, name, name->sa_len);
543 	memcpy(&peer->remote, &remote, sizeof(remote));
544 	free(name, M_SONAME);
545 	name = NULL;
546 
547 	if (peer->local.ss_family == AF_INET6 &&
548 	    IN6_IS_ADDR_V4MAPPED(&TO_IN6(&peer->remote)->sin6_addr)) {
549 		/* V4 mapped address, so treat this as v4, not v6. */
550 		in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->local);
551 		in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->remote);
552 	}
553 
554 #ifdef INET6
555 	if (peer->local.ss_family == AF_INET6 &&
556 	    IN6_IS_ADDR_UNSPECIFIED(&TO_IN6(&peer->local)->sin6_addr)) {
557 		NET_EPOCH_ENTER(et);
558 		ret = in6_selectsrc_addr(curthread->td_proc->p_fibnum,
559 		    &TO_IN6(&peer->remote)->sin6_addr,
560 		    0, NULL, &TO_IN6(&peer->local)->sin6_addr, NULL);
561 		NET_EPOCH_EXIT(et);
562 		if (ret != 0) {
563 			goto error;
564 		}
565 	}
566 #endif
567 	OVPN_WLOCK(sc);
568 
569 	/* Disallow peer id re-use. */
570 	if (ovpn_find_peer(sc, peerid) != NULL) {
571 		ret = EEXIST;
572 		goto error_locked;
573 	}
574 
575 	/* Make sure this is really a UDP socket. */
576 	if (so->so_type != SOCK_DGRAM || so->so_proto->pr_type != SOCK_DGRAM) {
577 		ret = EPROTOTYPE;
578 		goto error_locked;
579 	}
580 
581 	/* Must be the same socket as for other peers on this interface. */
582 	if (sc->so != NULL && so != sc->so)
583 		goto error_locked;
584 
585 	if (sc->so == NULL)
586 		sc->so = so;
587 
588 	/* Insert the peer into the list. */
589 	for (i = 0; i < OVPN_MAX_PEERS; i++) {
590 		if (sc->peers[i] != NULL)
591 			continue;
592 
593 		MPASS(sc->peers[i] == NULL);
594 		sc->peers[i] = peer;
595 		sc->peercount++;
596 		soref(sc->so);
597 		break;
598 	}
599 	if (i == OVPN_MAX_PEERS) {
600 		ret = ENOSPC;
601 		goto error_locked;
602 	}
603 
604 	ret = udp_set_kernel_tunneling(sc->so, ovpn_udp_input, NULL, sc);
605 	if (ret == EBUSY) {
606 		/* Fine, another peer already set the input function. */
607 		ret = 0;
608 	}
609 	if (ret != 0) {
610 		sc->peers[i] = NULL;
611 		sc->peercount--;
612 		goto error_locked;
613 	}
614 
615 	OVPN_WUNLOCK(sc);
616 
617 	goto done;
618 
619 error_locked:
620 	OVPN_WUNLOCK(sc);
621 error:
622 	free(name, M_SONAME);
623 	uma_zfree_pcpu(pcpu_zone_4, peer->last_active);
624 	free(peer, M_OVPN);
625 done:
626 	if (fp != NULL)
627 		fdrop(fp, td);
628 
629 	return (ret);
630 }
631 
632 static int
633 _ovpn_del_peer(struct ovpn_softc *sc, uint32_t peerid)
634 {
635 	struct ovpn_kpeer *peer;
636 	int i;
637 
638 	OVPN_WASSERT(sc);
639 	CURVNET_ASSERT_SET();
640 
641 	for (i = 0; i < OVPN_MAX_PEERS; i++) {
642 		if (sc->peers[i] == NULL)
643 			continue;
644 		if (sc->peers[i]->peerid != peerid)
645 			continue;
646 
647 		peer = sc->peers[i];
648 		break;
649 	}
650 	if (i == OVPN_MAX_PEERS)
651 		return (ENOENT);
652 
653 	sc->peers[i] = NULL;
654 	sc->peercount--;
655 
656 	ovpn_peer_release_ref(peer, true);
657 
658 	return (0);
659 }
660 
661 static int
662 ovpn_del_peer(struct ifnet *ifp, nvlist_t *nvl)
663 {
664 	struct ovpn_softc *sc = ifp->if_softc;
665 	uint32_t peerid;
666 	int ret;
667 
668 	OVPN_WASSERT(sc);
669 
670 	if (nvl == NULL)
671 		return (EINVAL);
672 
673 	if (! nvlist_exists_number(nvl, "peerid"))
674 		return (EINVAL);
675 
676 	peerid = nvlist_get_number(nvl, "peerid");
677 
678 	ret = _ovpn_del_peer(sc, peerid);
679 
680 	return (ret);
681 }
682 
683 static int
684 ovpn_create_kkey_dir(struct ovpn_kkey_dir **kdirp,
685     const nvlist_t *nvl)
686 {
687 	struct crypto_session_params csp;
688 	struct ovpn_kkey_dir *kdir;
689 	const char *ciphername;
690 	enum ovpn_key_cipher cipher;
691 	const void *key, *iv;
692 	size_t keylen = 0, ivlen = 0;
693 	int error;
694 
695 	if (! nvlist_exists_string(nvl, "cipher"))
696 		return (EINVAL);
697 	ciphername = nvlist_get_string(nvl, "cipher");
698 
699 	if (strcmp(ciphername, "none") == 0)
700 		cipher = OVPN_CIPHER_ALG_NONE;
701 	else if (strcmp(ciphername, "AES-256-GCM") == 0 ||
702 	    strcmp(ciphername, "AES-192-GCM") == 0 ||
703 	    strcmp(ciphername, "AES-128-GCM") == 0)
704 		cipher = OVPN_CIPHER_ALG_AES_GCM;
705 	else if (strcmp(ciphername, "CHACHA20-POLY1305") == 0)
706 		cipher = OVPN_CIPHER_ALG_CHACHA20_POLY1305;
707 	else
708 		return (EINVAL);
709 
710 	if (cipher != OVPN_CIPHER_ALG_NONE) {
711 		if (! nvlist_exists_binary(nvl, "key"))
712 			return (EINVAL);
713 		key = nvlist_get_binary(nvl, "key", &keylen);
714 		if (keylen > sizeof(kdir->key))
715 			return (E2BIG);
716 
717 		if (! nvlist_exists_binary(nvl, "iv"))
718 			return (EINVAL);
719 		iv = nvlist_get_binary(nvl, "iv", &ivlen);
720 		if (ivlen != 8)
721 			return (E2BIG);
722 	}
723 
724 	kdir = malloc(sizeof(struct ovpn_kkey_dir), M_OVPN,
725 	    M_WAITOK | M_ZERO);
726 
727 	kdir->cipher = cipher;
728 	kdir->keylen = keylen;
729 	memcpy(kdir->key, key, keylen);
730 	kdir->noncelen = ivlen;
731 	memcpy(kdir->nonce, iv, ivlen);
732 
733 	if (kdir->cipher != OVPN_CIPHER_ALG_NONE) {
734 		/* Crypto init */
735 		bzero(&csp, sizeof(csp));
736 		csp.csp_mode = CSP_MODE_AEAD;
737 
738 		if (kdir->cipher == OVPN_CIPHER_ALG_CHACHA20_POLY1305)
739 			csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305;
740 		else
741 			csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
742 
743 		csp.csp_flags |= CSP_F_SEPARATE_AAD;
744 
745 		csp.csp_cipher_klen = kdir->keylen;
746 		csp.csp_cipher_key = kdir->key;
747 		csp.csp_ivlen = 96 / 8;
748 
749 		error = crypto_newsession(&kdir->cryptoid, &csp,
750 		    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE);
751 		if (error) {
752 			free(kdir, M_OVPN);
753 			return (error);
754 		}
755 	}
756 
757 	mtx_init(&kdir->replay_mtx, "if_ovpn rx replay", NULL, MTX_DEF);
758 	*kdirp = kdir;
759 
760 	return (0);
761 }
762 
763 static void
764 ovpn_free_kkey_dir(struct ovpn_kkey_dir *kdir)
765 {
766 	if (kdir == NULL)
767 		return;
768 
769 	mtx_destroy(&kdir->replay_mtx);
770 
771 	crypto_freesession(kdir->cryptoid);
772 	free(kdir, M_OVPN);
773 }
774 
775 static int
776 ovpn_set_key(struct ifnet *ifp, const nvlist_t *nvl)
777 {
778 	struct ovpn_softc *sc = ifp->if_softc;
779 	struct ovpn_kkey_dir *enc, *dec;
780 	struct ovpn_kpeer *peer;
781 	int slot, keyid, peerid;
782 	int error;
783 
784 	if (nvl == NULL)
785 		return (EINVAL);
786 
787 	if (! nvlist_exists_number(nvl, "slot"))
788 		return (EINVAL);
789 	slot = nvlist_get_number(nvl, "slot");
790 
791 	if (! nvlist_exists_number(nvl, "keyid"))
792 		return (EINVAL);
793 	keyid = nvlist_get_number(nvl, "keyid");
794 
795 	if (! nvlist_exists_number(nvl, "peerid"))
796 		return (EINVAL);
797 	peerid = nvlist_get_number(nvl, "peerid");
798 
799 	if (slot != OVPN_KEY_SLOT_PRIMARY &&
800 	    slot != OVPN_KEY_SLOT_SECONDARY)
801 		return (EINVAL);
802 
803 	if (! nvlist_exists_nvlist(nvl, "encrypt") ||
804 	    ! nvlist_exists_nvlist(nvl, "decrypt"))
805 		return (EINVAL);
806 
807 	error = ovpn_create_kkey_dir(&enc, nvlist_get_nvlist(nvl, "encrypt"));
808 	if (error)
809 		return (error);
810 
811 	error = ovpn_create_kkey_dir(&dec, nvlist_get_nvlist(nvl, "decrypt"));
812 	if (error) {
813 		ovpn_free_kkey_dir(enc);
814 		return (error);
815 	}
816 
817 	OVPN_WLOCK(sc);
818 
819 	peer = ovpn_find_peer(sc, peerid);
820 	if (peer == NULL) {
821 		ovpn_free_kkey_dir(dec);
822 		ovpn_free_kkey_dir(enc);
823 		OVPN_WUNLOCK(sc);
824 		return (ENOENT);
825 	}
826 
827 	ovpn_free_kkey_dir(peer->keys[slot].encrypt);
828 	ovpn_free_kkey_dir(peer->keys[slot].decrypt);
829 
830 	peer->keys[slot].encrypt = enc;
831 	peer->keys[slot].decrypt = dec;
832 
833 	peer->keys[slot].keyid = keyid;
834 	peer->keys[slot].peerid = peerid;
835 
836 	OVPN_WUNLOCK(sc);
837 
838 	return (0);
839 }
840 
841 static int
842 ovpn_check_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, enum ovpn_key_slot slot)
843 {
844 	OVPN_ASSERT(sc);
845 
846 	if (peer->keys[slot].encrypt == NULL)
847 		return (ENOLINK);
848 
849 	if (peer->keys[slot].decrypt == NULL)
850 		return (ENOLINK);
851 
852 	return (0);
853 }
854 
855 static int
856 ovpn_start(struct ifnet *ifp)
857 {
858 	struct ovpn_softc *sc = ifp->if_softc;
859 
860 	OVPN_WLOCK(sc);
861 
862 	ifp->if_flags |= IFF_UP;
863 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
864 	if_link_state_change(ifp, LINK_STATE_UP);
865 
866 	OVPN_WUNLOCK(sc);
867 
868 	return (0);
869 }
870 
871 static int
872 ovpn_swap_keys(struct ifnet *ifp, nvlist_t *nvl)
873 {
874 	struct ovpn_softc *sc = ifp->if_softc;
875 	struct ovpn_kpeer *peer;
876 	struct ovpn_kkey tmpkey;
877 	int error;
878 
879 	if (nvl == NULL)
880 		return (EINVAL);
881 
882 	if (! nvlist_exists_number(nvl, "peerid"))
883 		return (EINVAL);
884 
885 	OVPN_WLOCK(sc);
886 
887 	peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid"));
888 	if (peer == NULL) {
889 		OVPN_WUNLOCK(sc);
890 		return (ENOENT);
891 	}
892 
893 	/* Check that we have a second key to swap to. */
894 	error = ovpn_check_key(sc, peer, OVPN_KEY_SLOT_SECONDARY);
895 	if (error) {
896 		OVPN_WUNLOCK(sc);
897 		return (error);
898 	}
899 
900 	tmpkey = peer->keys[0];
901 	peer->keys[0] = peer->keys[1];
902 	peer->keys[1] = tmpkey;
903 
904 	OVPN_WUNLOCK(sc);
905 
906 	return (0);
907 }
908 
909 static int
910 ovpn_del_key(struct ifnet *ifp, const nvlist_t *nvl)
911 {
912 	enum ovpn_key_slot slot;
913 	struct ovpn_kpeer *peer;
914 	struct ovpn_softc *sc = ifp->if_softc;
915 
916 	if (nvl == NULL)
917 		return (EINVAL);
918 
919 	if (! nvlist_exists_number(nvl, "peerid"))
920 		return (EINVAL);
921 
922 	if (! nvlist_exists_number(nvl, "slot"))
923 		return (EINVAL);
924 	slot = nvlist_get_number(nvl, "slot");
925 
926 	if (slot != OVPN_KEY_SLOT_PRIMARY &&
927 	    slot != OVPN_KEY_SLOT_SECONDARY)
928 		return (EINVAL);
929 
930 	OVPN_WLOCK(sc);
931 
932 	peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid"));
933 	if (peer == NULL) {
934 		OVPN_WUNLOCK(sc);
935 		return (ENOENT);
936 	}
937 
938 	ovpn_free_kkey_dir(peer->keys[slot].encrypt);
939 	ovpn_free_kkey_dir(peer->keys[slot].decrypt);
940 
941 	peer->keys[slot].encrypt = NULL;
942 	peer->keys[slot].decrypt = NULL;
943 
944 	peer->keys[slot].keyid = 0;
945 	peer->keys[slot].peerid = 0;
946 
947 	OVPN_WUNLOCK(sc);
948 
949 	return (0);
950 }
951 
952 static int
953 ovpn_send_pkt(struct ifnet *ifp, const nvlist_t *nvl)
954 {
955 	struct epoch_tracker et;
956 	struct ovpn_softc *sc = ifp->if_softc;
957 	struct mbuf *m;
958 	const uint8_t *pkt;
959 	size_t pktlen;
960 	uint32_t peerid;
961 	int ret;
962 
963 	if (nvl == NULL)
964 		return (EINVAL);
965 
966 	if (! nvlist_exists_binary(nvl, "packet"))
967 		return (EINVAL);
968 	pkt = nvlist_get_binary(nvl, "packet", &pktlen);
969 
970 	if (! nvlist_exists_number(nvl, "peerid"))
971 		return (EINVAL);
972 
973 	peerid = nvlist_get_number(nvl, "peerid");
974 
975 	/*
976 	 * Check that userspace isn't giving us a data packet. That might lead
977 	 * to IV re-use, which would be bad.
978 	 */
979 	if ((pkt[0] >> OVPN_OP_SHIFT) == OVPN_OP_DATA_V2)
980 		return (EINVAL);
981 
982 	m = m_get2(pktlen, M_WAITOK, MT_DATA, M_PKTHDR);
983 	if (m == NULL)
984 		return (ENOMEM);
985 
986 	m->m_len = m->m_pkthdr.len = pktlen;
987 	m_copyback(m, 0, m->m_len, pkt);
988 
989 	/* Now prepend IP/UDP headers and transmit the mbuf. */
990 	NET_EPOCH_ENTER(et);
991 	ret = ovpn_encap(sc, peerid, m);
992 	NET_EPOCH_EXIT(et);
993 	if (ret == 0)
994 		OVPN_COUNTER_ADD(sc, sent_ctrl_pkts, 1);
995 	else
996 		OVPN_COUNTER_ADD(sc, lost_ctrl_pkts_out, 1);
997 
998 	return (ret);
999 }
1000 
1001 static void
1002 ovpn_send_ping(void *arg)
1003 {
1004 	static const uint8_t ping_str[] = {
1005 		0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb,
1006 		0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48
1007 	};
1008 
1009 	struct epoch_tracker et;
1010 	struct ovpn_kpeer *peer = arg;
1011 	struct ovpn_softc *sc = peer->sc;
1012 	struct mbuf *m;
1013 
1014 	OVPN_RASSERT(sc);
1015 
1016 	/* Ensure we repeat! */
1017 	callout_reset(&peer->ping_send, peer->keepalive.interval * hz,
1018 	    ovpn_send_ping, peer);
1019 
1020 	m = m_get2(sizeof(ping_str), M_NOWAIT, MT_DATA, M_PKTHDR);
1021 	if (m == NULL)
1022 		return;
1023 
1024 	m_copyback(m, 0, sizeof(ping_str), ping_str);
1025 	m->m_len = m->m_pkthdr.len = sizeof(ping_str);
1026 
1027 	CURVNET_SET(sc->ifp->if_vnet);
1028 	NET_EPOCH_ENTER(et);
1029 	(void)ovpn_transmit_to_peer(sc->ifp, m, peer, NULL);
1030 	NET_EPOCH_EXIT(et);
1031 	CURVNET_RESTORE();
1032 }
1033 
1034 static void
1035 ovpn_timeout(void *arg)
1036 {
1037 	struct ovpn_kpeer *peer = arg;
1038 	struct ovpn_softc *sc = peer->sc;
1039 	uint32_t last, _last_active;
1040 	int ret __diagused;
1041 	int cpu;
1042 
1043 	OVPN_WASSERT(sc);
1044 
1045 	last = 0;
1046 	CPU_FOREACH(cpu) {
1047 		_last_active = *zpcpu_get_cpu(peer->last_active, cpu);
1048 		if (_last_active > last)
1049 			last = _last_active;
1050 	}
1051 
1052 	if (last + peer->keepalive.timeout > time_uptime) {
1053 		callout_reset(&peer->ping_rcv,
1054 		    (peer->keepalive.timeout - (time_uptime - last)) * hz,
1055 		    ovpn_timeout, peer);
1056 		return;
1057 	}
1058 
1059 	CURVNET_SET(sc->ifp->if_vnet);
1060 	ret = _ovpn_del_peer(sc, peer->peerid);
1061 	MPASS(ret == 0);
1062 	CURVNET_RESTORE();
1063 }
1064 
1065 static int
1066 ovpn_set_peer(struct ifnet *ifp, const nvlist_t *nvl)
1067 {
1068 	struct ovpn_softc *sc = ifp->if_softc;
1069 	struct ovpn_kpeer *peer;
1070 
1071 	if (nvl == NULL)
1072 		return (EINVAL);
1073 
1074 	if (! nvlist_exists_number(nvl, "interval") ||
1075 	    ! nvlist_exists_number(nvl, "timeout") ||
1076 	    ! nvlist_exists_number(nvl, "peerid"))
1077 		return (EINVAL);
1078 
1079 	OVPN_WLOCK(sc);
1080 
1081 	peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid"));
1082 	if (peer == NULL) {
1083 		OVPN_WUNLOCK(sc);
1084 		return (ENOENT);
1085 	}
1086 
1087 	peer->keepalive.interval = nvlist_get_number(nvl, "interval");
1088 	peer->keepalive.timeout = nvlist_get_number(nvl, "timeout");
1089 
1090 	if (peer->keepalive.interval > 0)
1091 		callout_reset(&peer->ping_send, peer->keepalive.interval * hz,
1092 		    ovpn_send_ping, peer);
1093 	if (peer->keepalive.timeout > 0)
1094 		callout_reset(&peer->ping_rcv, peer->keepalive.timeout * hz,
1095 		    ovpn_timeout, peer);
1096 
1097 	OVPN_WUNLOCK(sc);
1098 
1099 	return (0);
1100 }
1101 
1102 static int
1103 ovpn_set_ifmode(struct ifnet *ifp, const nvlist_t *nvl)
1104 {
1105 	struct ovpn_softc *sc = ifp->if_softc;
1106 	int ifmode;
1107 
1108 	if (nvl == NULL)
1109 		return (EINVAL);
1110 
1111 	if (! nvlist_exists_number(nvl, "ifmode") )
1112 		return (EINVAL);
1113 
1114 	ifmode = nvlist_get_number(nvl, "ifmode");
1115 
1116 	OVPN_WLOCK(sc);
1117 
1118 	/* deny this if UP */
1119 	if (ifp->if_flags & IFF_UP) {
1120 		OVPN_WUNLOCK(sc);
1121 		return (EBUSY);
1122 	}
1123 
1124 	switch (ifmode & ~IFF_MULTICAST) {
1125 	case IFF_POINTOPOINT:
1126 	case IFF_BROADCAST:
1127 		ifp->if_flags &=
1128 		    ~(IFF_BROADCAST|IFF_POINTOPOINT|IFF_MULTICAST);
1129 		ifp->if_flags |= ifmode;
1130 		break;
1131 	default:
1132 		OVPN_WUNLOCK(sc);
1133 		return (EINVAL);
1134 	}
1135 
1136 	OVPN_WUNLOCK(sc);
1137 
1138 	return (0);
1139 }
1140 
1141 static int
1142 ovpn_ioctl_set(struct ifnet *ifp, struct ifdrv *ifd)
1143 {
1144 	struct ovpn_softc *sc = ifp->if_softc;
1145 	uint8_t *buf = NULL;
1146 	nvlist_t *nvl = NULL;
1147 	int ret;
1148 
1149 	if (ifd->ifd_len != 0) {
1150 		if (ifd->ifd_len > OVPN_MAX_REQUEST_SIZE)
1151 			return (E2BIG);
1152 
1153 		buf = malloc(ifd->ifd_len, M_OVPN, M_WAITOK);
1154 
1155 		ret = copyin(ifd->ifd_data, buf, ifd->ifd_len);
1156 		if (ret != 0) {
1157 			free(buf, M_OVPN);
1158 			return (ret);
1159 		}
1160 
1161 		nvl = nvlist_unpack(buf, ifd->ifd_len, 0);
1162 		free(buf, M_OVPN);
1163 		if (nvl == NULL) {
1164 			return (EINVAL);
1165 		}
1166 	}
1167 
1168 	switch (ifd->ifd_cmd) {
1169 	case OVPN_NEW_PEER:
1170 		ret = ovpn_new_peer(ifp, nvl);
1171 		break;
1172 	case OVPN_DEL_PEER:
1173 		OVPN_WLOCK(sc);
1174 		ret = ovpn_del_peer(ifp, nvl);
1175 		OVPN_WUNLOCK(sc);
1176 		break;
1177 	case OVPN_NEW_KEY:
1178 		ret = ovpn_set_key(ifp, nvl);
1179 		break;
1180 	case OVPN_START_VPN:
1181 		ret = ovpn_start(ifp);
1182 		break;
1183 	case OVPN_SWAP_KEYS:
1184 		ret = ovpn_swap_keys(ifp, nvl);
1185 		break;
1186 	case OVPN_DEL_KEY:
1187 		ret = ovpn_del_key(ifp, nvl);
1188 		break;
1189 	case OVPN_SEND_PKT:
1190 		ret = ovpn_send_pkt(ifp, nvl);
1191 		break;
1192 	case OVPN_SET_PEER:
1193 		ret = ovpn_set_peer(ifp, nvl);
1194 		break;
1195 	case OVPN_SET_IFMODE:
1196 		ret = ovpn_set_ifmode(ifp, nvl);
1197 		break;
1198 	default:
1199 		ret = ENOTSUP;
1200 	}
1201 
1202 	nvlist_destroy(nvl);
1203 	return (ret);
1204 }
1205 
1206 static int
1207 ovpn_add_counters(nvlist_t *parent, const char *name, counter_u64_t in,
1208     counter_u64_t out)
1209 {
1210 	nvlist_t *nvl;
1211 
1212 	nvl = nvlist_create(0);
1213 	if (nvl == NULL)
1214 		return (ENOMEM);
1215 
1216 	nvlist_add_number(nvl, "in", counter_u64_fetch(in));
1217 	nvlist_add_number(nvl, "out", counter_u64_fetch(out));
1218 
1219 	nvlist_add_nvlist(parent, name, nvl);
1220 
1221 	nvlist_destroy(nvl);
1222 
1223 	return (0);
1224 }
1225 
1226 static int
1227 ovpn_get_stats(struct ovpn_softc *sc, nvlist_t **onvl)
1228 {
1229 	nvlist_t *nvl;
1230 	int ret;
1231 
1232 	nvl = nvlist_create(0);
1233 	if (nvl == NULL)
1234 		return (ENOMEM);
1235 
1236 #define OVPN_COUNTER_OUT(name, in, out) \
1237 	do { \
1238 		ret = ovpn_add_counters(nvl, name, \
1239 		    sc->counters[offsetof(struct ovpn_counters, in) / \
1240 		    sizeof(uint64_t)], \
1241 		    sc->counters[offsetof(struct ovpn_counters, out) / \
1242 		    sizeof(uint64_t)]); \
1243 		if (ret != 0) \
1244 			goto error; \
1245 	} while(0)
1246 
1247 	OVPN_COUNTER_OUT("lost_ctrl", lost_ctrl_pkts_in, lost_ctrl_pkts_out);
1248 	OVPN_COUNTER_OUT("lost_data", lost_data_pkts_in, lost_data_pkts_out);
1249 	OVPN_COUNTER_OUT("nomem_data", nomem_data_pkts_in,
1250 	    nomem_data_pkts_out);
1251 	OVPN_COUNTER_OUT("data", received_data_pkts, sent_data_pkts);
1252 	OVPN_COUNTER_OUT("ctrl", received_ctrl_pkts, sent_ctrl_pkts);
1253 	OVPN_COUNTER_OUT("tunnel", tunnel_bytes_received,
1254 	    tunnel_bytes_received);
1255 	OVPN_COUNTER_OUT("transport", transport_bytes_received,
1256 	    transport_bytes_received);
1257 #undef OVPN_COUNTER_OUT
1258 
1259 	*onvl = nvl;
1260 
1261 	return (0);
1262 
1263 error:
1264 	nvlist_destroy(nvl);
1265 	return (ret);
1266 }
1267 
1268 static int
1269 ovpn_poll_pkt(struct ovpn_softc *sc, nvlist_t **onvl)
1270 {
1271 	nvlist_t *nvl;
1272 
1273 	nvl = nvlist_create(0);
1274 	if (nvl == NULL)
1275 		return (ENOMEM);
1276 
1277 	nvlist_add_number(nvl, "pending", buf_ring_count(sc->notifring));
1278 
1279 	*onvl = nvl;
1280 
1281 	return (0);
1282 }
1283 
1284 static int
1285 opvn_get_pkt(struct ovpn_softc *sc, nvlist_t **onvl)
1286 {
1287 	struct ovpn_notification *n;
1288 	nvlist_t *nvl;
1289 
1290 	/* Check if we have notifications pending. */
1291 	n = buf_ring_dequeue_mc(sc->notifring);
1292 	if (n == NULL)
1293 		return (ENOENT);
1294 
1295 	nvl = nvlist_create(0);
1296 	if (nvl == NULL) {
1297 		free(n, M_OVPN);
1298 		return (ENOMEM);
1299 	}
1300 	nvlist_add_number(nvl, "peerid", n->peerid);
1301 	nvlist_add_number(nvl, "notification", n->type);
1302 	free(n, M_OVPN);
1303 
1304 	*onvl = nvl;
1305 
1306 	return (0);
1307 }
1308 
1309 static int
1310 ovpn_ioctl_get(struct ifnet *ifp, struct ifdrv *ifd)
1311 {
1312 	struct ovpn_softc *sc = ifp->if_softc;
1313 	nvlist_t *nvl = NULL;
1314 	int error;
1315 
1316 	switch (ifd->ifd_cmd) {
1317 	case OVPN_GET_STATS:
1318 		error = ovpn_get_stats(sc, &nvl);
1319 		break;
1320 	case OVPN_POLL_PKT:
1321 		error = ovpn_poll_pkt(sc, &nvl);
1322 		break;
1323 	case OVPN_GET_PKT:
1324 		error = opvn_get_pkt(sc, &nvl);
1325 		break;
1326 	default:
1327 		error = ENOTSUP;
1328 		break;
1329 	}
1330 
1331 	if (error == 0) {
1332 		void *packed = NULL;
1333 		size_t len;
1334 
1335 		MPASS(nvl != NULL);
1336 
1337 		packed = nvlist_pack(nvl, &len);
1338 		if (! packed) {
1339 			nvlist_destroy(nvl);
1340 			return (ENOMEM);
1341 		}
1342 
1343 		if (len > ifd->ifd_len) {
1344 			free(packed, M_NVLIST);
1345 			nvlist_destroy(nvl);
1346 			return (ENOSPC);
1347 		}
1348 
1349 		error = copyout(packed, ifd->ifd_data, len);
1350 		ifd->ifd_len = len;
1351 
1352 		free(packed, M_NVLIST);
1353 		nvlist_destroy(nvl);
1354 	}
1355 
1356 	return (error);
1357 }
1358 
1359 static int
1360 ovpn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
1361 {
1362 	struct ifdrv *ifd;
1363 	int error;
1364 
1365 	switch (cmd) {
1366 	case SIOCSDRVSPEC:
1367 	case SIOCGDRVSPEC:
1368 		error = priv_check(curthread, PRIV_NET_OVPN);
1369 		if (error)
1370 			return (error);
1371 		break;
1372 	}
1373 
1374 	switch (cmd) {
1375 	case SIOCSDRVSPEC:
1376 		ifd = (struct ifdrv *)data;
1377 		error = ovpn_ioctl_set(ifp, ifd);
1378 		break;
1379 	case SIOCGDRVSPEC:
1380 		ifd = (struct ifdrv *)data;
1381 		error = ovpn_ioctl_get(ifp, ifd);
1382 		break;
1383 	case SIOCSIFMTU: {
1384 		struct ifreq *ifr = (struct ifreq *)data;
1385 		if (ifr->ifr_mtu < OVPN_MTU_MIN || ifr->ifr_mtu > OVPN_MTU_MAX)
1386 			return (EINVAL);
1387 
1388 		ifp->if_mtu = ifr->ifr_mtu;
1389 		return (0);
1390 	}
1391 	case SIOCSIFADDR:
1392 	case SIOCADDMULTI:
1393 	case SIOCDELMULTI:
1394 	case SIOCGIFMTU:
1395 	case SIOCSIFFLAGS:
1396 		return (0);
1397 	default:
1398 		error = EINVAL;
1399 	}
1400 
1401 	return (error);
1402 }
1403 
1404 static int
1405 ovpn_encrypt_tx_cb(struct cryptop *crp)
1406 {
1407 	struct epoch_tracker et;
1408 	struct ovpn_kpeer *peer = crp->crp_opaque;
1409 	struct ovpn_softc *sc = peer->sc;
1410 	struct mbuf *m = crp->crp_buf.cb_mbuf;
1411 	int tunnel_len;
1412 	int ret;
1413 
1414 	CURVNET_SET(sc->ifp->if_vnet);
1415 	NET_EPOCH_ENTER(et);
1416 
1417 	if (crp->crp_etype != 0) {
1418 		crypto_freereq(crp);
1419 		ovpn_peer_release_ref(peer, false);
1420 		NET_EPOCH_EXIT(et);
1421 		CURVNET_RESTORE();
1422 		OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
1423 		m_freem(m);
1424 		return (0);
1425 	}
1426 
1427 	MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF);
1428 
1429 	tunnel_len = m->m_pkthdr.len - sizeof(struct ovpn_wire_header);
1430 	ret = ovpn_encap(sc, peer->peerid, m);
1431 	if (ret == 0) {
1432 		OVPN_COUNTER_ADD(sc, sent_data_pkts, 1);
1433 		OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len);
1434 	}
1435 
1436 	crypto_freereq(crp);
1437 	ovpn_peer_release_ref(peer, false);
1438 
1439 	NET_EPOCH_EXIT(et);
1440 	CURVNET_RESTORE();
1441 
1442 	return (0);
1443 }
1444 
1445 static void
1446 ovpn_finish_rx(struct ovpn_softc *sc, struct mbuf *m,
1447     struct ovpn_kpeer *peer, struct ovpn_kkey *key, uint32_t seq,
1448     struct rm_priotracker *_ovpn_lock_trackerp)
1449 {
1450 	uint32_t af;
1451 
1452 	OVPN_RASSERT(sc);
1453 	NET_EPOCH_ASSERT();
1454 
1455 	/* Replay protection. */
1456 	if (V_replay_protection && ! ovpn_check_replay(key->decrypt, seq)) {
1457 		OVPN_RUNLOCK(sc);
1458 		OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1459 		m_freem(m);
1460 		return;
1461 	}
1462 
1463 	critical_enter();
1464 	*zpcpu_get(peer->last_active) = time_uptime;
1465 	critical_exit();
1466 
1467 	OVPN_RUNLOCK(sc);
1468 
1469 	OVPN_COUNTER_ADD(sc, received_data_pkts, 1);
1470 	OVPN_COUNTER_ADD(sc, tunnel_bytes_received, m->m_pkthdr.len);
1471 
1472 	/* Receive the packet on our interface. */
1473 	m->m_pkthdr.rcvif = sc->ifp;
1474 
1475 	/* Clear checksum flags in case the real hardware set them. */
1476 	m->m_pkthdr.csum_flags = 0;
1477 
1478 	/* Ensure we can read the first byte. */
1479 	m = m_pullup(m, 1);
1480 	if (m == NULL) {
1481 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1);
1482 		return;
1483 	}
1484 
1485 	/*
1486 	 * Check for address family, and disregard any control packets (e.g.
1487 	 * keepalive).
1488 	 */
1489 	af = ovpn_get_af(m);
1490 	if (af != 0) {
1491 		BPF_MTAP2(sc->ifp, &af, sizeof(af), m);
1492 		if (V_async_netisr_queue)
1493 			netisr_queue(af == AF_INET ? NETISR_IP : NETISR_IPV6, m);
1494 		else
1495 			netisr_dispatch(af == AF_INET ? NETISR_IP : NETISR_IPV6, m);
1496 	} else {
1497 		OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1498 		m_freem(m);
1499 	}
1500 }
1501 
1502 static struct ovpn_kkey *
1503 ovpn_find_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer,
1504     const struct ovpn_wire_header *ohdr)
1505 {
1506 	struct ovpn_kkey *key = NULL;
1507 	uint8_t keyid;
1508 
1509 	OVPN_RASSERT(sc);
1510 
1511 	keyid = (ntohl(ohdr->opcode) >> 24) & 0x07;
1512 
1513 	if (peer->keys[0].keyid == keyid)
1514 		key = &peer->keys[0];
1515 	else if (peer->keys[1].keyid == keyid)
1516 		key = &peer->keys[1];
1517 
1518 	return (key);
1519 }
1520 
1521 static int
1522 ovpn_decrypt_rx_cb(struct cryptop *crp)
1523 {
1524 	struct epoch_tracker et;
1525 	struct ovpn_softc *sc = crp->crp_opaque;
1526 	struct mbuf *m = crp->crp_buf.cb_mbuf;
1527 	struct ovpn_kkey *key;
1528 	struct ovpn_kpeer *peer;
1529 	struct ovpn_wire_header *ohdr;
1530 	uint32_t peerid;
1531 
1532 	OVPN_RLOCK_TRACKER;
1533 
1534 	OVPN_RLOCK(sc);
1535 
1536 	MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF);
1537 
1538 	if (crp->crp_etype != 0) {
1539 		crypto_freereq(crp);
1540 		atomic_add_int(&sc->refcount, -1);
1541 		OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1542 		OVPN_RUNLOCK(sc);
1543 		m_freem(m);
1544 		return (0);
1545 	}
1546 
1547 	CURVNET_SET(sc->ifp->if_vnet);
1548 
1549 	ohdr = mtodo(m, sizeof(struct udphdr));
1550 
1551 	peerid = ntohl(ohdr->opcode) & 0x00ffffff;
1552 	peer = ovpn_find_peer(sc, peerid);
1553 	if (peer == NULL) {
1554 		/* No such peer. Drop packet. */
1555 		crypto_freereq(crp);
1556 		atomic_add_int(&sc->refcount, -1);
1557 		OVPN_RUNLOCK(sc);
1558 		OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1559 		m_freem(m);
1560 		CURVNET_RESTORE();
1561 		return (0);
1562 	}
1563 
1564 	key = ovpn_find_key(sc, peer, ohdr);
1565 	if (key == NULL) {
1566 		crypto_freereq(crp);
1567 		atomic_add_int(&sc->refcount, -1);
1568 		/*
1569 		 * Has this key been removed between us starting the decrypt
1570 		 * and finishing it?
1571 		 */
1572 		OVPN_RUNLOCK(sc);
1573 		OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
1574 		m_freem(m);
1575 		CURVNET_RESTORE();
1576 		return (0);
1577 	}
1578 
1579 	/* Now remove the outer headers */
1580 	m_adj_decap(m, sizeof(struct udphdr) +
1581 	    sizeof(struct ovpn_wire_header));
1582 
1583 	NET_EPOCH_ENTER(et);
1584 	ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), _ovpn_lock_trackerp);
1585 	NET_EPOCH_EXIT(et);
1586 	OVPN_UNLOCK_ASSERT(sc);
1587 
1588 	CURVNET_RESTORE();
1589 
1590 	crypto_freereq(crp);
1591 	atomic_add_int(&sc->refcount, -1);
1592 
1593 	return (0);
1594 }
1595 
1596 static int
1597 ovpn_get_af(struct mbuf *m)
1598 {
1599 	struct ip *ip;
1600 	struct ip6_hdr *ip6;
1601 
1602 	/*
1603 	 * We should pullup, but we're only interested in the first byte, so
1604 	 * that'll always be contiguous.
1605 	 */
1606 	ip = mtod(m, struct ip *);
1607 	if (ip->ip_v == IPVERSION)
1608 		return (AF_INET);
1609 
1610 	ip6 = mtod(m, struct ip6_hdr *);
1611 	if ((ip6->ip6_vfc & IPV6_VERSION_MASK) == IPV6_VERSION)
1612 		return (AF_INET6);
1613 
1614 	return (0);
1615 }
1616 
1617 #ifdef INET
1618 static struct ovpn_kpeer *
1619 ovpn_find_peer_by_ip(struct ovpn_softc *sc, const struct in_addr addr)
1620 {
1621 	struct ovpn_kpeer *peer = NULL;
1622 
1623 	OVPN_ASSERT(sc);
1624 
1625 	for (int i = 0; i < OVPN_MAX_PEERS; i++) {
1626 		if (sc->peers[i] == NULL)
1627 			continue;
1628 		if (addr.s_addr == sc->peers[i]->vpn4.s_addr) {
1629 			peer = sc->peers[i];
1630 			break;
1631 		}
1632 	}
1633 
1634 	return (peer);
1635 }
1636 #endif
1637 
1638 #ifdef INET6
1639 static struct ovpn_kpeer *
1640 ovpn_find_peer_by_ip6(struct ovpn_softc *sc, const struct in6_addr *addr)
1641 {
1642 	struct ovpn_kpeer *peer = NULL;
1643 
1644 	OVPN_ASSERT(sc);
1645 
1646 	for (int i = 0; i < OVPN_MAX_PEERS; i++) {
1647 		if (sc->peers[i] == NULL)
1648 			continue;
1649 		if (memcmp(addr, &sc->peers[i]->vpn6, sizeof(*addr)) == 0) {
1650 			peer = sc->peers[i];
1651 			break;
1652 		}
1653 	}
1654 
1655 	return (peer);
1656 }
1657 #endif
1658 
1659 static struct ovpn_kpeer *
1660 ovpn_route_peer(struct ovpn_softc *sc, struct mbuf **m0,
1661     const struct sockaddr *dst)
1662 {
1663 	struct ovpn_kpeer *peer = NULL;
1664 	int af;
1665 
1666 	NET_EPOCH_ASSERT();
1667 	OVPN_ASSERT(sc);
1668 
1669 	/* Shortcut if we're a client (or are a server and have only one client). */
1670 	if (sc->peercount == 1)
1671 		return (ovpn_find_only_peer(sc));
1672 
1673 	if (dst != NULL)
1674 		af = dst->sa_family;
1675 	else
1676 		af = ovpn_get_af(*m0);
1677 
1678 	switch (af) {
1679 #ifdef INET
1680 	case AF_INET: {
1681 		const struct sockaddr_in *sa = (const struct sockaddr_in *)dst;
1682 		struct nhop_object *nh;
1683 		const struct in_addr *ip_dst;
1684 
1685 		if (sa != NULL) {
1686 			ip_dst = &sa->sin_addr;
1687 		} else {
1688 			struct ip *ip;
1689 
1690 			*m0 = m_pullup(*m0, sizeof(struct ip));
1691 			if (*m0 == NULL)
1692 				return (NULL);
1693 			ip = mtod(*m0, struct ip *);
1694 			ip_dst = &ip->ip_dst;
1695 		}
1696 
1697 		peer = ovpn_find_peer_by_ip(sc, *ip_dst);
1698 		SDT_PROBE2(if_ovpn, tx, route, ip4, ip_dst, peer);
1699 		if (peer == NULL) {
1700 			nh = fib4_lookup(M_GETFIB(*m0), *ip_dst, 0,
1701 			    NHR_NONE, 0);
1702 			if (nh && (nh->nh_flags & NHF_GATEWAY)) {
1703 				peer = ovpn_find_peer_by_ip(sc,
1704 				    nh->gw4_sa.sin_addr);
1705 				SDT_PROBE2(if_ovpn, tx, route, ip4,
1706 				    &nh->gw4_sa.sin_addr, peer);
1707 			}
1708 		}
1709 		break;
1710 	}
1711 #endif
1712 #ifdef INET6
1713 	case AF_INET6: {
1714 		const struct sockaddr_in6 *sa6 =
1715 		    (const struct sockaddr_in6 *)dst;
1716 		struct nhop_object *nh;
1717 		const struct in6_addr *ip6_dst;
1718 
1719 		if (sa6 != NULL) {
1720 			ip6_dst = &sa6->sin6_addr;
1721 		} else {
1722 			struct ip6_hdr *ip6;
1723 
1724 			*m0 = m_pullup(*m0, sizeof(struct ip6_hdr));
1725 			if (*m0 == NULL)
1726 				return (NULL);
1727 			ip6 = mtod(*m0, struct ip6_hdr *);
1728 			ip6_dst = &ip6->ip6_dst;
1729 		}
1730 
1731 		peer = ovpn_find_peer_by_ip6(sc, ip6_dst);
1732 		SDT_PROBE2(if_ovpn, tx, route, ip6, ip6_dst, peer);
1733 		if (peer == NULL) {
1734 			nh = fib6_lookup(M_GETFIB(*m0), ip6_dst, 0,
1735 			    NHR_NONE, 0);
1736 			if (nh && (nh->nh_flags & NHF_GATEWAY)) {
1737 				peer = ovpn_find_peer_by_ip6(sc,
1738 				    &nh->gw6_sa.sin6_addr);
1739 				SDT_PROBE2(if_ovpn, tx, route, ip6,
1740 				    &nh->gw6_sa.sin6_addr, peer);
1741 			}
1742 		}
1743 		break;
1744 	}
1745 #endif
1746 	}
1747 
1748 	return (peer);
1749 }
1750 
1751 static int
1752 ovpn_transmit(struct ifnet *ifp, struct mbuf *m)
1753 {
1754 	return (ifp->if_output(ifp, m, NULL, NULL));
1755 }
1756 
1757 static int
1758 ovpn_transmit_to_peer(struct ifnet *ifp, struct mbuf *m,
1759     struct ovpn_kpeer *peer, struct rm_priotracker *_ovpn_lock_trackerp)
1760 {
1761 	struct ovpn_wire_header *ohdr;
1762 	struct ovpn_kkey *key;
1763 	struct ovpn_softc *sc;
1764 	struct cryptop *crp;
1765 	uint32_t af, seq;
1766 	size_t len, ovpn_hdr_len;
1767 	int tunnel_len;
1768 	int ret;
1769 
1770 	sc = ifp->if_softc;
1771 
1772 	OVPN_RASSERT(sc);
1773 
1774 	tunnel_len = m->m_pkthdr.len;
1775 
1776 	key = &peer->keys[OVPN_KEY_SLOT_PRIMARY];
1777 	if (key->encrypt == NULL) {
1778 		if (_ovpn_lock_trackerp != NULL)
1779 			OVPN_RUNLOCK(sc);
1780 		m_freem(m);
1781 		return (ENOLINK);
1782 	}
1783 
1784 	af = ovpn_get_af(m);
1785 	/* Don't capture control packets. */
1786 	if (af != 0)
1787 		BPF_MTAP2(ifp, &af, sizeof(af), m);
1788 
1789 	len = m->m_pkthdr.len;
1790 	MPASS(len <= ifp->if_mtu);
1791 
1792 	ovpn_hdr_len = sizeof(struct ovpn_wire_header);
1793 	if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE)
1794 		ovpn_hdr_len -= 16; /* No auth tag. */
1795 
1796 	M_PREPEND(m, ovpn_hdr_len, M_NOWAIT);
1797 	if (m == NULL) {
1798 		if (_ovpn_lock_trackerp != NULL)
1799 			OVPN_RUNLOCK(sc);
1800 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1801 		return (ENOBUFS);
1802 	}
1803 	ohdr = mtod(m, struct ovpn_wire_header *);
1804 	ohdr->opcode = (OVPN_OP_DATA_V2 << OVPN_OP_SHIFT) | key->keyid;
1805 	ohdr->opcode <<= 24;
1806 	ohdr->opcode |= key->peerid;
1807 	ohdr->opcode = htonl(ohdr->opcode);
1808 
1809 	seq = atomic_fetchadd_32(&peer->tx_seq, 1);
1810 	seq = htonl(seq);
1811 	ohdr->seq = seq;
1812 
1813 	if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) {
1814 		ret = ovpn_encap(sc, peer->peerid, m);
1815 		if (_ovpn_lock_trackerp != NULL)
1816 			OVPN_RUNLOCK(sc);
1817 		if (ret == 0) {
1818 			OVPN_COUNTER_ADD(sc, sent_data_pkts, 1);
1819 			OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len);
1820 		}
1821 		return (ret);
1822 	}
1823 
1824 	crp = crypto_getreq(key->encrypt->cryptoid, M_NOWAIT);
1825 	if (crp == NULL) {
1826 		if (_ovpn_lock_trackerp != NULL)
1827 			OVPN_RUNLOCK(sc);
1828 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1829 		m_freem(m);
1830 		return (ENOBUFS);
1831 	}
1832 
1833 	/* Encryption covers only the payload, not the header. */
1834 	crp->crp_payload_start = sizeof(*ohdr);
1835 	crp->crp_payload_length = len;
1836 	crp->crp_op = CRYPTO_OP_ENCRYPT;
1837 
1838 	/*
1839 	 * AAD data covers the ovpn_wire_header minus the auth
1840 	 * tag.
1841 	 */
1842 	crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag);
1843 	crp->crp_aad = ohdr;
1844 	crp->crp_aad_start = 0;
1845 	crp->crp_op |= CRYPTO_OP_COMPUTE_DIGEST;
1846 	crp->crp_digest_start = offsetof(struct ovpn_wire_header, auth_tag);
1847 
1848 	crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
1849 	memcpy(crp->crp_iv, &seq, sizeof(seq));
1850 	memcpy(crp->crp_iv + sizeof(seq), key->encrypt->nonce,
1851 	    key->encrypt->noncelen);
1852 
1853 	crypto_use_mbuf(crp, m);
1854 	crp->crp_flags |= CRYPTO_F_CBIFSYNC;
1855 	crp->crp_callback = ovpn_encrypt_tx_cb;
1856 	crp->crp_opaque = peer;
1857 
1858 	atomic_add_int(&peer->refcount, 1);
1859 	if (_ovpn_lock_trackerp != NULL)
1860 		OVPN_RUNLOCK(sc);
1861 	if (V_async_crypto)
1862 		ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED);
1863 	else
1864 		ret = crypto_dispatch(crp);
1865 	if (ret) {
1866 		OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
1867 	}
1868 
1869 	return (ret);
1870 }
1871 
1872 /*
1873  * Note: Expects to hold the read lock on entry, and will release it itself.
1874  */
1875 static int
1876 ovpn_encap(struct ovpn_softc *sc, uint32_t peerid, struct mbuf *m)
1877 {
1878 	struct udphdr *udp;
1879 	struct ovpn_kpeer *peer;
1880 	int len;
1881 
1882 	OVPN_RLOCK_TRACKER;
1883 
1884 	OVPN_RLOCK(sc);
1885 	NET_EPOCH_ASSERT();
1886 
1887 	peer = ovpn_find_peer(sc, peerid);
1888 	if (peer == NULL || sc->ifp->if_link_state != LINK_STATE_UP) {
1889 		OVPN_RUNLOCK(sc);
1890 		OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
1891 		m_freem(m);
1892 		return (ENETDOWN);
1893 	}
1894 
1895 	len = m->m_pkthdr.len;
1896 
1897 	M_PREPEND(m, sizeof(struct udphdr), M_NOWAIT);
1898 	if (m == NULL) {
1899 		OVPN_RUNLOCK(sc);
1900 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1901 		m_freem(m);
1902 		return (ENOBUFS);
1903 	}
1904 	udp = mtod(m, struct udphdr *);
1905 
1906 	MPASS(peer->local.ss_family == peer->remote.ss_family);
1907 
1908 	udp->uh_sport = ovpn_get_port(&peer->local);
1909 	udp->uh_dport = ovpn_get_port(&peer->remote);
1910 	udp->uh_ulen = htons(sizeof(struct udphdr) + len);
1911 
1912 	switch (peer->remote.ss_family) {
1913 #ifdef INET
1914 	case AF_INET: {
1915 		struct sockaddr_in *in_local = TO_IN(&peer->local);
1916 		struct sockaddr_in *in_remote = TO_IN(&peer->remote);
1917 		struct ip *ip;
1918 
1919 		/*
1920 		 * This requires knowing the source IP, which we don't. Happily
1921 		 * we're allowed to keep this at 0, and the checksum won't do
1922 		 * anything the crypto won't already do.
1923 		 */
1924 		udp->uh_sum = 0;
1925 
1926 		/* Set the checksum flags so we recalculate checksums. */
1927 		m->m_pkthdr.csum_flags |= CSUM_IP;
1928 		m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
1929 
1930 		M_PREPEND(m, sizeof(struct ip), M_NOWAIT);
1931 		if (m == NULL) {
1932 			OVPN_RUNLOCK(sc);
1933 			OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1934 			return (ENOBUFS);
1935 		}
1936 		ip = mtod(m, struct ip *);
1937 
1938 		ip->ip_tos = 0;
1939 		ip->ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) +
1940 		   len);
1941 		ip->ip_off = 0;
1942 		ip->ip_ttl = V_ip_defttl;
1943 		ip->ip_p = IPPROTO_UDP;
1944 		ip->ip_sum = 0;
1945 		if (in_local->sin_port != 0)
1946 			ip->ip_src = in_local->sin_addr;
1947 		else
1948 			ip->ip_src.s_addr = INADDR_ANY;
1949 		ip->ip_dst = in_remote->sin_addr;
1950 
1951 		OVPN_RUNLOCK(sc);
1952 		OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len);
1953 
1954 		return (ip_output(m, NULL, NULL, 0, NULL, NULL));
1955 	}
1956 #endif
1957 #ifdef INET6
1958 	case AF_INET6: {
1959 		struct sockaddr_in6 *in6_local = TO_IN6(&peer->local);
1960 		struct sockaddr_in6 *in6_remote = TO_IN6(&peer->remote);
1961 		struct ip6_hdr *ip6;
1962 
1963 		M_PREPEND(m, sizeof(struct ip6_hdr), M_NOWAIT);
1964 		if (m == NULL) {
1965 			OVPN_RUNLOCK(sc);
1966 			OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1967 			return (ENOBUFS);
1968 		}
1969 		m = m_pullup(m, sizeof(*ip6) + sizeof(*udp));
1970 		if (m == NULL) {
1971 			OVPN_RUNLOCK(sc);
1972 			OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1);
1973 			return (ENOBUFS);
1974 		}
1975 
1976 		ip6 = mtod(m, struct ip6_hdr *);
1977 
1978 		ip6->ip6_vfc = IPV6_VERSION;
1979 		ip6->ip6_flow &= ~IPV6_FLOWINFO_MASK;
1980 		ip6->ip6_plen = htons(sizeof(*ip6) + sizeof(struct udphdr) +
1981 		    len);
1982 		ip6->ip6_nxt = IPPROTO_UDP;
1983 		ip6->ip6_hlim = V_ip6_defhlim;
1984 
1985 		memcpy(&ip6->ip6_src, &in6_local->sin6_addr,
1986 		    sizeof(ip6->ip6_src));
1987 		memcpy(&ip6->ip6_dst, &in6_remote->sin6_addr,
1988 		    sizeof(ip6->ip6_dst));
1989 
1990 		udp = mtodo(m, sizeof(*ip6));
1991 		udp->uh_sum = in6_cksum_pseudo(ip6,
1992 		    m->m_pkthdr.len - sizeof(struct ip6_hdr),
1993 		    IPPROTO_UDP, 0);
1994 
1995 		m->m_pkthdr.csum_flags |= CSUM_UDP_IPV6;
1996 		m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
1997 
1998 		OVPN_RUNLOCK(sc);
1999 		OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len);
2000 
2001 		return (ip6_output(m, NULL, NULL, IPV6_UNSPECSRC, NULL, NULL,
2002 		    NULL));
2003 	}
2004 #endif
2005 	default:
2006 		panic("Unsupported address family %d",
2007 		    peer->remote.ss_family);
2008 	}
2009 }
2010 
2011 static int
2012 ovpn_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
2013 	struct route *ro)
2014 {
2015 	struct ovpn_softc *sc;
2016 	struct ovpn_kpeer *peer;
2017 
2018 	OVPN_RLOCK_TRACKER;
2019 
2020 	sc = ifp->if_softc;
2021 
2022 	OVPN_RLOCK(sc);
2023 
2024 	SDT_PROBE1(if_ovpn, tx, transmit, start, m);
2025 
2026 	if (__predict_false(ifp->if_link_state != LINK_STATE_UP)) {
2027 		OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
2028 		OVPN_RUNLOCK(sc);
2029 		m_freem(m);
2030 		return (ENETDOWN);
2031 	}
2032 
2033 	/**
2034 	 * Only obey 'dst' (i.e. the gateway) if no route is supplied.
2035 	 * That's our indication that we're being called through pf's route-to,
2036 	 * and we should route according to 'dst' instead. We can't do so
2037 	 * consistently, because the usual openvpn configuration sets the first
2038 	 * non-server IP in the subnet as the gateway. If we always use that
2039 	 * one we'd end up routing all traffic to the first client.
2040 	 * tl;dr: 'ro == NULL' tells us pf is doing a route-to, and then but
2041 	 * only then, we should treat 'dst' as the destination. */
2042 	peer = ovpn_route_peer(sc, &m, ro == NULL ? dst : NULL);
2043 	if (peer == NULL) {
2044 		/* No destination. */
2045 		OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1);
2046 		OVPN_RUNLOCK(sc);
2047 		m_freem(m);
2048 		return (ENETDOWN);
2049 	}
2050 
2051 	return (ovpn_transmit_to_peer(ifp, m, peer, _ovpn_lock_trackerp));
2052 }
2053 
2054 static bool
2055 ovpn_check_replay(struct ovpn_kkey_dir *key, uint32_t seq)
2056 {
2057 	uint32_t d;
2058 
2059 	mtx_lock(&key->replay_mtx);
2060 
2061 	/* Sequence number must be strictly greater than rx_seq */
2062 	if (seq <= key->rx_seq) {
2063 		mtx_unlock(&key->replay_mtx);
2064 		return (false);
2065 	}
2066 
2067 	/* Large jump. The packet authenticated okay, so just accept that. */
2068 	if (seq > (key->rx_seq + (sizeof(key->rx_window) * 8))) {
2069 		key->rx_seq = seq;
2070 		key->rx_window = 0;
2071 		mtx_unlock(&key->replay_mtx);
2072 		return (true);
2073 	}
2074 
2075 	/* Happy case. */
2076 	if ((seq == key->rx_seq + 1) && key->rx_window == 0) {
2077 		key->rx_seq++;
2078 		mtx_unlock(&key->replay_mtx);
2079 		return (true);
2080 	}
2081 
2082 	d = seq - key->rx_seq - 1;
2083 
2084 	if (key->rx_window & ((uint64_t)1 << d)) {
2085 		/* Dupe! */
2086 		mtx_unlock(&key->replay_mtx);
2087 		return (false);
2088 	}
2089 
2090 	key->rx_window |= (uint64_t)1 << d;
2091 
2092 	while (key->rx_window & 1) {
2093 		key->rx_seq++;
2094 		key->rx_window >>= 1;
2095 	}
2096 
2097 	mtx_unlock(&key->replay_mtx);
2098 
2099 	return (true);
2100 }
2101 
2102 static struct ovpn_kpeer *
2103 ovpn_peer_from_mbuf(struct ovpn_softc *sc, struct mbuf *m, int off)
2104 {
2105 	struct ovpn_wire_header ohdr;
2106 	uint32_t peerid;
2107 	const size_t hdrlen = sizeof(ohdr) - sizeof(ohdr.auth_tag);
2108 
2109 	OVPN_RASSERT(sc);
2110 
2111 	if (m_length(m, NULL) < (off + sizeof(struct udphdr) + hdrlen))
2112 		return (NULL);
2113 
2114 	m_copydata(m, off + sizeof(struct udphdr), hdrlen, (caddr_t)&ohdr);
2115 
2116 	peerid = ntohl(ohdr.opcode) & 0x00ffffff;
2117 
2118 	return (ovpn_find_peer(sc, peerid));
2119 }
2120 
2121 static bool
2122 ovpn_udp_input(struct mbuf *m, int off, struct inpcb *inp,
2123     const struct sockaddr *sa, void *ctx)
2124 {
2125 	struct ovpn_softc *sc = ctx;
2126 	struct ovpn_wire_header tmphdr;
2127 	struct ovpn_wire_header *ohdr;
2128 	struct udphdr *uhdr;
2129 	struct ovpn_kkey *key;
2130 	struct cryptop *crp;
2131 	struct ovpn_kpeer *peer;
2132 	size_t ohdrlen;
2133 	int ret;
2134 	uint8_t op;
2135 
2136 	OVPN_RLOCK_TRACKER;
2137 
2138 	M_ASSERTPKTHDR(m);
2139 
2140 	OVPN_COUNTER_ADD(sc, transport_bytes_received, m->m_pkthdr.len - off);
2141 
2142 	ohdrlen = sizeof(*ohdr) - sizeof(ohdr->auth_tag);
2143 
2144 	OVPN_RLOCK(sc);
2145 
2146 	peer = ovpn_peer_from_mbuf(sc, m, off);
2147 	if (peer == NULL) {
2148 		OVPN_RUNLOCK(sc);
2149 		return (false);
2150 	}
2151 
2152 	if (m_length(m, NULL) < (off + sizeof(*uhdr) + ohdrlen)) {
2153 		/* Short packet. */
2154 		OVPN_RUNLOCK(sc);
2155 		return (false);
2156 	}
2157 
2158 	m_copydata(m, off + sizeof(*uhdr), ohdrlen, (caddr_t)&tmphdr);
2159 
2160 	op = ntohl(tmphdr.opcode) >> 24 >> OVPN_OP_SHIFT;
2161 	if (op != OVPN_OP_DATA_V2) {
2162 		/* Control packet? */
2163 		OVPN_RUNLOCK(sc);
2164 		return (false);
2165 	}
2166 
2167 	m = m_pullup(m, off + sizeof(*uhdr) + ohdrlen);
2168 	if (m == NULL) {
2169 		OVPN_RUNLOCK(sc);
2170 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1);
2171 		return (true);
2172 	}
2173 
2174 	/*
2175 	 * Simplify things by getting rid of the preceding headers, we don't
2176 	 * care about them.
2177 	 */
2178 	m_adj_decap(m, off);
2179 
2180 	uhdr = mtodo(m, 0);
2181 	ohdr = mtodo(m, sizeof(*uhdr));
2182 
2183 	key = ovpn_find_key(sc, peer, ohdr);
2184 	if (key == NULL || key->decrypt == NULL) {
2185 		OVPN_RUNLOCK(sc);
2186 		OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
2187 		m_freem(m);
2188 		return (true);
2189 	}
2190 
2191 	if (key->decrypt->cipher == OVPN_CIPHER_ALG_NONE) {
2192 		/* Now remove the outer headers */
2193 		m_adj_decap(m, sizeof(struct udphdr) + ohdrlen);
2194 
2195 		ohdr = mtodo(m, sizeof(*uhdr));
2196 
2197 		ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq),
2198 		    _ovpn_lock_trackerp);
2199 		OVPN_UNLOCK_ASSERT(sc);
2200 		return (true);
2201 	}
2202 
2203 	ohdrlen += sizeof(ohdr->auth_tag);
2204 
2205 	m = m_pullup(m, sizeof(*uhdr) + ohdrlen);
2206 	if (m == NULL) {
2207 		OVPN_RUNLOCK(sc);
2208 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1);
2209 		return (true);
2210 	}
2211 	uhdr = mtodo(m, 0);
2212 	ohdr = mtodo(m, sizeof(*uhdr));
2213 
2214 	/* Decrypt */
2215 	crp = crypto_getreq(key->decrypt->cryptoid, M_NOWAIT);
2216 	if (crp == NULL) {
2217 		OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1);
2218 		OVPN_RUNLOCK(sc);
2219 		m_freem(m);
2220 		return (true);
2221 	}
2222 
2223 	crp->crp_payload_start = sizeof(struct udphdr) + sizeof(*ohdr);
2224 	crp->crp_payload_length = ntohs(uhdr->uh_ulen) -
2225 	    sizeof(*uhdr) - sizeof(*ohdr);
2226 	crp->crp_op = CRYPTO_OP_DECRYPT;
2227 
2228 	/* AAD validation. */
2229 	crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag);
2230 	crp->crp_aad = ohdr;
2231 	crp->crp_aad_start = 0;
2232 	crp->crp_op |= CRYPTO_OP_VERIFY_DIGEST;
2233 	crp->crp_digest_start = sizeof(struct udphdr) +
2234 	    offsetof(struct ovpn_wire_header, auth_tag);
2235 
2236 	crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
2237 	memcpy(crp->crp_iv, &ohdr->seq, sizeof(ohdr->seq));
2238 	memcpy(crp->crp_iv + sizeof(ohdr->seq), key->decrypt->nonce,
2239 	    key->decrypt->noncelen);
2240 
2241 	crypto_use_mbuf(crp, m);
2242 	crp->crp_flags |= CRYPTO_F_CBIFSYNC;
2243 	crp->crp_callback = ovpn_decrypt_rx_cb;
2244 	crp->crp_opaque = sc;
2245 
2246 	atomic_add_int(&sc->refcount, 1);
2247 	OVPN_RUNLOCK(sc);
2248 	if (V_async_crypto)
2249 		ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED);
2250 	else
2251 		ret = crypto_dispatch(crp);
2252 	if (ret != 0) {
2253 		OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1);
2254 	}
2255 
2256 	return (true);
2257 }
2258 
2259 static void
2260 ovpn_qflush(struct ifnet *ifp __unused)
2261 {
2262 
2263 }
2264 
2265 static void
2266 ovpn_flush_rxring(struct ovpn_softc *sc)
2267 {
2268 	struct ovpn_notification *n;
2269 
2270 	OVPN_WASSERT(sc);
2271 
2272 	while (! buf_ring_empty(sc->notifring)) {
2273 		n = buf_ring_dequeue_sc(sc->notifring);
2274 		free(n, M_OVPN);
2275 	}
2276 }
2277 
2278 #ifdef VIMAGE
2279 static void
2280 ovpn_reassign(struct ifnet *ifp, struct vnet *new_vnet __unused,
2281     char *unused __unused)
2282 {
2283 	struct ovpn_softc *sc = ifp->if_softc;
2284 	int i;
2285 	int ret __diagused;
2286 
2287 	i = 0;
2288 
2289 	OVPN_WLOCK(sc);
2290 
2291 	/* Flush keys & configuration. */
2292 	do {
2293 		if (sc->peers[i] != NULL) {
2294 			ret = _ovpn_del_peer(sc, sc->peers[i]->peerid);
2295 			MPASS(ret == 0);
2296 		}
2297 		i++;
2298 	} while (i < OVPN_MAX_PEERS);
2299 
2300 	ovpn_flush_rxring(sc);
2301 
2302 	OVPN_WUNLOCK(sc);
2303 }
2304 #endif
2305 
2306 static int
2307 ovpn_clone_match(struct if_clone *ifc, const char *name)
2308 {
2309 	/*
2310 	 * Allow all names that start with 'ovpn', specifically because pfSense
2311 	 * uses ovpnc1 / ovpns2
2312 	 */
2313 	return (strncmp(ovpnname, name, strlen(ovpnname)) == 0);
2314 }
2315 
2316 static int
2317 ovpn_clone_create(struct if_clone *ifc, char *name, size_t len,
2318     struct ifc_data *ifd, struct ifnet **ifpp)
2319 {
2320 	struct ovpn_softc *sc;
2321 	struct ifnet *ifp;
2322 	char *dp;
2323 	int error, unit, wildcard;
2324 
2325 	/* Try to see if a special unit was requested. */
2326 	error = ifc_name2unit(name, &unit);
2327 	if (error != 0)
2328 		return (error);
2329 	wildcard = (unit < 0);
2330 
2331 	error = ifc_alloc_unit(ifc, &unit);
2332 	if (error != 0)
2333 		return (error);
2334 
2335 	/*
2336 	 * If no unit had been given, we need to adjust the ifName.
2337 	 */
2338 	for (dp = name; *dp != '\0'; dp++);
2339 	if (wildcard) {
2340 		error = snprintf(dp, len - (dp - name), "%d", unit);
2341 		if (error > len - (dp - name)) {
2342 			/* ifName too long. */
2343 			ifc_free_unit(ifc, unit);
2344 			return (ENOSPC);
2345 		}
2346 		dp += error;
2347 	}
2348 
2349 	/* Make sure it doesn't already exist. */
2350 	if (ifunit(name) != NULL)
2351 		return (EEXIST);
2352 
2353 	sc = malloc(sizeof(struct ovpn_softc), M_OVPN, M_WAITOK | M_ZERO);
2354 	sc->ifp = if_alloc(IFT_ENC);
2355 	rm_init_flags(&sc->lock, "if_ovpn_lock", RM_RECURSE);
2356 	sc->refcount = 0;
2357 
2358 	sc->notifring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL);
2359 
2360 	COUNTER_ARRAY_ALLOC(sc->counters, OVPN_COUNTER_SIZE, M_WAITOK);
2361 
2362 	ifp = sc->ifp;
2363 	ifp->if_softc = sc;
2364 	strlcpy(ifp->if_xname, name, IFNAMSIZ);
2365 	ifp->if_dname = ovpngroupname;
2366 	ifp->if_dunit = unit;
2367 
2368 	ifp->if_addrlen = 0;
2369 	ifp->if_mtu = 1428;
2370 	ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST;
2371 	ifp->if_ioctl = ovpn_ioctl;
2372 	ifp->if_transmit = ovpn_transmit;
2373 	ifp->if_output = ovpn_output;
2374 	ifp->if_qflush = ovpn_qflush;
2375 #ifdef VIMAGE
2376 	ifp->if_reassign = ovpn_reassign;
2377 #endif
2378 	ifp->if_capabilities |= IFCAP_LINKSTATE;
2379 	ifp->if_capenable |= IFCAP_LINKSTATE;
2380 
2381 	if_attach(ifp);
2382 	bpfattach(ifp, DLT_NULL, sizeof(uint32_t));
2383 	*ifpp = ifp;
2384 
2385 	return (0);
2386 }
2387 
2388 static void
2389 ovpn_clone_destroy_cb(struct epoch_context *ctx)
2390 {
2391 	struct ovpn_softc *sc;
2392 
2393 	sc = __containerof(ctx, struct ovpn_softc, epoch_ctx);
2394 
2395 	MPASS(sc->peercount == 0);
2396 	for (int i = 0; i < OVPN_MAX_PEERS; i++) {
2397 		MPASS(sc->peers[i] == NULL);
2398 	}
2399 
2400 	COUNTER_ARRAY_FREE(sc->counters, OVPN_COUNTER_SIZE);
2401 
2402 	if_free(sc->ifp);
2403 	free(sc, M_OVPN);
2404 }
2405 
2406 static int
2407 ovpn_clone_destroy(struct if_clone *ifc, struct ifnet *ifp, uint32_t flags)
2408 {
2409 	struct ovpn_softc *sc;
2410 	int unit;
2411 	int i;
2412 	int ret __diagused;
2413 
2414 	sc = ifp->if_softc;
2415 	unit = ifp->if_dunit;
2416 
2417 	OVPN_WLOCK(sc);
2418 
2419 	if (atomic_load_int(&sc->refcount) > 0) {
2420 		OVPN_WUNLOCK(sc);
2421 		return (EBUSY);
2422 	}
2423 
2424 	i = 0;
2425 	do {
2426 		if (sc->peers[i] != NULL) {
2427 			ret = _ovpn_del_peer(sc, sc->peers[i]->peerid);
2428 			MPASS(ret == 0);
2429 		}
2430 		i++;
2431 	} while (i < OVPN_MAX_PEERS);
2432 
2433 	ovpn_flush_rxring(sc);
2434 	buf_ring_free(sc->notifring, M_OVPN);
2435 
2436 	OVPN_WUNLOCK(sc);
2437 
2438 	bpfdetach(ifp);
2439 	if_detach(ifp);
2440 	ifp->if_softc = NULL;
2441 
2442 	NET_EPOCH_CALL(ovpn_clone_destroy_cb, &sc->epoch_ctx);
2443 
2444 	if (unit != IF_DUNIT_NONE)
2445 		ifc_free_unit(ifc, unit);
2446 
2447 	NET_EPOCH_DRAIN_CALLBACKS();
2448 
2449 	return (0);
2450 }
2451 
2452 static void
2453 vnet_ovpn_init(const void *unused __unused)
2454 {
2455 	struct if_clone_addreq req = {
2456 		.match_f = ovpn_clone_match,
2457 		.create_f = ovpn_clone_create,
2458 		.destroy_f = ovpn_clone_destroy,
2459 	};
2460 	V_ovpn_cloner = ifc_attach_cloner(ovpngroupname, &req);
2461 }
2462 VNET_SYSINIT(vnet_ovpn_init, SI_SUB_PSEUDO, SI_ORDER_ANY,
2463     vnet_ovpn_init, NULL);
2464 
2465 static void
2466 vnet_ovpn_uninit(const void *unused __unused)
2467 {
2468 	if_clone_detach(V_ovpn_cloner);
2469 }
2470 VNET_SYSUNINIT(vnet_ovpn_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY,
2471     vnet_ovpn_uninit, NULL);
2472 
2473 static int
2474 ovpnmodevent(module_t mod, int type, void *data)
2475 {
2476 	switch (type) {
2477 	case MOD_LOAD:
2478 		/* Done in vnet_ovpn_init() */
2479 		break;
2480 	case MOD_UNLOAD:
2481 		/* Done in vnet_ovpn_uninit() */
2482 		break;
2483 	default:
2484 		return (EOPNOTSUPP);
2485 	}
2486 
2487 	return (0);
2488 }
2489 
2490 static moduledata_t ovpn_mod = {
2491 	"if_ovpn",
2492 	ovpnmodevent,
2493 	0
2494 };
2495 
2496 DECLARE_MODULE(if_ovpn, ovpn_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
2497 MODULE_VERSION(if_ovpn, 1);
2498