xref: /freebsd/sys/dev/wg/if_wg.c (revision e32fecd0c2c3ee37c47ee100f169e7eb0282a873)
1 /* SPDX-License-Identifier: ISC
2  *
3  * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
4  * Copyright (C) 2019-2021 Matt Dunwoodie <ncon@noconroy.net>
5  * Copyright (c) 2019-2020 Rubicon Communications, LLC (Netgate)
6  * Copyright (c) 2021 Kyle Evans <kevans@FreeBSD.org>
7  * Copyright (c) 2022 The FreeBSD Foundation
8  */
9 
10 #include "opt_inet.h"
11 #include "opt_inet6.h"
12 
13 #include <sys/param.h>
14 #include <sys/systm.h>
15 #include <sys/counter.h>
16 #include <sys/gtaskqueue.h>
17 #include <sys/jail.h>
18 #include <sys/kernel.h>
19 #include <sys/lock.h>
20 #include <sys/mbuf.h>
21 #include <sys/module.h>
22 #include <sys/nv.h>
23 #include <sys/priv.h>
24 #include <sys/protosw.h>
25 #include <sys/rmlock.h>
26 #include <sys/rwlock.h>
27 #include <sys/smp.h>
28 #include <sys/socket.h>
29 #include <sys/socketvar.h>
30 #include <sys/sockio.h>
31 #include <sys/sysctl.h>
32 #include <sys/sx.h>
33 #include <machine/_inttypes.h>
34 #include <net/bpf.h>
35 #include <net/ethernet.h>
36 #include <net/if.h>
37 #include <net/if_clone.h>
38 #include <net/if_types.h>
39 #include <net/if_var.h>
40 #include <net/netisr.h>
41 #include <net/radix.h>
42 #include <netinet/in.h>
43 #include <netinet6/in6_var.h>
44 #include <netinet/ip.h>
45 #include <netinet/ip6.h>
46 #include <netinet/ip_icmp.h>
47 #include <netinet/icmp6.h>
48 #include <netinet/udp_var.h>
49 #include <netinet6/nd6.h>
50 
51 #include "wg_noise.h"
52 #include "wg_cookie.h"
53 #include "version.h"
54 #include "if_wg.h"
55 
56 #define DEFAULT_MTU		(ETHERMTU - 80)
57 #define MAX_MTU			(IF_MAXMTU - 80)
58 
59 #define MAX_STAGED_PKT		128
60 #define MAX_QUEUED_PKT		1024
61 #define MAX_QUEUED_PKT_MASK	(MAX_QUEUED_PKT - 1)
62 
63 #define MAX_QUEUED_HANDSHAKES	4096
64 
65 #define REKEY_TIMEOUT_JITTER	334 /* 1/3 sec, round for arc4random_uniform */
66 #define MAX_TIMER_HANDSHAKES	(90 / REKEY_TIMEOUT)
67 #define NEW_HANDSHAKE_TIMEOUT	(REKEY_TIMEOUT + KEEPALIVE_TIMEOUT)
68 #define UNDERLOAD_TIMEOUT	1
69 
70 #define DPRINTF(sc, ...) if (if_getflags(sc->sc_ifp) & IFF_DEBUG) if_printf(sc->sc_ifp, ##__VA_ARGS__)
71 
72 /* First byte indicating packet type on the wire */
73 #define WG_PKT_INITIATION htole32(1)
74 #define WG_PKT_RESPONSE htole32(2)
75 #define WG_PKT_COOKIE htole32(3)
76 #define WG_PKT_DATA htole32(4)
77 
78 #define WG_PKT_PADDING		16
79 #define WG_KEY_SIZE		32
80 
81 struct wg_pkt_initiation {
82 	uint32_t		t;
83 	uint32_t		s_idx;
84 	uint8_t			ue[NOISE_PUBLIC_KEY_LEN];
85 	uint8_t			es[NOISE_PUBLIC_KEY_LEN + NOISE_AUTHTAG_LEN];
86 	uint8_t			ets[NOISE_TIMESTAMP_LEN + NOISE_AUTHTAG_LEN];
87 	struct cookie_macs	m;
88 };
89 
90 struct wg_pkt_response {
91 	uint32_t		t;
92 	uint32_t		s_idx;
93 	uint32_t		r_idx;
94 	uint8_t			ue[NOISE_PUBLIC_KEY_LEN];
95 	uint8_t			en[0 + NOISE_AUTHTAG_LEN];
96 	struct cookie_macs	m;
97 };
98 
99 struct wg_pkt_cookie {
100 	uint32_t		t;
101 	uint32_t		r_idx;
102 	uint8_t			nonce[COOKIE_NONCE_SIZE];
103 	uint8_t			ec[COOKIE_ENCRYPTED_SIZE];
104 };
105 
106 struct wg_pkt_data {
107 	uint32_t		t;
108 	uint32_t		r_idx;
109 	uint64_t		nonce;
110 	uint8_t			buf[];
111 };
112 
113 struct wg_endpoint {
114 	union {
115 		struct sockaddr		r_sa;
116 		struct sockaddr_in	r_sin;
117 #ifdef INET6
118 		struct sockaddr_in6	r_sin6;
119 #endif
120 	} e_remote;
121 	union {
122 		struct in_addr		l_in;
123 #ifdef INET6
124 		struct in6_pktinfo	l_pktinfo6;
125 #define l_in6 l_pktinfo6.ipi6_addr
126 #endif
127 	} e_local;
128 };
129 
130 struct aip_addr {
131 	uint8_t		length;
132 	union {
133 		uint8_t		bytes[16];
134 		uint32_t	ip;
135 		uint32_t	ip6[4];
136 		struct in_addr	in;
137 		struct in6_addr	in6;
138 	};
139 };
140 
141 struct wg_aip {
142 	struct radix_node	 a_nodes[2];
143 	LIST_ENTRY(wg_aip)	 a_entry;
144 	struct aip_addr		 a_addr;
145 	struct aip_addr		 a_mask;
146 	struct wg_peer		*a_peer;
147 	sa_family_t		 a_af;
148 };
149 
150 struct wg_packet {
151 	STAILQ_ENTRY(wg_packet)	 p_serial;
152 	STAILQ_ENTRY(wg_packet)	 p_parallel;
153 	struct wg_endpoint	 p_endpoint;
154 	struct noise_keypair	*p_keypair;
155 	uint64_t		 p_nonce;
156 	struct mbuf		*p_mbuf;
157 	int			 p_mtu;
158 	sa_family_t		 p_af;
159 	enum wg_ring_state {
160 		WG_PACKET_UNCRYPTED,
161 		WG_PACKET_CRYPTED,
162 		WG_PACKET_DEAD,
163 	}			 p_state;
164 };
165 
166 STAILQ_HEAD(wg_packet_list, wg_packet);
167 
168 struct wg_queue {
169 	struct mtx		 q_mtx;
170 	struct wg_packet_list	 q_queue;
171 	size_t			 q_len;
172 };
173 
174 struct wg_peer {
175 	TAILQ_ENTRY(wg_peer)		 p_entry;
176 	uint64_t			 p_id;
177 	struct wg_softc			*p_sc;
178 
179 	struct noise_remote		*p_remote;
180 	struct cookie_maker		 p_cookie;
181 
182 	struct rwlock			 p_endpoint_lock;
183 	struct wg_endpoint		 p_endpoint;
184 
185 	struct wg_queue	 		 p_stage_queue;
186 	struct wg_queue	 		 p_encrypt_serial;
187 	struct wg_queue	 		 p_decrypt_serial;
188 
189 	bool				 p_enabled;
190 	bool				 p_need_another_keepalive;
191 	uint16_t			 p_persistent_keepalive_interval;
192 	struct callout			 p_new_handshake;
193 	struct callout			 p_send_keepalive;
194 	struct callout			 p_retry_handshake;
195 	struct callout			 p_zero_key_material;
196 	struct callout			 p_persistent_keepalive;
197 
198 	struct mtx			 p_handshake_mtx;
199 	struct timespec			 p_handshake_complete;	/* nanotime */
200 	int				 p_handshake_retries;
201 
202 	struct grouptask		 p_send;
203 	struct grouptask		 p_recv;
204 
205 	counter_u64_t			 p_tx_bytes;
206 	counter_u64_t			 p_rx_bytes;
207 
208 	LIST_HEAD(, wg_aip)		 p_aips;
209 	size_t				 p_aips_num;
210 };
211 
212 struct wg_socket {
213 	struct socket	*so_so4;
214 	struct socket	*so_so6;
215 	uint32_t	 so_user_cookie;
216 	int		 so_fibnum;
217 	in_port_t	 so_port;
218 };
219 
220 struct wg_softc {
221 	LIST_ENTRY(wg_softc)	 sc_entry;
222 	if_t			 sc_ifp;
223 	int			 sc_flags;
224 
225 	struct ucred		*sc_ucred;
226 	struct wg_socket	 sc_socket;
227 
228 	TAILQ_HEAD(,wg_peer)	 sc_peers;
229 	size_t			 sc_peers_num;
230 
231 	struct noise_local	*sc_local;
232 	struct cookie_checker	 sc_cookie;
233 
234 	struct radix_node_head	*sc_aip4;
235 	struct radix_node_head	*sc_aip6;
236 
237 	struct grouptask	 sc_handshake;
238 	struct wg_queue		 sc_handshake_queue;
239 
240 	struct grouptask	*sc_encrypt;
241 	struct grouptask	*sc_decrypt;
242 	struct wg_queue		 sc_encrypt_parallel;
243 	struct wg_queue		 sc_decrypt_parallel;
244 	u_int			 sc_encrypt_last_cpu;
245 	u_int			 sc_decrypt_last_cpu;
246 
247 	struct sx		 sc_lock;
248 };
249 
250 #define	WGF_DYING	0x0001
251 
252 #define MAX_LOOPS	8
253 #define MTAG_WGLOOP	0x77676c70 /* wglp */
254 #ifndef ENOKEY
255 #define	ENOKEY	ENOTCAPABLE
256 #endif
257 
258 #define	GROUPTASK_DRAIN(gtask)			\
259 	gtaskqueue_drain((gtask)->gt_taskqueue, &(gtask)->gt_task)
260 
261 #define BPF_MTAP2_AF(ifp, m, af) do { \
262 		uint32_t __bpf_tap_af = (af); \
263 		BPF_MTAP2(ifp, &__bpf_tap_af, sizeof(__bpf_tap_af), m); \
264 	} while (0)
265 
266 static int clone_count;
267 static uma_zone_t wg_packet_zone;
268 static volatile unsigned long peer_counter = 0;
269 static const char wgname[] = "wg";
270 static unsigned wg_osd_jail_slot;
271 
272 static struct sx wg_sx;
273 SX_SYSINIT(wg_sx, &wg_sx, "wg_sx");
274 
275 static LIST_HEAD(, wg_softc) wg_list = LIST_HEAD_INITIALIZER(wg_list);
276 
277 static TASKQGROUP_DEFINE(wg_tqg, mp_ncpus, 1);
278 
279 MALLOC_DEFINE(M_WG, "WG", "wireguard");
280 
281 VNET_DEFINE_STATIC(struct if_clone *, wg_cloner);
282 
283 #define	V_wg_cloner	VNET(wg_cloner)
284 #define	WG_CAPS		IFCAP_LINKSTATE
285 
286 struct wg_timespec64 {
287 	uint64_t	tv_sec;
288 	uint64_t	tv_nsec;
289 };
290 
291 static int wg_socket_init(struct wg_softc *, in_port_t);
292 static int wg_socket_bind(struct socket **, struct socket **, in_port_t *);
293 static void wg_socket_set(struct wg_softc *, struct socket *, struct socket *);
294 static void wg_socket_uninit(struct wg_softc *);
295 static int wg_socket_set_sockopt(struct socket *, struct socket *, int, void *, size_t);
296 static int wg_socket_set_cookie(struct wg_softc *, uint32_t);
297 static int wg_socket_set_fibnum(struct wg_softc *, int);
298 static int wg_send(struct wg_softc *, struct wg_endpoint *, struct mbuf *);
299 static void wg_timers_enable(struct wg_peer *);
300 static void wg_timers_disable(struct wg_peer *);
301 static void wg_timers_set_persistent_keepalive(struct wg_peer *, uint16_t);
302 static void wg_timers_get_last_handshake(struct wg_peer *, struct wg_timespec64 *);
303 static void wg_timers_event_data_sent(struct wg_peer *);
304 static void wg_timers_event_data_received(struct wg_peer *);
305 static void wg_timers_event_any_authenticated_packet_sent(struct wg_peer *);
306 static void wg_timers_event_any_authenticated_packet_received(struct wg_peer *);
307 static void wg_timers_event_any_authenticated_packet_traversal(struct wg_peer *);
308 static void wg_timers_event_handshake_initiated(struct wg_peer *);
309 static void wg_timers_event_handshake_complete(struct wg_peer *);
310 static void wg_timers_event_session_derived(struct wg_peer *);
311 static void wg_timers_event_want_initiation(struct wg_peer *);
312 static void wg_timers_run_send_initiation(struct wg_peer *, bool);
313 static void wg_timers_run_retry_handshake(void *);
314 static void wg_timers_run_send_keepalive(void *);
315 static void wg_timers_run_new_handshake(void *);
316 static void wg_timers_run_zero_key_material(void *);
317 static void wg_timers_run_persistent_keepalive(void *);
318 static int wg_aip_add(struct wg_softc *, struct wg_peer *, sa_family_t, const void *, uint8_t);
319 static struct wg_peer *wg_aip_lookup(struct wg_softc *, sa_family_t, void *);
320 static void wg_aip_remove_all(struct wg_softc *, struct wg_peer *);
321 static struct wg_peer *wg_peer_alloc(struct wg_softc *, const uint8_t [WG_KEY_SIZE]);
322 static void wg_peer_free_deferred(struct noise_remote *);
323 static void wg_peer_destroy(struct wg_peer *);
324 static void wg_peer_destroy_all(struct wg_softc *);
325 static void wg_peer_send_buf(struct wg_peer *, uint8_t *, size_t);
326 static void wg_send_initiation(struct wg_peer *);
327 static void wg_send_response(struct wg_peer *);
328 static void wg_send_cookie(struct wg_softc *, struct cookie_macs *, uint32_t, struct wg_endpoint *);
329 static void wg_peer_set_endpoint(struct wg_peer *, struct wg_endpoint *);
330 static void wg_peer_clear_src(struct wg_peer *);
331 static void wg_peer_get_endpoint(struct wg_peer *, struct wg_endpoint *);
332 static void wg_send_buf(struct wg_softc *, struct wg_endpoint *, uint8_t *, size_t);
333 static void wg_send_keepalive(struct wg_peer *);
334 static void wg_handshake(struct wg_softc *, struct wg_packet *);
335 static void wg_encrypt(struct wg_softc *, struct wg_packet *);
336 static void wg_decrypt(struct wg_softc *, struct wg_packet *);
337 static void wg_softc_handshake_receive(struct wg_softc *);
338 static void wg_softc_decrypt(struct wg_softc *);
339 static void wg_softc_encrypt(struct wg_softc *);
340 static void wg_encrypt_dispatch(struct wg_softc *);
341 static void wg_decrypt_dispatch(struct wg_softc *);
342 static void wg_deliver_out(struct wg_peer *);
343 static void wg_deliver_in(struct wg_peer *);
344 static struct wg_packet *wg_packet_alloc(struct mbuf *);
345 static void wg_packet_free(struct wg_packet *);
346 static void wg_queue_init(struct wg_queue *, const char *);
347 static void wg_queue_deinit(struct wg_queue *);
348 static size_t wg_queue_len(struct wg_queue *);
349 static int wg_queue_enqueue_handshake(struct wg_queue *, struct wg_packet *);
350 static struct wg_packet *wg_queue_dequeue_handshake(struct wg_queue *);
351 static void wg_queue_push_staged(struct wg_queue *, struct wg_packet *);
352 static void wg_queue_enlist_staged(struct wg_queue *, struct wg_packet_list *);
353 static void wg_queue_delist_staged(struct wg_queue *, struct wg_packet_list *);
354 static void wg_queue_purge(struct wg_queue *);
355 static int wg_queue_both(struct wg_queue *, struct wg_queue *, struct wg_packet *);
356 static struct wg_packet *wg_queue_dequeue_serial(struct wg_queue *);
357 static struct wg_packet *wg_queue_dequeue_parallel(struct wg_queue *);
358 static bool wg_input(struct mbuf *, int, struct inpcb *, const struct sockaddr *, void *);
359 static void wg_peer_send_staged(struct wg_peer *);
360 static int wg_clone_create(struct if_clone *ifc, char *name, size_t len,
361 	struct ifc_data *ifd, if_t *ifpp);
362 static void wg_qflush(if_t);
363 static inline int determine_af_and_pullup(struct mbuf **m, sa_family_t *af);
364 static int wg_xmit(if_t, struct mbuf *, sa_family_t, uint32_t);
365 static int wg_transmit(if_t, struct mbuf *);
366 static int wg_output(if_t, struct mbuf *, const struct sockaddr *, struct route *);
367 static int wg_clone_destroy(struct if_clone *ifc, if_t ifp,
368 	uint32_t flags);
369 static bool wgc_privileged(struct wg_softc *);
370 static int wgc_get(struct wg_softc *, struct wg_data_io *);
371 static int wgc_set(struct wg_softc *, struct wg_data_io *);
372 static int wg_up(struct wg_softc *);
373 static void wg_down(struct wg_softc *);
374 static void wg_reassign(if_t, struct vnet *, char *unused);
375 static void wg_init(void *);
376 static int wg_ioctl(if_t, u_long, caddr_t);
377 static void vnet_wg_init(const void *);
378 static void vnet_wg_uninit(const void *);
379 static int wg_module_init(void);
380 static void wg_module_deinit(void);
381 
382 /* TODO Peer */
383 static struct wg_peer *
384 wg_peer_alloc(struct wg_softc *sc, const uint8_t pub_key[WG_KEY_SIZE])
385 {
386 	struct wg_peer *peer;
387 
388 	sx_assert(&sc->sc_lock, SX_XLOCKED);
389 
390 	peer = malloc(sizeof(*peer), M_WG, M_WAITOK | M_ZERO);
391 	peer->p_remote = noise_remote_alloc(sc->sc_local, peer, pub_key);
392 	peer->p_tx_bytes = counter_u64_alloc(M_WAITOK);
393 	peer->p_rx_bytes = counter_u64_alloc(M_WAITOK);
394 	peer->p_id = peer_counter++;
395 	peer->p_sc = sc;
396 
397 	cookie_maker_init(&peer->p_cookie, pub_key);
398 
399 	rw_init(&peer->p_endpoint_lock, "wg_peer_endpoint");
400 
401 	wg_queue_init(&peer->p_stage_queue, "stageq");
402 	wg_queue_init(&peer->p_encrypt_serial, "txq");
403 	wg_queue_init(&peer->p_decrypt_serial, "rxq");
404 
405 	peer->p_enabled = false;
406 	peer->p_need_another_keepalive = false;
407 	peer->p_persistent_keepalive_interval = 0;
408 	callout_init(&peer->p_new_handshake, true);
409 	callout_init(&peer->p_send_keepalive, true);
410 	callout_init(&peer->p_retry_handshake, true);
411 	callout_init(&peer->p_persistent_keepalive, true);
412 	callout_init(&peer->p_zero_key_material, true);
413 
414 	mtx_init(&peer->p_handshake_mtx, "peer handshake", NULL, MTX_DEF);
415 	bzero(&peer->p_handshake_complete, sizeof(peer->p_handshake_complete));
416 	peer->p_handshake_retries = 0;
417 
418 	GROUPTASK_INIT(&peer->p_send, 0, (gtask_fn_t *)wg_deliver_out, peer);
419 	taskqgroup_attach(qgroup_wg_tqg, &peer->p_send, peer, NULL, NULL, "wg send");
420 	GROUPTASK_INIT(&peer->p_recv, 0, (gtask_fn_t *)wg_deliver_in, peer);
421 	taskqgroup_attach(qgroup_wg_tqg, &peer->p_recv, peer, NULL, NULL, "wg recv");
422 
423 	LIST_INIT(&peer->p_aips);
424 	peer->p_aips_num = 0;
425 
426 	return (peer);
427 }
428 
429 static void
430 wg_peer_free_deferred(struct noise_remote *r)
431 {
432 	struct wg_peer *peer = noise_remote_arg(r);
433 
434 	/* While there are no references remaining, we may still have
435 	 * p_{send,recv} executing (think empty queue, but wg_deliver_{in,out}
436 	 * needs to check the queue. We should wait for them and then free. */
437 	GROUPTASK_DRAIN(&peer->p_recv);
438 	GROUPTASK_DRAIN(&peer->p_send);
439 	taskqgroup_detach(qgroup_wg_tqg, &peer->p_recv);
440 	taskqgroup_detach(qgroup_wg_tqg, &peer->p_send);
441 
442 	wg_queue_deinit(&peer->p_decrypt_serial);
443 	wg_queue_deinit(&peer->p_encrypt_serial);
444 	wg_queue_deinit(&peer->p_stage_queue);
445 
446 	counter_u64_free(peer->p_tx_bytes);
447 	counter_u64_free(peer->p_rx_bytes);
448 	rw_destroy(&peer->p_endpoint_lock);
449 	mtx_destroy(&peer->p_handshake_mtx);
450 
451 	cookie_maker_free(&peer->p_cookie);
452 
453 	free(peer, M_WG);
454 }
455 
456 static void
457 wg_peer_destroy(struct wg_peer *peer)
458 {
459 	struct wg_softc *sc = peer->p_sc;
460 	sx_assert(&sc->sc_lock, SX_XLOCKED);
461 
462 	/* Disable remote and timers. This will prevent any new handshakes
463 	 * occuring. */
464 	noise_remote_disable(peer->p_remote);
465 	wg_timers_disable(peer);
466 
467 	/* Now we can remove all allowed IPs so no more packets will be routed
468 	 * to the peer. */
469 	wg_aip_remove_all(sc, peer);
470 
471 	/* Remove peer from the interface, then free. Some references may still
472 	 * exist to p_remote, so noise_remote_free will wait until they're all
473 	 * put to call wg_peer_free_deferred. */
474 	sc->sc_peers_num--;
475 	TAILQ_REMOVE(&sc->sc_peers, peer, p_entry);
476 	DPRINTF(sc, "Peer %" PRIu64 " destroyed\n", peer->p_id);
477 	noise_remote_free(peer->p_remote, wg_peer_free_deferred);
478 }
479 
480 static void
481 wg_peer_destroy_all(struct wg_softc *sc)
482 {
483 	struct wg_peer *peer, *tpeer;
484 	TAILQ_FOREACH_SAFE(peer, &sc->sc_peers, p_entry, tpeer)
485 		wg_peer_destroy(peer);
486 }
487 
488 static void
489 wg_peer_set_endpoint(struct wg_peer *peer, struct wg_endpoint *e)
490 {
491 	MPASS(e->e_remote.r_sa.sa_family != 0);
492 	if (memcmp(e, &peer->p_endpoint, sizeof(*e)) == 0)
493 		return;
494 
495 	rw_wlock(&peer->p_endpoint_lock);
496 	peer->p_endpoint = *e;
497 	rw_wunlock(&peer->p_endpoint_lock);
498 }
499 
500 static void
501 wg_peer_clear_src(struct wg_peer *peer)
502 {
503 	rw_wlock(&peer->p_endpoint_lock);
504 	bzero(&peer->p_endpoint.e_local, sizeof(peer->p_endpoint.e_local));
505 	rw_wunlock(&peer->p_endpoint_lock);
506 }
507 
508 static void
509 wg_peer_get_endpoint(struct wg_peer *peer, struct wg_endpoint *e)
510 {
511 	rw_rlock(&peer->p_endpoint_lock);
512 	*e = peer->p_endpoint;
513 	rw_runlock(&peer->p_endpoint_lock);
514 }
515 
516 /* Allowed IP */
517 static int
518 wg_aip_add(struct wg_softc *sc, struct wg_peer *peer, sa_family_t af, const void *addr, uint8_t cidr)
519 {
520 	struct radix_node_head	*root;
521 	struct radix_node	*node;
522 	struct wg_aip		*aip;
523 	int			 ret = 0;
524 
525 	aip = malloc(sizeof(*aip), M_WG, M_WAITOK | M_ZERO);
526 	aip->a_peer = peer;
527 	aip->a_af = af;
528 
529 	switch (af) {
530 #ifdef INET
531 	case AF_INET:
532 		if (cidr > 32) cidr = 32;
533 		root = sc->sc_aip4;
534 		aip->a_addr.in = *(const struct in_addr *)addr;
535 		aip->a_mask.ip = htonl(~((1LL << (32 - cidr)) - 1) & 0xffffffff);
536 		aip->a_addr.ip &= aip->a_mask.ip;
537 		aip->a_addr.length = aip->a_mask.length = offsetof(struct aip_addr, in) + sizeof(struct in_addr);
538 		break;
539 #endif
540 #ifdef INET6
541 	case AF_INET6:
542 		if (cidr > 128) cidr = 128;
543 		root = sc->sc_aip6;
544 		aip->a_addr.in6 = *(const struct in6_addr *)addr;
545 		in6_prefixlen2mask(&aip->a_mask.in6, cidr);
546 		for (int i = 0; i < 4; i++)
547 			aip->a_addr.ip6[i] &= aip->a_mask.ip6[i];
548 		aip->a_addr.length = aip->a_mask.length = offsetof(struct aip_addr, in6) + sizeof(struct in6_addr);
549 		break;
550 #endif
551 	default:
552 		free(aip, M_WG);
553 		return (EAFNOSUPPORT);
554 	}
555 
556 	RADIX_NODE_HEAD_LOCK(root);
557 	node = root->rnh_addaddr(&aip->a_addr, &aip->a_mask, &root->rh, aip->a_nodes);
558 	if (node == aip->a_nodes) {
559 		LIST_INSERT_HEAD(&peer->p_aips, aip, a_entry);
560 		peer->p_aips_num++;
561 	} else if (!node)
562 		node = root->rnh_lookup(&aip->a_addr, &aip->a_mask, &root->rh);
563 	if (!node) {
564 		free(aip, M_WG);
565 		return (ENOMEM);
566 	} else if (node != aip->a_nodes) {
567 		free(aip, M_WG);
568 		aip = (struct wg_aip *)node;
569 		if (aip->a_peer != peer) {
570 			LIST_REMOVE(aip, a_entry);
571 			aip->a_peer->p_aips_num--;
572 			aip->a_peer = peer;
573 			LIST_INSERT_HEAD(&peer->p_aips, aip, a_entry);
574 			aip->a_peer->p_aips_num++;
575 		}
576 	}
577 	RADIX_NODE_HEAD_UNLOCK(root);
578 	return (ret);
579 }
580 
581 static struct wg_peer *
582 wg_aip_lookup(struct wg_softc *sc, sa_family_t af, void *a)
583 {
584 	struct radix_node_head	*root;
585 	struct radix_node	*node;
586 	struct wg_peer		*peer;
587 	struct aip_addr		 addr;
588 	RADIX_NODE_HEAD_RLOCK_TRACKER;
589 
590 	switch (af) {
591 	case AF_INET:
592 		root = sc->sc_aip4;
593 		memcpy(&addr.in, a, sizeof(addr.in));
594 		addr.length = offsetof(struct aip_addr, in) + sizeof(struct in_addr);
595 		break;
596 	case AF_INET6:
597 		root = sc->sc_aip6;
598 		memcpy(&addr.in6, a, sizeof(addr.in6));
599 		addr.length = offsetof(struct aip_addr, in6) + sizeof(struct in6_addr);
600 		break;
601 	default:
602 		return NULL;
603 	}
604 
605 	RADIX_NODE_HEAD_RLOCK(root);
606 	node = root->rnh_matchaddr(&addr, &root->rh);
607 	if (node != NULL) {
608 		peer = ((struct wg_aip *)node)->a_peer;
609 		noise_remote_ref(peer->p_remote);
610 	} else {
611 		peer = NULL;
612 	}
613 	RADIX_NODE_HEAD_RUNLOCK(root);
614 
615 	return (peer);
616 }
617 
618 static void
619 wg_aip_remove_all(struct wg_softc *sc, struct wg_peer *peer)
620 {
621 	struct wg_aip		*aip, *taip;
622 
623 	RADIX_NODE_HEAD_LOCK(sc->sc_aip4);
624 	LIST_FOREACH_SAFE(aip, &peer->p_aips, a_entry, taip) {
625 		if (aip->a_af == AF_INET) {
626 			if (sc->sc_aip4->rnh_deladdr(&aip->a_addr, &aip->a_mask, &sc->sc_aip4->rh) == NULL)
627 				panic("failed to delete aip %p", aip);
628 			LIST_REMOVE(aip, a_entry);
629 			peer->p_aips_num--;
630 			free(aip, M_WG);
631 		}
632 	}
633 	RADIX_NODE_HEAD_UNLOCK(sc->sc_aip4);
634 
635 	RADIX_NODE_HEAD_LOCK(sc->sc_aip6);
636 	LIST_FOREACH_SAFE(aip, &peer->p_aips, a_entry, taip) {
637 		if (aip->a_af == AF_INET6) {
638 			if (sc->sc_aip6->rnh_deladdr(&aip->a_addr, &aip->a_mask, &sc->sc_aip6->rh) == NULL)
639 				panic("failed to delete aip %p", aip);
640 			LIST_REMOVE(aip, a_entry);
641 			peer->p_aips_num--;
642 			free(aip, M_WG);
643 		}
644 	}
645 	RADIX_NODE_HEAD_UNLOCK(sc->sc_aip6);
646 
647 	if (!LIST_EMPTY(&peer->p_aips) || peer->p_aips_num != 0)
648 		panic("wg_aip_remove_all could not delete all %p", peer);
649 }
650 
651 static int
652 wg_socket_init(struct wg_softc *sc, in_port_t port)
653 {
654 	struct ucred *cred = sc->sc_ucred;
655 	struct socket *so4 = NULL, *so6 = NULL;
656 	int rc;
657 
658 	sx_assert(&sc->sc_lock, SX_XLOCKED);
659 
660 	if (!cred)
661 		return (EBUSY);
662 
663 	/*
664 	 * For socket creation, we use the creds of the thread that created the
665 	 * tunnel rather than the current thread to maintain the semantics that
666 	 * WireGuard has on Linux with network namespaces -- that the sockets
667 	 * are created in their home vnet so that they can be configured and
668 	 * functionally attached to a foreign vnet as the jail's only interface
669 	 * to the network.
670 	 */
671 #ifdef INET
672 	rc = socreate(AF_INET, &so4, SOCK_DGRAM, IPPROTO_UDP, cred, curthread);
673 	if (rc)
674 		goto out;
675 
676 	rc = udp_set_kernel_tunneling(so4, wg_input, NULL, sc);
677 	/*
678 	 * udp_set_kernel_tunneling can only fail if there is already a tunneling function set.
679 	 * This should never happen with a new socket.
680 	 */
681 	MPASS(rc == 0);
682 #endif
683 
684 #ifdef INET6
685 	rc = socreate(AF_INET6, &so6, SOCK_DGRAM, IPPROTO_UDP, cred, curthread);
686 	if (rc)
687 		goto out;
688 	rc = udp_set_kernel_tunneling(so6, wg_input, NULL, sc);
689 	MPASS(rc == 0);
690 #endif
691 
692 	if (sc->sc_socket.so_user_cookie) {
693 		rc = wg_socket_set_sockopt(so4, so6, SO_USER_COOKIE, &sc->sc_socket.so_user_cookie, sizeof(sc->sc_socket.so_user_cookie));
694 		if (rc)
695 			goto out;
696 	}
697 	rc = wg_socket_set_sockopt(so4, so6, SO_SETFIB, &sc->sc_socket.so_fibnum, sizeof(sc->sc_socket.so_fibnum));
698 	if (rc)
699 		goto out;
700 
701 	rc = wg_socket_bind(&so4, &so6, &port);
702 	if (!rc) {
703 		sc->sc_socket.so_port = port;
704 		wg_socket_set(sc, so4, so6);
705 	}
706 out:
707 	if (rc) {
708 		if (so4 != NULL)
709 			soclose(so4);
710 		if (so6 != NULL)
711 			soclose(so6);
712 	}
713 	return (rc);
714 }
715 
716 static int wg_socket_set_sockopt(struct socket *so4, struct socket *so6, int name, void *val, size_t len)
717 {
718 	int ret4 = 0, ret6 = 0;
719 	struct sockopt sopt = {
720 		.sopt_dir = SOPT_SET,
721 		.sopt_level = SOL_SOCKET,
722 		.sopt_name = name,
723 		.sopt_val = val,
724 		.sopt_valsize = len
725 	};
726 
727 	if (so4)
728 		ret4 = sosetopt(so4, &sopt);
729 	if (so6)
730 		ret6 = sosetopt(so6, &sopt);
731 	return (ret4 ?: ret6);
732 }
733 
734 static int wg_socket_set_cookie(struct wg_softc *sc, uint32_t user_cookie)
735 {
736 	struct wg_socket *so = &sc->sc_socket;
737 	int ret;
738 
739 	sx_assert(&sc->sc_lock, SX_XLOCKED);
740 	ret = wg_socket_set_sockopt(so->so_so4, so->so_so6, SO_USER_COOKIE, &user_cookie, sizeof(user_cookie));
741 	if (!ret)
742 		so->so_user_cookie = user_cookie;
743 	return (ret);
744 }
745 
746 static int wg_socket_set_fibnum(struct wg_softc *sc, int fibnum)
747 {
748 	struct wg_socket *so = &sc->sc_socket;
749 	int ret;
750 
751 	sx_assert(&sc->sc_lock, SX_XLOCKED);
752 
753 	ret = wg_socket_set_sockopt(so->so_so4, so->so_so6, SO_SETFIB, &fibnum, sizeof(fibnum));
754 	if (!ret)
755 		so->so_fibnum = fibnum;
756 	return (ret);
757 }
758 
759 static void
760 wg_socket_uninit(struct wg_softc *sc)
761 {
762 	wg_socket_set(sc, NULL, NULL);
763 }
764 
765 static void
766 wg_socket_set(struct wg_softc *sc, struct socket *new_so4, struct socket *new_so6)
767 {
768 	struct wg_socket *so = &sc->sc_socket;
769 	struct socket *so4, *so6;
770 
771 	sx_assert(&sc->sc_lock, SX_XLOCKED);
772 
773 	so4 = atomic_load_ptr(&so->so_so4);
774 	so6 = atomic_load_ptr(&so->so_so6);
775 	atomic_store_ptr(&so->so_so4, new_so4);
776 	atomic_store_ptr(&so->so_so6, new_so6);
777 
778 	if (!so4 && !so6)
779 		return;
780 	NET_EPOCH_WAIT();
781 	if (so4)
782 		soclose(so4);
783 	if (so6)
784 		soclose(so6);
785 }
786 
787 static int
788 wg_socket_bind(struct socket **in_so4, struct socket **in_so6, in_port_t *requested_port)
789 {
790 	struct socket *so4 = *in_so4, *so6 = *in_so6;
791 	int ret4 = 0, ret6 = 0;
792 	in_port_t port = *requested_port;
793 	struct sockaddr_in sin = {
794 		.sin_len = sizeof(struct sockaddr_in),
795 		.sin_family = AF_INET,
796 		.sin_port = htons(port)
797 	};
798 	struct sockaddr_in6 sin6 = {
799 		.sin6_len = sizeof(struct sockaddr_in6),
800 		.sin6_family = AF_INET6,
801 		.sin6_port = htons(port)
802 	};
803 
804 	if (so4) {
805 		ret4 = sobind(so4, (struct sockaddr *)&sin, curthread);
806 		if (ret4 && ret4 != EADDRNOTAVAIL)
807 			return (ret4);
808 		if (!ret4 && !sin.sin_port) {
809 			struct sockaddr_in *bound_sin;
810 			int ret = so4->so_proto->pr_sockaddr(so4,
811 			    (struct sockaddr **)&bound_sin);
812 			if (ret)
813 				return (ret);
814 			port = ntohs(bound_sin->sin_port);
815 			sin6.sin6_port = bound_sin->sin_port;
816 			free(bound_sin, M_SONAME);
817 		}
818 	}
819 
820 	if (so6) {
821 		ret6 = sobind(so6, (struct sockaddr *)&sin6, curthread);
822 		if (ret6 && ret6 != EADDRNOTAVAIL)
823 			return (ret6);
824 		if (!ret6 && !sin6.sin6_port) {
825 			struct sockaddr_in6 *bound_sin6;
826 			int ret = so6->so_proto->pr_sockaddr(so6,
827 			    (struct sockaddr **)&bound_sin6);
828 			if (ret)
829 				return (ret);
830 			port = ntohs(bound_sin6->sin6_port);
831 			free(bound_sin6, M_SONAME);
832 		}
833 	}
834 
835 	if (ret4 && ret6)
836 		return (ret4);
837 	*requested_port = port;
838 	if (ret4 && !ret6 && so4) {
839 		soclose(so4);
840 		*in_so4 = NULL;
841 	} else if (ret6 && !ret4 && so6) {
842 		soclose(so6);
843 		*in_so6 = NULL;
844 	}
845 	return (0);
846 }
847 
848 static int
849 wg_send(struct wg_softc *sc, struct wg_endpoint *e, struct mbuf *m)
850 {
851 	struct epoch_tracker et;
852 	struct sockaddr *sa;
853 	struct wg_socket *so = &sc->sc_socket;
854 	struct socket *so4, *so6;
855 	struct mbuf *control = NULL;
856 	int ret = 0;
857 	size_t len = m->m_pkthdr.len;
858 
859 	/* Get local control address before locking */
860 	if (e->e_remote.r_sa.sa_family == AF_INET) {
861 		if (e->e_local.l_in.s_addr != INADDR_ANY)
862 			control = sbcreatecontrol((caddr_t)&e->e_local.l_in,
863 			    sizeof(struct in_addr), IP_SENDSRCADDR,
864 			    IPPROTO_IP, M_NOWAIT);
865 #ifdef INET6
866 	} else if (e->e_remote.r_sa.sa_family == AF_INET6) {
867 		if (!IN6_IS_ADDR_UNSPECIFIED(&e->e_local.l_in6))
868 			control = sbcreatecontrol((caddr_t)&e->e_local.l_pktinfo6,
869 			    sizeof(struct in6_pktinfo), IPV6_PKTINFO,
870 			    IPPROTO_IPV6, M_NOWAIT);
871 #endif
872 	} else {
873 		m_freem(m);
874 		return (EAFNOSUPPORT);
875 	}
876 
877 	/* Get remote address */
878 	sa = &e->e_remote.r_sa;
879 
880 	NET_EPOCH_ENTER(et);
881 	so4 = atomic_load_ptr(&so->so_so4);
882 	so6 = atomic_load_ptr(&so->so_so6);
883 	if (e->e_remote.r_sa.sa_family == AF_INET && so4 != NULL)
884 		ret = sosend(so4, sa, NULL, m, control, 0, curthread);
885 	else if (e->e_remote.r_sa.sa_family == AF_INET6 && so6 != NULL)
886 		ret = sosend(so6, sa, NULL, m, control, 0, curthread);
887 	else {
888 		ret = ENOTCONN;
889 		m_freem(control);
890 		m_freem(m);
891 	}
892 	NET_EPOCH_EXIT(et);
893 	if (ret == 0) {
894 		if_inc_counter(sc->sc_ifp, IFCOUNTER_OPACKETS, 1);
895 		if_inc_counter(sc->sc_ifp, IFCOUNTER_OBYTES, len);
896 	}
897 	return (ret);
898 }
899 
900 static void
901 wg_send_buf(struct wg_softc *sc, struct wg_endpoint *e, uint8_t *buf, size_t len)
902 {
903 	struct mbuf	*m;
904 	int		 ret = 0;
905 	bool		 retried = false;
906 
907 retry:
908 	m = m_get2(len, M_NOWAIT, MT_DATA, M_PKTHDR);
909 	if (!m) {
910 		ret = ENOMEM;
911 		goto out;
912 	}
913 	m_copyback(m, 0, len, buf);
914 
915 	if (ret == 0) {
916 		ret = wg_send(sc, e, m);
917 		/* Retry if we couldn't bind to e->e_local */
918 		if (ret == EADDRNOTAVAIL && !retried) {
919 			bzero(&e->e_local, sizeof(e->e_local));
920 			retried = true;
921 			goto retry;
922 		}
923 	} else {
924 		ret = wg_send(sc, e, m);
925 	}
926 out:
927 	if (ret)
928 		DPRINTF(sc, "Unable to send packet: %d\n", ret);
929 }
930 
931 /* Timers */
932 static void
933 wg_timers_enable(struct wg_peer *peer)
934 {
935 	atomic_store_bool(&peer->p_enabled, true);
936 	wg_timers_run_persistent_keepalive(peer);
937 }
938 
939 static void
940 wg_timers_disable(struct wg_peer *peer)
941 {
942 	/* By setting p_enabled = false, then calling NET_EPOCH_WAIT, we can be
943 	 * sure no new handshakes are created after the wait. This is because
944 	 * all callout_resets (scheduling the callout) are guarded by
945 	 * p_enabled. We can be sure all sections that read p_enabled and then
946 	 * optionally call callout_reset are finished as they are surrounded by
947 	 * NET_EPOCH_{ENTER,EXIT}.
948 	 *
949 	 * However, as new callouts may be scheduled during NET_EPOCH_WAIT (but
950 	 * not after), we stop all callouts leaving no callouts active.
951 	 *
952 	 * We should also pull NET_EPOCH_WAIT out of the FOREACH(peer) loops, but the
953 	 * performance impact is acceptable for the time being. */
954 	atomic_store_bool(&peer->p_enabled, false);
955 	NET_EPOCH_WAIT();
956 	atomic_store_bool(&peer->p_need_another_keepalive, false);
957 
958 	callout_stop(&peer->p_new_handshake);
959 	callout_stop(&peer->p_send_keepalive);
960 	callout_stop(&peer->p_retry_handshake);
961 	callout_stop(&peer->p_persistent_keepalive);
962 	callout_stop(&peer->p_zero_key_material);
963 }
964 
965 static void
966 wg_timers_set_persistent_keepalive(struct wg_peer *peer, uint16_t interval)
967 {
968 	struct epoch_tracker et;
969 	if (interval != peer->p_persistent_keepalive_interval) {
970 		atomic_store_16(&peer->p_persistent_keepalive_interval, interval);
971 		NET_EPOCH_ENTER(et);
972 		if (atomic_load_bool(&peer->p_enabled))
973 			wg_timers_run_persistent_keepalive(peer);
974 		NET_EPOCH_EXIT(et);
975 	}
976 }
977 
978 static void
979 wg_timers_get_last_handshake(struct wg_peer *peer, struct wg_timespec64 *time)
980 {
981 	mtx_lock(&peer->p_handshake_mtx);
982 	time->tv_sec = peer->p_handshake_complete.tv_sec;
983 	time->tv_nsec = peer->p_handshake_complete.tv_nsec;
984 	mtx_unlock(&peer->p_handshake_mtx);
985 }
986 
987 static void
988 wg_timers_event_data_sent(struct wg_peer *peer)
989 {
990 	struct epoch_tracker et;
991 	NET_EPOCH_ENTER(et);
992 	if (atomic_load_bool(&peer->p_enabled) &&
993 	    !callout_pending(&peer->p_new_handshake))
994 		callout_reset(&peer->p_new_handshake, MSEC_2_TICKS(
995 		    NEW_HANDSHAKE_TIMEOUT * 1000 +
996 		    arc4random_uniform(REKEY_TIMEOUT_JITTER)),
997 		    wg_timers_run_new_handshake, peer);
998 	NET_EPOCH_EXIT(et);
999 }
1000 
1001 static void
1002 wg_timers_event_data_received(struct wg_peer *peer)
1003 {
1004 	struct epoch_tracker et;
1005 	NET_EPOCH_ENTER(et);
1006 	if (atomic_load_bool(&peer->p_enabled)) {
1007 		if (!callout_pending(&peer->p_send_keepalive))
1008 			callout_reset(&peer->p_send_keepalive,
1009 			    MSEC_2_TICKS(KEEPALIVE_TIMEOUT * 1000),
1010 			    wg_timers_run_send_keepalive, peer);
1011 		else
1012 			atomic_store_bool(&peer->p_need_another_keepalive,
1013 			    true);
1014 	}
1015 	NET_EPOCH_EXIT(et);
1016 }
1017 
1018 static void
1019 wg_timers_event_any_authenticated_packet_sent(struct wg_peer *peer)
1020 {
1021 	callout_stop(&peer->p_send_keepalive);
1022 }
1023 
1024 static void
1025 wg_timers_event_any_authenticated_packet_received(struct wg_peer *peer)
1026 {
1027 	callout_stop(&peer->p_new_handshake);
1028 }
1029 
1030 static void
1031 wg_timers_event_any_authenticated_packet_traversal(struct wg_peer *peer)
1032 {
1033 	struct epoch_tracker et;
1034 	uint16_t interval;
1035 	NET_EPOCH_ENTER(et);
1036 	interval = atomic_load_16(&peer->p_persistent_keepalive_interval);
1037 	if (atomic_load_bool(&peer->p_enabled) && interval > 0)
1038 		callout_reset(&peer->p_persistent_keepalive,
1039 		     MSEC_2_TICKS(interval * 1000),
1040 		     wg_timers_run_persistent_keepalive, peer);
1041 	NET_EPOCH_EXIT(et);
1042 }
1043 
1044 static void
1045 wg_timers_event_handshake_initiated(struct wg_peer *peer)
1046 {
1047 	struct epoch_tracker et;
1048 	NET_EPOCH_ENTER(et);
1049 	if (atomic_load_bool(&peer->p_enabled))
1050 		callout_reset(&peer->p_retry_handshake, MSEC_2_TICKS(
1051 		    REKEY_TIMEOUT * 1000 +
1052 		    arc4random_uniform(REKEY_TIMEOUT_JITTER)),
1053 		    wg_timers_run_retry_handshake, peer);
1054 	NET_EPOCH_EXIT(et);
1055 }
1056 
1057 static void
1058 wg_timers_event_handshake_complete(struct wg_peer *peer)
1059 {
1060 	struct epoch_tracker et;
1061 	NET_EPOCH_ENTER(et);
1062 	if (atomic_load_bool(&peer->p_enabled)) {
1063 		mtx_lock(&peer->p_handshake_mtx);
1064 		callout_stop(&peer->p_retry_handshake);
1065 		peer->p_handshake_retries = 0;
1066 		getnanotime(&peer->p_handshake_complete);
1067 		mtx_unlock(&peer->p_handshake_mtx);
1068 		wg_timers_run_send_keepalive(peer);
1069 	}
1070 	NET_EPOCH_EXIT(et);
1071 }
1072 
1073 static void
1074 wg_timers_event_session_derived(struct wg_peer *peer)
1075 {
1076 	struct epoch_tracker et;
1077 	NET_EPOCH_ENTER(et);
1078 	if (atomic_load_bool(&peer->p_enabled))
1079 		callout_reset(&peer->p_zero_key_material,
1080 		    MSEC_2_TICKS(REJECT_AFTER_TIME * 3 * 1000),
1081 		    wg_timers_run_zero_key_material, peer);
1082 	NET_EPOCH_EXIT(et);
1083 }
1084 
1085 static void
1086 wg_timers_event_want_initiation(struct wg_peer *peer)
1087 {
1088 	struct epoch_tracker et;
1089 	NET_EPOCH_ENTER(et);
1090 	if (atomic_load_bool(&peer->p_enabled))
1091 		wg_timers_run_send_initiation(peer, false);
1092 	NET_EPOCH_EXIT(et);
1093 }
1094 
1095 static void
1096 wg_timers_run_send_initiation(struct wg_peer *peer, bool is_retry)
1097 {
1098 	if (!is_retry)
1099 		peer->p_handshake_retries = 0;
1100 	if (noise_remote_initiation_expired(peer->p_remote) == ETIMEDOUT)
1101 		wg_send_initiation(peer);
1102 }
1103 
1104 static void
1105 wg_timers_run_retry_handshake(void *_peer)
1106 {
1107 	struct epoch_tracker et;
1108 	struct wg_peer *peer = _peer;
1109 
1110 	mtx_lock(&peer->p_handshake_mtx);
1111 	if (peer->p_handshake_retries <= MAX_TIMER_HANDSHAKES) {
1112 		peer->p_handshake_retries++;
1113 		mtx_unlock(&peer->p_handshake_mtx);
1114 
1115 		DPRINTF(peer->p_sc, "Handshake for peer %" PRIu64 " did not complete "
1116 		    "after %d seconds, retrying (try %d)\n", peer->p_id,
1117 		    REKEY_TIMEOUT, peer->p_handshake_retries + 1);
1118 		wg_peer_clear_src(peer);
1119 		wg_timers_run_send_initiation(peer, true);
1120 	} else {
1121 		mtx_unlock(&peer->p_handshake_mtx);
1122 
1123 		DPRINTF(peer->p_sc, "Handshake for peer %" PRIu64 " did not complete "
1124 		    "after %d retries, giving up\n", peer->p_id,
1125 		    MAX_TIMER_HANDSHAKES + 2);
1126 
1127 		callout_stop(&peer->p_send_keepalive);
1128 		wg_queue_purge(&peer->p_stage_queue);
1129 		NET_EPOCH_ENTER(et);
1130 		if (atomic_load_bool(&peer->p_enabled) &&
1131 		    !callout_pending(&peer->p_zero_key_material))
1132 			callout_reset(&peer->p_zero_key_material,
1133 			    MSEC_2_TICKS(REJECT_AFTER_TIME * 3 * 1000),
1134 			    wg_timers_run_zero_key_material, peer);
1135 		NET_EPOCH_EXIT(et);
1136 	}
1137 }
1138 
1139 static void
1140 wg_timers_run_send_keepalive(void *_peer)
1141 {
1142 	struct epoch_tracker et;
1143 	struct wg_peer *peer = _peer;
1144 
1145 	wg_send_keepalive(peer);
1146 	NET_EPOCH_ENTER(et);
1147 	if (atomic_load_bool(&peer->p_enabled) &&
1148 	    atomic_load_bool(&peer->p_need_another_keepalive)) {
1149 		atomic_store_bool(&peer->p_need_another_keepalive, false);
1150 		callout_reset(&peer->p_send_keepalive,
1151 		    MSEC_2_TICKS(KEEPALIVE_TIMEOUT * 1000),
1152 		    wg_timers_run_send_keepalive, peer);
1153 	}
1154 	NET_EPOCH_EXIT(et);
1155 }
1156 
1157 static void
1158 wg_timers_run_new_handshake(void *_peer)
1159 {
1160 	struct wg_peer *peer = _peer;
1161 
1162 	DPRINTF(peer->p_sc, "Retrying handshake with peer %" PRIu64 " because we "
1163 	    "stopped hearing back after %d seconds\n",
1164 	    peer->p_id, NEW_HANDSHAKE_TIMEOUT);
1165 
1166 	wg_peer_clear_src(peer);
1167 	wg_timers_run_send_initiation(peer, false);
1168 }
1169 
1170 static void
1171 wg_timers_run_zero_key_material(void *_peer)
1172 {
1173 	struct wg_peer *peer = _peer;
1174 
1175 	DPRINTF(peer->p_sc, "Zeroing out keys for peer %" PRIu64 ", since we "
1176 	    "haven't received a new one in %d seconds\n",
1177 	    peer->p_id, REJECT_AFTER_TIME * 3);
1178 	noise_remote_keypairs_clear(peer->p_remote);
1179 }
1180 
1181 static void
1182 wg_timers_run_persistent_keepalive(void *_peer)
1183 {
1184 	struct wg_peer *peer = _peer;
1185 
1186 	if (atomic_load_16(&peer->p_persistent_keepalive_interval) > 0)
1187 		wg_send_keepalive(peer);
1188 }
1189 
1190 /* TODO Handshake */
1191 static void
1192 wg_peer_send_buf(struct wg_peer *peer, uint8_t *buf, size_t len)
1193 {
1194 	struct wg_endpoint endpoint;
1195 
1196 	counter_u64_add(peer->p_tx_bytes, len);
1197 	wg_timers_event_any_authenticated_packet_traversal(peer);
1198 	wg_timers_event_any_authenticated_packet_sent(peer);
1199 	wg_peer_get_endpoint(peer, &endpoint);
1200 	wg_send_buf(peer->p_sc, &endpoint, buf, len);
1201 }
1202 
1203 static void
1204 wg_send_initiation(struct wg_peer *peer)
1205 {
1206 	struct wg_pkt_initiation pkt;
1207 
1208 	if (noise_create_initiation(peer->p_remote, &pkt.s_idx, pkt.ue,
1209 	    pkt.es, pkt.ets) != 0)
1210 		return;
1211 
1212 	DPRINTF(peer->p_sc, "Sending handshake initiation to peer %" PRIu64 "\n", peer->p_id);
1213 
1214 	pkt.t = WG_PKT_INITIATION;
1215 	cookie_maker_mac(&peer->p_cookie, &pkt.m, &pkt,
1216 	    sizeof(pkt) - sizeof(pkt.m));
1217 	wg_peer_send_buf(peer, (uint8_t *)&pkt, sizeof(pkt));
1218 	wg_timers_event_handshake_initiated(peer);
1219 }
1220 
1221 static void
1222 wg_send_response(struct wg_peer *peer)
1223 {
1224 	struct wg_pkt_response pkt;
1225 
1226 	if (noise_create_response(peer->p_remote, &pkt.s_idx, &pkt.r_idx,
1227 	    pkt.ue, pkt.en) != 0)
1228 		return;
1229 
1230 	DPRINTF(peer->p_sc, "Sending handshake response to peer %" PRIu64 "\n", peer->p_id);
1231 
1232 	wg_timers_event_session_derived(peer);
1233 	pkt.t = WG_PKT_RESPONSE;
1234 	cookie_maker_mac(&peer->p_cookie, &pkt.m, &pkt,
1235 	     sizeof(pkt)-sizeof(pkt.m));
1236 	wg_peer_send_buf(peer, (uint8_t*)&pkt, sizeof(pkt));
1237 }
1238 
1239 static void
1240 wg_send_cookie(struct wg_softc *sc, struct cookie_macs *cm, uint32_t idx,
1241     struct wg_endpoint *e)
1242 {
1243 	struct wg_pkt_cookie	pkt;
1244 
1245 	DPRINTF(sc, "Sending cookie response for denied handshake message\n");
1246 
1247 	pkt.t = WG_PKT_COOKIE;
1248 	pkt.r_idx = idx;
1249 
1250 	cookie_checker_create_payload(&sc->sc_cookie, cm, pkt.nonce,
1251 	    pkt.ec, &e->e_remote.r_sa);
1252 	wg_send_buf(sc, e, (uint8_t *)&pkt, sizeof(pkt));
1253 }
1254 
1255 static void
1256 wg_send_keepalive(struct wg_peer *peer)
1257 {
1258 	struct wg_packet *pkt;
1259 	struct mbuf *m;
1260 
1261 	if (wg_queue_len(&peer->p_stage_queue) > 0)
1262 		goto send;
1263 	if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
1264 		return;
1265 	if ((pkt = wg_packet_alloc(m)) == NULL) {
1266 		m_freem(m);
1267 		return;
1268 	}
1269 	wg_queue_push_staged(&peer->p_stage_queue, pkt);
1270 	DPRINTF(peer->p_sc, "Sending keepalive packet to peer %" PRIu64 "\n", peer->p_id);
1271 send:
1272 	wg_peer_send_staged(peer);
1273 }
1274 
1275 static void
1276 wg_handshake(struct wg_softc *sc, struct wg_packet *pkt)
1277 {
1278 	struct wg_pkt_initiation	*init;
1279 	struct wg_pkt_response		*resp;
1280 	struct wg_pkt_cookie		*cook;
1281 	struct wg_endpoint		*e;
1282 	struct wg_peer			*peer;
1283 	struct mbuf			*m;
1284 	struct noise_remote		*remote = NULL;
1285 	int				 res;
1286 	bool				 underload = false;
1287 	static sbintime_t		 wg_last_underload; /* sbinuptime */
1288 
1289 	underload = wg_queue_len(&sc->sc_handshake_queue) >= MAX_QUEUED_HANDSHAKES / 8;
1290 	if (underload) {
1291 		wg_last_underload = getsbinuptime();
1292 	} else if (wg_last_underload) {
1293 		underload = wg_last_underload + UNDERLOAD_TIMEOUT * SBT_1S > getsbinuptime();
1294 		if (!underload)
1295 			wg_last_underload = 0;
1296 	}
1297 
1298 	m = pkt->p_mbuf;
1299 	e = &pkt->p_endpoint;
1300 
1301 	if ((pkt->p_mbuf = m = m_pullup(m, m->m_pkthdr.len)) == NULL)
1302 		goto error;
1303 
1304 	switch (*mtod(m, uint32_t *)) {
1305 	case WG_PKT_INITIATION:
1306 		init = mtod(m, struct wg_pkt_initiation *);
1307 
1308 		res = cookie_checker_validate_macs(&sc->sc_cookie, &init->m,
1309 				init, sizeof(*init) - sizeof(init->m),
1310 				underload, &e->e_remote.r_sa,
1311 				if_getvnet(sc->sc_ifp));
1312 
1313 		if (res == EINVAL) {
1314 			DPRINTF(sc, "Invalid initiation MAC\n");
1315 			goto error;
1316 		} else if (res == ECONNREFUSED) {
1317 			DPRINTF(sc, "Handshake ratelimited\n");
1318 			goto error;
1319 		} else if (res == EAGAIN) {
1320 			wg_send_cookie(sc, &init->m, init->s_idx, e);
1321 			goto error;
1322 		} else if (res != 0) {
1323 			panic("unexpected response: %d\n", res);
1324 		}
1325 
1326 		if (noise_consume_initiation(sc->sc_local, &remote,
1327 		    init->s_idx, init->ue, init->es, init->ets) != 0) {
1328 			DPRINTF(sc, "Invalid handshake initiation\n");
1329 			goto error;
1330 		}
1331 
1332 		peer = noise_remote_arg(remote);
1333 
1334 		DPRINTF(sc, "Receiving handshake initiation from peer %" PRIu64 "\n", peer->p_id);
1335 
1336 		wg_peer_set_endpoint(peer, e);
1337 		wg_send_response(peer);
1338 		break;
1339 	case WG_PKT_RESPONSE:
1340 		resp = mtod(m, struct wg_pkt_response *);
1341 
1342 		res = cookie_checker_validate_macs(&sc->sc_cookie, &resp->m,
1343 				resp, sizeof(*resp) - sizeof(resp->m),
1344 				underload, &e->e_remote.r_sa,
1345 				if_getvnet(sc->sc_ifp));
1346 
1347 		if (res == EINVAL) {
1348 			DPRINTF(sc, "Invalid response MAC\n");
1349 			goto error;
1350 		} else if (res == ECONNREFUSED) {
1351 			DPRINTF(sc, "Handshake ratelimited\n");
1352 			goto error;
1353 		} else if (res == EAGAIN) {
1354 			wg_send_cookie(sc, &resp->m, resp->s_idx, e);
1355 			goto error;
1356 		} else if (res != 0) {
1357 			panic("unexpected response: %d\n", res);
1358 		}
1359 
1360 		if (noise_consume_response(sc->sc_local, &remote,
1361 		    resp->s_idx, resp->r_idx, resp->ue, resp->en) != 0) {
1362 			DPRINTF(sc, "Invalid handshake response\n");
1363 			goto error;
1364 		}
1365 
1366 		peer = noise_remote_arg(remote);
1367 		DPRINTF(sc, "Receiving handshake response from peer %" PRIu64 "\n", peer->p_id);
1368 
1369 		wg_peer_set_endpoint(peer, e);
1370 		wg_timers_event_session_derived(peer);
1371 		wg_timers_event_handshake_complete(peer);
1372 		break;
1373 	case WG_PKT_COOKIE:
1374 		cook = mtod(m, struct wg_pkt_cookie *);
1375 
1376 		if ((remote = noise_remote_index(sc->sc_local, cook->r_idx)) == NULL) {
1377 			DPRINTF(sc, "Unknown cookie index\n");
1378 			goto error;
1379 		}
1380 
1381 		peer = noise_remote_arg(remote);
1382 
1383 		if (cookie_maker_consume_payload(&peer->p_cookie,
1384 		    cook->nonce, cook->ec) == 0) {
1385 			DPRINTF(sc, "Receiving cookie response\n");
1386 		} else {
1387 			DPRINTF(sc, "Could not decrypt cookie response\n");
1388 			goto error;
1389 		}
1390 
1391 		goto not_authenticated;
1392 	default:
1393 		panic("invalid packet in handshake queue");
1394 	}
1395 
1396 	wg_timers_event_any_authenticated_packet_received(peer);
1397 	wg_timers_event_any_authenticated_packet_traversal(peer);
1398 
1399 not_authenticated:
1400 	counter_u64_add(peer->p_rx_bytes, m->m_pkthdr.len);
1401 	if_inc_counter(sc->sc_ifp, IFCOUNTER_IPACKETS, 1);
1402 	if_inc_counter(sc->sc_ifp, IFCOUNTER_IBYTES, m->m_pkthdr.len);
1403 error:
1404 	if (remote != NULL)
1405 		noise_remote_put(remote);
1406 	wg_packet_free(pkt);
1407 }
1408 
1409 static void
1410 wg_softc_handshake_receive(struct wg_softc *sc)
1411 {
1412 	struct wg_packet *pkt;
1413 	while ((pkt = wg_queue_dequeue_handshake(&sc->sc_handshake_queue)) != NULL)
1414 		wg_handshake(sc, pkt);
1415 }
1416 
1417 static void
1418 wg_mbuf_reset(struct mbuf *m)
1419 {
1420 
1421 	struct m_tag *t, *tmp;
1422 
1423 	/*
1424 	 * We want to reset the mbuf to a newly allocated state, containing
1425 	 * just the packet contents. Unfortunately FreeBSD doesn't seem to
1426 	 * offer this anywhere, so we have to make it up as we go. If we can
1427 	 * get this in kern/kern_mbuf.c, that would be best.
1428 	 *
1429 	 * Notice: this may break things unexpectedly but it is better to fail
1430 	 *         closed in the extreme case than leak informtion in every
1431 	 *         case.
1432 	 *
1433 	 * With that said, all this attempts to do is remove any extraneous
1434 	 * information that could be present.
1435 	 */
1436 
1437 	M_ASSERTPKTHDR(m);
1438 
1439 	m->m_flags &= ~(M_BCAST|M_MCAST|M_VLANTAG|M_PROMISC|M_PROTOFLAGS);
1440 
1441 	M_HASHTYPE_CLEAR(m);
1442 #ifdef NUMA
1443         m->m_pkthdr.numa_domain = M_NODOM;
1444 #endif
1445 	SLIST_FOREACH_SAFE(t, &m->m_pkthdr.tags, m_tag_link, tmp) {
1446 		if ((t->m_tag_id != 0 || t->m_tag_cookie != MTAG_WGLOOP) &&
1447 		    t->m_tag_id != PACKET_TAG_MACLABEL)
1448 			m_tag_delete(m, t);
1449 	}
1450 
1451 	KASSERT((m->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0,
1452 	    ("%s: mbuf %p has a send tag", __func__, m));
1453 
1454 	m->m_pkthdr.csum_flags = 0;
1455 	m->m_pkthdr.PH_per.sixtyfour[0] = 0;
1456 	m->m_pkthdr.PH_loc.sixtyfour[0] = 0;
1457 }
1458 
1459 static inline unsigned int
1460 calculate_padding(struct wg_packet *pkt)
1461 {
1462 	unsigned int padded_size, last_unit = pkt->p_mbuf->m_pkthdr.len;
1463 
1464 	if (__predict_false(!pkt->p_mtu))
1465 		return (last_unit + (WG_PKT_PADDING - 1)) & ~(WG_PKT_PADDING - 1);
1466 
1467 	if (__predict_false(last_unit > pkt->p_mtu))
1468 		last_unit %= pkt->p_mtu;
1469 
1470 	padded_size = (last_unit + (WG_PKT_PADDING - 1)) & ~(WG_PKT_PADDING - 1);
1471 	if (pkt->p_mtu < padded_size)
1472 		padded_size = pkt->p_mtu;
1473 	return padded_size - last_unit;
1474 }
1475 
1476 static void
1477 wg_encrypt(struct wg_softc *sc, struct wg_packet *pkt)
1478 {
1479 	static const uint8_t	 padding[WG_PKT_PADDING] = { 0 };
1480 	struct wg_pkt_data	*data;
1481 	struct wg_peer		*peer;
1482 	struct noise_remote	*remote;
1483 	struct mbuf		*m;
1484 	uint32_t		 idx;
1485 	unsigned int		 padlen;
1486 	enum wg_ring_state	 state = WG_PACKET_DEAD;
1487 
1488 	remote = noise_keypair_remote(pkt->p_keypair);
1489 	peer = noise_remote_arg(remote);
1490 	m = pkt->p_mbuf;
1491 
1492 	/* Pad the packet */
1493 	padlen = calculate_padding(pkt);
1494 	if (padlen != 0 && !m_append(m, padlen, padding))
1495 		goto out;
1496 
1497 	/* Do encryption */
1498 	if (noise_keypair_encrypt(pkt->p_keypair, &idx, pkt->p_nonce, m) != 0)
1499 		goto out;
1500 
1501 	/* Put header into packet */
1502 	M_PREPEND(m, sizeof(struct wg_pkt_data), M_NOWAIT);
1503 	if (m == NULL)
1504 		goto out;
1505 	data = mtod(m, struct wg_pkt_data *);
1506 	data->t = WG_PKT_DATA;
1507 	data->r_idx = idx;
1508 	data->nonce = htole64(pkt->p_nonce);
1509 
1510 	wg_mbuf_reset(m);
1511 	state = WG_PACKET_CRYPTED;
1512 out:
1513 	pkt->p_mbuf = m;
1514 	wmb();
1515 	pkt->p_state = state;
1516 	GROUPTASK_ENQUEUE(&peer->p_send);
1517 	noise_remote_put(remote);
1518 }
1519 
1520 static void
1521 wg_decrypt(struct wg_softc *sc, struct wg_packet *pkt)
1522 {
1523 	struct wg_peer		*peer, *allowed_peer;
1524 	struct noise_remote	*remote;
1525 	struct mbuf		*m;
1526 	int			 len;
1527 	enum wg_ring_state	 state = WG_PACKET_DEAD;
1528 
1529 	remote = noise_keypair_remote(pkt->p_keypair);
1530 	peer = noise_remote_arg(remote);
1531 	m = pkt->p_mbuf;
1532 
1533 	/* Read nonce and then adjust to remove the header. */
1534 	pkt->p_nonce = le64toh(mtod(m, struct wg_pkt_data *)->nonce);
1535 	m_adj(m, sizeof(struct wg_pkt_data));
1536 
1537 	if (noise_keypair_decrypt(pkt->p_keypair, pkt->p_nonce, m) != 0)
1538 		goto out;
1539 
1540 	/* A packet with length 0 is a keepalive packet */
1541 	if (__predict_false(m->m_pkthdr.len == 0)) {
1542 		DPRINTF(sc, "Receiving keepalive packet from peer "
1543 		    "%" PRIu64 "\n", peer->p_id);
1544 		state = WG_PACKET_CRYPTED;
1545 		goto out;
1546 	}
1547 
1548 	/*
1549 	 * We can let the network stack handle the intricate validation of the
1550 	 * IP header, we just worry about the sizeof and the version, so we can
1551 	 * read the source address in wg_aip_lookup.
1552 	 */
1553 
1554 	if (determine_af_and_pullup(&m, &pkt->p_af) == 0) {
1555 		if (pkt->p_af == AF_INET) {
1556 			struct ip *ip = mtod(m, struct ip *);
1557 			allowed_peer = wg_aip_lookup(sc, AF_INET, &ip->ip_src);
1558 			len = ntohs(ip->ip_len);
1559 			if (len >= sizeof(struct ip) && len < m->m_pkthdr.len)
1560 				m_adj(m, len - m->m_pkthdr.len);
1561 		} else if (pkt->p_af == AF_INET6) {
1562 			struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
1563 			allowed_peer = wg_aip_lookup(sc, AF_INET6, &ip6->ip6_src);
1564 			len = ntohs(ip6->ip6_plen) + sizeof(struct ip6_hdr);
1565 			if (len < m->m_pkthdr.len)
1566 				m_adj(m, len - m->m_pkthdr.len);
1567 		} else
1568 			panic("determine_af_and_pullup returned unexpected value");
1569 	} else {
1570 		DPRINTF(sc, "Packet is neither ipv4 nor ipv6 from peer %" PRIu64 "\n", peer->p_id);
1571 		goto out;
1572 	}
1573 
1574 	/* We only want to compare the address, not dereference, so drop the ref. */
1575 	if (allowed_peer != NULL)
1576 		noise_remote_put(allowed_peer->p_remote);
1577 
1578 	if (__predict_false(peer != allowed_peer)) {
1579 		DPRINTF(sc, "Packet has unallowed src IP from peer %" PRIu64 "\n", peer->p_id);
1580 		goto out;
1581 	}
1582 
1583 	wg_mbuf_reset(m);
1584 	state = WG_PACKET_CRYPTED;
1585 out:
1586 	pkt->p_mbuf = m;
1587 	wmb();
1588 	pkt->p_state = state;
1589 	GROUPTASK_ENQUEUE(&peer->p_recv);
1590 	noise_remote_put(remote);
1591 }
1592 
1593 static void
1594 wg_softc_decrypt(struct wg_softc *sc)
1595 {
1596 	struct wg_packet *pkt;
1597 
1598 	while ((pkt = wg_queue_dequeue_parallel(&sc->sc_decrypt_parallel)) != NULL)
1599 		wg_decrypt(sc, pkt);
1600 }
1601 
1602 static void
1603 wg_softc_encrypt(struct wg_softc *sc)
1604 {
1605 	struct wg_packet *pkt;
1606 
1607 	while ((pkt = wg_queue_dequeue_parallel(&sc->sc_encrypt_parallel)) != NULL)
1608 		wg_encrypt(sc, pkt);
1609 }
1610 
1611 static void
1612 wg_encrypt_dispatch(struct wg_softc *sc)
1613 {
1614 	/*
1615 	 * The update to encrypt_last_cpu is racey such that we may
1616 	 * reschedule the task for the same CPU multiple times, but
1617 	 * the race doesn't really matter.
1618 	 */
1619 	u_int cpu = (sc->sc_encrypt_last_cpu + 1) % mp_ncpus;
1620 	sc->sc_encrypt_last_cpu = cpu;
1621 	GROUPTASK_ENQUEUE(&sc->sc_encrypt[cpu]);
1622 }
1623 
1624 static void
1625 wg_decrypt_dispatch(struct wg_softc *sc)
1626 {
1627 	u_int cpu = (sc->sc_decrypt_last_cpu + 1) % mp_ncpus;
1628 	sc->sc_decrypt_last_cpu = cpu;
1629 	GROUPTASK_ENQUEUE(&sc->sc_decrypt[cpu]);
1630 }
1631 
1632 static void
1633 wg_deliver_out(struct wg_peer *peer)
1634 {
1635 	struct wg_endpoint	 endpoint;
1636 	struct wg_softc		*sc = peer->p_sc;
1637 	struct wg_packet	*pkt;
1638 	struct mbuf		*m;
1639 	int			 rc, len;
1640 
1641 	wg_peer_get_endpoint(peer, &endpoint);
1642 
1643 	while ((pkt = wg_queue_dequeue_serial(&peer->p_encrypt_serial)) != NULL) {
1644 		if (pkt->p_state != WG_PACKET_CRYPTED)
1645 			goto error;
1646 
1647 		m = pkt->p_mbuf;
1648 		pkt->p_mbuf = NULL;
1649 
1650 		len = m->m_pkthdr.len;
1651 
1652 		wg_timers_event_any_authenticated_packet_traversal(peer);
1653 		wg_timers_event_any_authenticated_packet_sent(peer);
1654 		rc = wg_send(sc, &endpoint, m);
1655 		if (rc == 0) {
1656 			if (len > (sizeof(struct wg_pkt_data) + NOISE_AUTHTAG_LEN))
1657 				wg_timers_event_data_sent(peer);
1658 			counter_u64_add(peer->p_tx_bytes, len);
1659 		} else if (rc == EADDRNOTAVAIL) {
1660 			wg_peer_clear_src(peer);
1661 			wg_peer_get_endpoint(peer, &endpoint);
1662 			goto error;
1663 		} else {
1664 			goto error;
1665 		}
1666 		wg_packet_free(pkt);
1667 		if (noise_keep_key_fresh_send(peer->p_remote))
1668 			wg_timers_event_want_initiation(peer);
1669 		continue;
1670 error:
1671 		if_inc_counter(sc->sc_ifp, IFCOUNTER_OERRORS, 1);
1672 		wg_packet_free(pkt);
1673 	}
1674 }
1675 
1676 static void
1677 wg_deliver_in(struct wg_peer *peer)
1678 {
1679 	struct wg_softc		*sc = peer->p_sc;
1680 	if_t			 ifp = sc->sc_ifp;
1681 	struct wg_packet	*pkt;
1682 	struct mbuf		*m;
1683 	struct epoch_tracker	 et;
1684 
1685 	while ((pkt = wg_queue_dequeue_serial(&peer->p_decrypt_serial)) != NULL) {
1686 		if (pkt->p_state != WG_PACKET_CRYPTED)
1687 			goto error;
1688 
1689 		m = pkt->p_mbuf;
1690 		if (noise_keypair_nonce_check(pkt->p_keypair, pkt->p_nonce) != 0)
1691 			goto error;
1692 
1693 		if (noise_keypair_received_with(pkt->p_keypair) == ECONNRESET)
1694 			wg_timers_event_handshake_complete(peer);
1695 
1696 		wg_timers_event_any_authenticated_packet_received(peer);
1697 		wg_timers_event_any_authenticated_packet_traversal(peer);
1698 		wg_peer_set_endpoint(peer, &pkt->p_endpoint);
1699 
1700 		counter_u64_add(peer->p_rx_bytes, m->m_pkthdr.len +
1701 		    sizeof(struct wg_pkt_data) + NOISE_AUTHTAG_LEN);
1702 		if_inc_counter(sc->sc_ifp, IFCOUNTER_IPACKETS, 1);
1703 		if_inc_counter(sc->sc_ifp, IFCOUNTER_IBYTES, m->m_pkthdr.len +
1704 		    sizeof(struct wg_pkt_data) + NOISE_AUTHTAG_LEN);
1705 
1706 		if (m->m_pkthdr.len == 0)
1707 			goto done;
1708 
1709 		MPASS(pkt->p_af == AF_INET || pkt->p_af == AF_INET6);
1710 		pkt->p_mbuf = NULL;
1711 
1712 		m->m_pkthdr.rcvif = ifp;
1713 
1714 		NET_EPOCH_ENTER(et);
1715 		BPF_MTAP2_AF(ifp, m, pkt->p_af);
1716 
1717 		CURVNET_SET(if_getvnet(ifp));
1718 		M_SETFIB(m, if_getfib(ifp));
1719 		if (pkt->p_af == AF_INET)
1720 			netisr_dispatch(NETISR_IP, m);
1721 		if (pkt->p_af == AF_INET6)
1722 			netisr_dispatch(NETISR_IPV6, m);
1723 		CURVNET_RESTORE();
1724 		NET_EPOCH_EXIT(et);
1725 
1726 		wg_timers_event_data_received(peer);
1727 
1728 done:
1729 		if (noise_keep_key_fresh_recv(peer->p_remote))
1730 			wg_timers_event_want_initiation(peer);
1731 		wg_packet_free(pkt);
1732 		continue;
1733 error:
1734 		if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
1735 		wg_packet_free(pkt);
1736 	}
1737 }
1738 
1739 static struct wg_packet *
1740 wg_packet_alloc(struct mbuf *m)
1741 {
1742 	struct wg_packet *pkt;
1743 
1744 	if ((pkt = uma_zalloc(wg_packet_zone, M_NOWAIT | M_ZERO)) == NULL)
1745 		return (NULL);
1746 	pkt->p_mbuf = m;
1747 	return (pkt);
1748 }
1749 
1750 static void
1751 wg_packet_free(struct wg_packet *pkt)
1752 {
1753 	if (pkt->p_keypair != NULL)
1754 		noise_keypair_put(pkt->p_keypair);
1755 	if (pkt->p_mbuf != NULL)
1756 		m_freem(pkt->p_mbuf);
1757 	uma_zfree(wg_packet_zone, pkt);
1758 }
1759 
1760 static void
1761 wg_queue_init(struct wg_queue *queue, const char *name)
1762 {
1763 	mtx_init(&queue->q_mtx, name, NULL, MTX_DEF);
1764 	STAILQ_INIT(&queue->q_queue);
1765 	queue->q_len = 0;
1766 }
1767 
1768 static void
1769 wg_queue_deinit(struct wg_queue *queue)
1770 {
1771 	wg_queue_purge(queue);
1772 	mtx_destroy(&queue->q_mtx);
1773 }
1774 
1775 static size_t
1776 wg_queue_len(struct wg_queue *queue)
1777 {
1778 	return (queue->q_len);
1779 }
1780 
1781 static int
1782 wg_queue_enqueue_handshake(struct wg_queue *hs, struct wg_packet *pkt)
1783 {
1784 	int ret = 0;
1785 	mtx_lock(&hs->q_mtx);
1786 	if (hs->q_len < MAX_QUEUED_HANDSHAKES) {
1787 		STAILQ_INSERT_TAIL(&hs->q_queue, pkt, p_parallel);
1788 		hs->q_len++;
1789 	} else {
1790 		ret = ENOBUFS;
1791 	}
1792 	mtx_unlock(&hs->q_mtx);
1793 	if (ret != 0)
1794 		wg_packet_free(pkt);
1795 	return (ret);
1796 }
1797 
1798 static struct wg_packet *
1799 wg_queue_dequeue_handshake(struct wg_queue *hs)
1800 {
1801 	struct wg_packet *pkt;
1802 	mtx_lock(&hs->q_mtx);
1803 	if ((pkt = STAILQ_FIRST(&hs->q_queue)) != NULL) {
1804 		STAILQ_REMOVE_HEAD(&hs->q_queue, p_parallel);
1805 		hs->q_len--;
1806 	}
1807 	mtx_unlock(&hs->q_mtx);
1808 	return (pkt);
1809 }
1810 
1811 static void
1812 wg_queue_push_staged(struct wg_queue *staged, struct wg_packet *pkt)
1813 {
1814 	struct wg_packet *old = NULL;
1815 
1816 	mtx_lock(&staged->q_mtx);
1817 	if (staged->q_len >= MAX_STAGED_PKT) {
1818 		old = STAILQ_FIRST(&staged->q_queue);
1819 		STAILQ_REMOVE_HEAD(&staged->q_queue, p_parallel);
1820 		staged->q_len--;
1821 	}
1822 	STAILQ_INSERT_TAIL(&staged->q_queue, pkt, p_parallel);
1823 	staged->q_len++;
1824 	mtx_unlock(&staged->q_mtx);
1825 
1826 	if (old != NULL)
1827 		wg_packet_free(old);
1828 }
1829 
1830 static void
1831 wg_queue_enlist_staged(struct wg_queue *staged, struct wg_packet_list *list)
1832 {
1833 	struct wg_packet *pkt, *tpkt;
1834 	STAILQ_FOREACH_SAFE(pkt, list, p_parallel, tpkt)
1835 		wg_queue_push_staged(staged, pkt);
1836 }
1837 
1838 static void
1839 wg_queue_delist_staged(struct wg_queue *staged, struct wg_packet_list *list)
1840 {
1841 	STAILQ_INIT(list);
1842 	mtx_lock(&staged->q_mtx);
1843 	STAILQ_CONCAT(list, &staged->q_queue);
1844 	staged->q_len = 0;
1845 	mtx_unlock(&staged->q_mtx);
1846 }
1847 
1848 static void
1849 wg_queue_purge(struct wg_queue *staged)
1850 {
1851 	struct wg_packet_list list;
1852 	struct wg_packet *pkt, *tpkt;
1853 	wg_queue_delist_staged(staged, &list);
1854 	STAILQ_FOREACH_SAFE(pkt, &list, p_parallel, tpkt)
1855 		wg_packet_free(pkt);
1856 }
1857 
1858 static int
1859 wg_queue_both(struct wg_queue *parallel, struct wg_queue *serial, struct wg_packet *pkt)
1860 {
1861 	pkt->p_state = WG_PACKET_UNCRYPTED;
1862 
1863 	mtx_lock(&serial->q_mtx);
1864 	if (serial->q_len < MAX_QUEUED_PKT) {
1865 		serial->q_len++;
1866 		STAILQ_INSERT_TAIL(&serial->q_queue, pkt, p_serial);
1867 	} else {
1868 		mtx_unlock(&serial->q_mtx);
1869 		wg_packet_free(pkt);
1870 		return (ENOBUFS);
1871 	}
1872 	mtx_unlock(&serial->q_mtx);
1873 
1874 	mtx_lock(&parallel->q_mtx);
1875 	if (parallel->q_len < MAX_QUEUED_PKT) {
1876 		parallel->q_len++;
1877 		STAILQ_INSERT_TAIL(&parallel->q_queue, pkt, p_parallel);
1878 	} else {
1879 		mtx_unlock(&parallel->q_mtx);
1880 		pkt->p_state = WG_PACKET_DEAD;
1881 		return (ENOBUFS);
1882 	}
1883 	mtx_unlock(&parallel->q_mtx);
1884 
1885 	return (0);
1886 }
1887 
1888 static struct wg_packet *
1889 wg_queue_dequeue_serial(struct wg_queue *serial)
1890 {
1891 	struct wg_packet *pkt = NULL;
1892 	mtx_lock(&serial->q_mtx);
1893 	if (serial->q_len > 0 && STAILQ_FIRST(&serial->q_queue)->p_state != WG_PACKET_UNCRYPTED) {
1894 		serial->q_len--;
1895 		pkt = STAILQ_FIRST(&serial->q_queue);
1896 		STAILQ_REMOVE_HEAD(&serial->q_queue, p_serial);
1897 	}
1898 	mtx_unlock(&serial->q_mtx);
1899 	return (pkt);
1900 }
1901 
1902 static struct wg_packet *
1903 wg_queue_dequeue_parallel(struct wg_queue *parallel)
1904 {
1905 	struct wg_packet *pkt = NULL;
1906 	mtx_lock(&parallel->q_mtx);
1907 	if (parallel->q_len > 0) {
1908 		parallel->q_len--;
1909 		pkt = STAILQ_FIRST(&parallel->q_queue);
1910 		STAILQ_REMOVE_HEAD(&parallel->q_queue, p_parallel);
1911 	}
1912 	mtx_unlock(&parallel->q_mtx);
1913 	return (pkt);
1914 }
1915 
1916 static bool
1917 wg_input(struct mbuf *m, int offset, struct inpcb *inpcb,
1918     const struct sockaddr *sa, void *_sc)
1919 {
1920 #ifdef INET
1921 	const struct sockaddr_in	*sin;
1922 #endif
1923 #ifdef INET6
1924 	const struct sockaddr_in6	*sin6;
1925 #endif
1926 	struct noise_remote		*remote;
1927 	struct wg_pkt_data		*data;
1928 	struct wg_packet		*pkt;
1929 	struct wg_peer			*peer;
1930 	struct wg_softc			*sc = _sc;
1931 	struct mbuf			*defragged;
1932 
1933 	defragged = m_defrag(m, M_NOWAIT);
1934 	if (defragged)
1935 		m = defragged;
1936 	m = m_unshare(m, M_NOWAIT);
1937 	if (!m) {
1938 		if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1);
1939 		return true;
1940 	}
1941 
1942 	/* Caller provided us with `sa`, no need for this header. */
1943 	m_adj(m, offset + sizeof(struct udphdr));
1944 
1945 	/* Pullup enough to read packet type */
1946 	if ((m = m_pullup(m, sizeof(uint32_t))) == NULL) {
1947 		if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1);
1948 		return true;
1949 	}
1950 
1951 	if ((pkt = wg_packet_alloc(m)) == NULL) {
1952 		if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1);
1953 		m_freem(m);
1954 		return true;
1955 	}
1956 
1957 	/* Save send/recv address and port for later. */
1958 	switch (sa->sa_family) {
1959 #ifdef INET
1960 	case AF_INET:
1961 		sin = (const struct sockaddr_in *)sa;
1962 		pkt->p_endpoint.e_remote.r_sin = sin[0];
1963 		pkt->p_endpoint.e_local.l_in = sin[1].sin_addr;
1964 		break;
1965 #endif
1966 #ifdef INET6
1967 	case AF_INET6:
1968 		sin6 = (const struct sockaddr_in6 *)sa;
1969 		pkt->p_endpoint.e_remote.r_sin6 = sin6[0];
1970 		pkt->p_endpoint.e_local.l_in6 = sin6[1].sin6_addr;
1971 		break;
1972 #endif
1973 	default:
1974 		goto error;
1975 	}
1976 
1977 	if ((m->m_pkthdr.len == sizeof(struct wg_pkt_initiation) &&
1978 		*mtod(m, uint32_t *) == WG_PKT_INITIATION) ||
1979 	    (m->m_pkthdr.len == sizeof(struct wg_pkt_response) &&
1980 		*mtod(m, uint32_t *) == WG_PKT_RESPONSE) ||
1981 	    (m->m_pkthdr.len == sizeof(struct wg_pkt_cookie) &&
1982 		*mtod(m, uint32_t *) == WG_PKT_COOKIE)) {
1983 
1984 		if (wg_queue_enqueue_handshake(&sc->sc_handshake_queue, pkt) != 0) {
1985 			if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1);
1986 			DPRINTF(sc, "Dropping handshake packet\n");
1987 		}
1988 		GROUPTASK_ENQUEUE(&sc->sc_handshake);
1989 	} else if (m->m_pkthdr.len >= sizeof(struct wg_pkt_data) +
1990 	    NOISE_AUTHTAG_LEN && *mtod(m, uint32_t *) == WG_PKT_DATA) {
1991 
1992 		/* Pullup whole header to read r_idx below. */
1993 		if ((pkt->p_mbuf = m_pullup(m, sizeof(struct wg_pkt_data))) == NULL)
1994 			goto error;
1995 
1996 		data = mtod(pkt->p_mbuf, struct wg_pkt_data *);
1997 		if ((pkt->p_keypair = noise_keypair_lookup(sc->sc_local, data->r_idx)) == NULL)
1998 			goto error;
1999 
2000 		remote = noise_keypair_remote(pkt->p_keypair);
2001 		peer = noise_remote_arg(remote);
2002 		if (wg_queue_both(&sc->sc_decrypt_parallel, &peer->p_decrypt_serial, pkt) != 0)
2003 			if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1);
2004 		wg_decrypt_dispatch(sc);
2005 		noise_remote_put(remote);
2006 	} else {
2007 		goto error;
2008 	}
2009 	return true;
2010 error:
2011 	if_inc_counter(sc->sc_ifp, IFCOUNTER_IERRORS, 1);
2012 	wg_packet_free(pkt);
2013 	return true;
2014 }
2015 
2016 static void
2017 wg_peer_send_staged(struct wg_peer *peer)
2018 {
2019 	struct wg_packet_list	 list;
2020 	struct noise_keypair	*keypair;
2021 	struct wg_packet	*pkt, *tpkt;
2022 	struct wg_softc		*sc = peer->p_sc;
2023 
2024 	wg_queue_delist_staged(&peer->p_stage_queue, &list);
2025 
2026 	if (STAILQ_EMPTY(&list))
2027 		return;
2028 
2029 	if ((keypair = noise_keypair_current(peer->p_remote)) == NULL)
2030 		goto error;
2031 
2032 	STAILQ_FOREACH(pkt, &list, p_parallel) {
2033 		if (noise_keypair_nonce_next(keypair, &pkt->p_nonce) != 0)
2034 			goto error_keypair;
2035 	}
2036 	STAILQ_FOREACH_SAFE(pkt, &list, p_parallel, tpkt) {
2037 		pkt->p_keypair = noise_keypair_ref(keypair);
2038 		if (wg_queue_both(&sc->sc_encrypt_parallel, &peer->p_encrypt_serial, pkt) != 0)
2039 			if_inc_counter(sc->sc_ifp, IFCOUNTER_OQDROPS, 1);
2040 	}
2041 	wg_encrypt_dispatch(sc);
2042 	noise_keypair_put(keypair);
2043 	return;
2044 
2045 error_keypair:
2046 	noise_keypair_put(keypair);
2047 error:
2048 	wg_queue_enlist_staged(&peer->p_stage_queue, &list);
2049 	wg_timers_event_want_initiation(peer);
2050 }
2051 
2052 static inline void
2053 xmit_err(if_t ifp, struct mbuf *m, struct wg_packet *pkt, sa_family_t af)
2054 {
2055 	if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
2056 	switch (af) {
2057 #ifdef INET
2058 	case AF_INET:
2059 		icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_HOST, 0, 0);
2060 		if (pkt)
2061 			pkt->p_mbuf = NULL;
2062 		m = NULL;
2063 		break;
2064 #endif
2065 #ifdef INET6
2066 	case AF_INET6:
2067 		icmp6_error(m, ICMP6_DST_UNREACH, 0, 0);
2068 		if (pkt)
2069 			pkt->p_mbuf = NULL;
2070 		m = NULL;
2071 		break;
2072 #endif
2073 	}
2074 	if (pkt)
2075 		wg_packet_free(pkt);
2076 	else if (m)
2077 		m_freem(m);
2078 }
2079 
2080 static int
2081 wg_xmit(if_t ifp, struct mbuf *m, sa_family_t af, uint32_t mtu)
2082 {
2083 	struct wg_packet	*pkt = NULL;
2084 	struct wg_softc		*sc = if_getsoftc(ifp);
2085 	struct wg_peer		*peer;
2086 	int			 rc = 0;
2087 	sa_family_t		 peer_af;
2088 
2089 	/* Work around lifetime issue in the ipv6 mld code. */
2090 	if (__predict_false((if_getflags(ifp) & IFF_DYING) || !sc)) {
2091 		rc = ENXIO;
2092 		goto err_xmit;
2093 	}
2094 
2095 	if ((pkt = wg_packet_alloc(m)) == NULL) {
2096 		rc = ENOBUFS;
2097 		goto err_xmit;
2098 	}
2099 	pkt->p_mtu = mtu;
2100 	pkt->p_af = af;
2101 
2102 	if (af == AF_INET) {
2103 		peer = wg_aip_lookup(sc, AF_INET, &mtod(m, struct ip *)->ip_dst);
2104 	} else if (af == AF_INET6) {
2105 		peer = wg_aip_lookup(sc, AF_INET6, &mtod(m, struct ip6_hdr *)->ip6_dst);
2106 	} else {
2107 		rc = EAFNOSUPPORT;
2108 		goto err_xmit;
2109 	}
2110 
2111 	BPF_MTAP2_AF(ifp, m, pkt->p_af);
2112 
2113 	if (__predict_false(peer == NULL)) {
2114 		rc = ENOKEY;
2115 		goto err_xmit;
2116 	}
2117 
2118 	if (__predict_false(if_tunnel_check_nesting(ifp, m, MTAG_WGLOOP, MAX_LOOPS))) {
2119 		DPRINTF(sc, "Packet looped");
2120 		rc = ELOOP;
2121 		goto err_peer;
2122 	}
2123 
2124 	peer_af = peer->p_endpoint.e_remote.r_sa.sa_family;
2125 	if (__predict_false(peer_af != AF_INET && peer_af != AF_INET6)) {
2126 		DPRINTF(sc, "No valid endpoint has been configured or "
2127 			    "discovered for peer %" PRIu64 "\n", peer->p_id);
2128 		rc = EHOSTUNREACH;
2129 		goto err_peer;
2130 	}
2131 
2132 	wg_queue_push_staged(&peer->p_stage_queue, pkt);
2133 	wg_peer_send_staged(peer);
2134 	noise_remote_put(peer->p_remote);
2135 	return (0);
2136 
2137 err_peer:
2138 	noise_remote_put(peer->p_remote);
2139 err_xmit:
2140 	xmit_err(ifp, m, pkt, af);
2141 	return (rc);
2142 }
2143 
2144 static inline int
2145 determine_af_and_pullup(struct mbuf **m, sa_family_t *af)
2146 {
2147 	u_char ipv;
2148 	if ((*m)->m_pkthdr.len >= sizeof(struct ip6_hdr))
2149 		*m = m_pullup(*m, sizeof(struct ip6_hdr));
2150 	else if ((*m)->m_pkthdr.len >= sizeof(struct ip))
2151 		*m = m_pullup(*m, sizeof(struct ip));
2152 	else
2153 		return (EAFNOSUPPORT);
2154 	if (*m == NULL)
2155 		return (ENOBUFS);
2156 	ipv = mtod(*m, struct ip *)->ip_v;
2157 	if (ipv == 4)
2158 		*af = AF_INET;
2159 	else if (ipv == 6 && (*m)->m_pkthdr.len >= sizeof(struct ip6_hdr))
2160 		*af = AF_INET6;
2161 	else
2162 		return (EAFNOSUPPORT);
2163 	return (0);
2164 }
2165 
2166 static int
2167 wg_transmit(if_t ifp, struct mbuf *m)
2168 {
2169 	sa_family_t af;
2170 	int ret;
2171 	struct mbuf *defragged;
2172 
2173 	defragged = m_defrag(m, M_NOWAIT);
2174 	if (defragged)
2175 		m = defragged;
2176 	m = m_unshare(m, M_NOWAIT);
2177 	if (!m) {
2178 		xmit_err(ifp, m, NULL, AF_UNSPEC);
2179 		return (ENOBUFS);
2180 	}
2181 
2182 	ret = determine_af_and_pullup(&m, &af);
2183 	if (ret) {
2184 		xmit_err(ifp, m, NULL, AF_UNSPEC);
2185 		return (ret);
2186 	}
2187 	return (wg_xmit(ifp, m, af, if_getmtu(ifp)));
2188 }
2189 
2190 static int
2191 wg_output(if_t ifp, struct mbuf *m, const struct sockaddr *dst, struct route *ro)
2192 {
2193 	sa_family_t parsed_af;
2194 	uint32_t af, mtu;
2195 	int ret;
2196 	struct mbuf *defragged;
2197 
2198 	if (dst->sa_family == AF_UNSPEC)
2199 		memcpy(&af, dst->sa_data, sizeof(af));
2200 	else
2201 		af = dst->sa_family;
2202 	if (af == AF_UNSPEC) {
2203 		xmit_err(ifp, m, NULL, af);
2204 		return (EAFNOSUPPORT);
2205 	}
2206 
2207 	defragged = m_defrag(m, M_NOWAIT);
2208 	if (defragged)
2209 		m = defragged;
2210 	m = m_unshare(m, M_NOWAIT);
2211 	if (!m) {
2212 		xmit_err(ifp, m, NULL, AF_UNSPEC);
2213 		return (ENOBUFS);
2214 	}
2215 
2216 	ret = determine_af_and_pullup(&m, &parsed_af);
2217 	if (ret) {
2218 		xmit_err(ifp, m, NULL, AF_UNSPEC);
2219 		return (ret);
2220 	}
2221 	if (parsed_af != af) {
2222 		xmit_err(ifp, m, NULL, AF_UNSPEC);
2223 		return (EAFNOSUPPORT);
2224 	}
2225 	mtu = (ro != NULL && ro->ro_mtu > 0) ? ro->ro_mtu : if_getmtu(ifp);
2226 	return (wg_xmit(ifp, m, parsed_af, mtu));
2227 }
2228 
2229 static int
2230 wg_peer_add(struct wg_softc *sc, const nvlist_t *nvl)
2231 {
2232 	uint8_t			 public[WG_KEY_SIZE];
2233 	const void *pub_key, *preshared_key = NULL;
2234 	const struct sockaddr *endpoint;
2235 	int err;
2236 	size_t size;
2237 	struct noise_remote *remote;
2238 	struct wg_peer *peer = NULL;
2239 	bool need_insert = false;
2240 
2241 	sx_assert(&sc->sc_lock, SX_XLOCKED);
2242 
2243 	if (!nvlist_exists_binary(nvl, "public-key")) {
2244 		return (EINVAL);
2245 	}
2246 	pub_key = nvlist_get_binary(nvl, "public-key", &size);
2247 	if (size != WG_KEY_SIZE) {
2248 		return (EINVAL);
2249 	}
2250 	if (noise_local_keys(sc->sc_local, public, NULL) == 0 &&
2251 	    bcmp(public, pub_key, WG_KEY_SIZE) == 0) {
2252 		return (0); // Silently ignored; not actually a failure.
2253 	}
2254 	if ((remote = noise_remote_lookup(sc->sc_local, pub_key)) != NULL)
2255 		peer = noise_remote_arg(remote);
2256 	if (nvlist_exists_bool(nvl, "remove") &&
2257 		nvlist_get_bool(nvl, "remove")) {
2258 		if (remote != NULL) {
2259 			wg_peer_destroy(peer);
2260 			noise_remote_put(remote);
2261 		}
2262 		return (0);
2263 	}
2264 	if (nvlist_exists_bool(nvl, "replace-allowedips") &&
2265 		nvlist_get_bool(nvl, "replace-allowedips") &&
2266 	    peer != NULL) {
2267 
2268 		wg_aip_remove_all(sc, peer);
2269 	}
2270 	if (peer == NULL) {
2271 		peer = wg_peer_alloc(sc, pub_key);
2272 		need_insert = true;
2273 	}
2274 	if (nvlist_exists_binary(nvl, "endpoint")) {
2275 		endpoint = nvlist_get_binary(nvl, "endpoint", &size);
2276 		if (size > sizeof(peer->p_endpoint.e_remote)) {
2277 			err = EINVAL;
2278 			goto out;
2279 		}
2280 		memcpy(&peer->p_endpoint.e_remote, endpoint, size);
2281 	}
2282 	if (nvlist_exists_binary(nvl, "preshared-key")) {
2283 		preshared_key = nvlist_get_binary(nvl, "preshared-key", &size);
2284 		if (size != WG_KEY_SIZE) {
2285 			err = EINVAL;
2286 			goto out;
2287 		}
2288 		noise_remote_set_psk(peer->p_remote, preshared_key);
2289 	}
2290 	if (nvlist_exists_number(nvl, "persistent-keepalive-interval")) {
2291 		uint64_t pki = nvlist_get_number(nvl, "persistent-keepalive-interval");
2292 		if (pki > UINT16_MAX) {
2293 			err = EINVAL;
2294 			goto out;
2295 		}
2296 		wg_timers_set_persistent_keepalive(peer, pki);
2297 	}
2298 	if (nvlist_exists_nvlist_array(nvl, "allowed-ips")) {
2299 		const void *addr;
2300 		uint64_t cidr;
2301 		const nvlist_t * const * aipl;
2302 		size_t allowedip_count;
2303 
2304 		aipl = nvlist_get_nvlist_array(nvl, "allowed-ips", &allowedip_count);
2305 		for (size_t idx = 0; idx < allowedip_count; idx++) {
2306 			if (!nvlist_exists_number(aipl[idx], "cidr"))
2307 				continue;
2308 			cidr = nvlist_get_number(aipl[idx], "cidr");
2309 			if (nvlist_exists_binary(aipl[idx], "ipv4")) {
2310 				addr = nvlist_get_binary(aipl[idx], "ipv4", &size);
2311 				if (addr == NULL || cidr > 32 || size != sizeof(struct in_addr)) {
2312 					err = EINVAL;
2313 					goto out;
2314 				}
2315 				if ((err = wg_aip_add(sc, peer, AF_INET, addr, cidr)) != 0)
2316 					goto out;
2317 			} else if (nvlist_exists_binary(aipl[idx], "ipv6")) {
2318 				addr = nvlist_get_binary(aipl[idx], "ipv6", &size);
2319 				if (addr == NULL || cidr > 128 || size != sizeof(struct in6_addr)) {
2320 					err = EINVAL;
2321 					goto out;
2322 				}
2323 				if ((err = wg_aip_add(sc, peer, AF_INET6, addr, cidr)) != 0)
2324 					goto out;
2325 			} else {
2326 				continue;
2327 			}
2328 		}
2329 	}
2330 	if (need_insert) {
2331 		if ((err = noise_remote_enable(peer->p_remote)) != 0)
2332 			goto out;
2333 		TAILQ_INSERT_TAIL(&sc->sc_peers, peer, p_entry);
2334 		sc->sc_peers_num++;
2335 		if (if_getlinkstate(sc->sc_ifp) == LINK_STATE_UP)
2336 			wg_timers_enable(peer);
2337 	}
2338 	if (remote != NULL)
2339 		noise_remote_put(remote);
2340 	return (0);
2341 out:
2342 	if (need_insert) /* If we fail, only destroy if it was new. */
2343 		wg_peer_destroy(peer);
2344 	if (remote != NULL)
2345 		noise_remote_put(remote);
2346 	return (err);
2347 }
2348 
2349 static int
2350 wgc_set(struct wg_softc *sc, struct wg_data_io *wgd)
2351 {
2352 	uint8_t public[WG_KEY_SIZE], private[WG_KEY_SIZE];
2353 	if_t ifp;
2354 	void *nvlpacked;
2355 	nvlist_t *nvl;
2356 	ssize_t size;
2357 	int err;
2358 
2359 	ifp = sc->sc_ifp;
2360 	if (wgd->wgd_size == 0 || wgd->wgd_data == NULL)
2361 		return (EFAULT);
2362 
2363 	/* Can nvlists be streamed in? It's not nice to impose arbitrary limits like that but
2364 	 * there needs to be _some_ limitation. */
2365 	if (wgd->wgd_size >= UINT32_MAX / 2)
2366 		return (E2BIG);
2367 
2368 	nvlpacked = malloc(wgd->wgd_size, M_TEMP, M_WAITOK | M_ZERO);
2369 
2370 	err = copyin(wgd->wgd_data, nvlpacked, wgd->wgd_size);
2371 	if (err)
2372 		goto out;
2373 	nvl = nvlist_unpack(nvlpacked, wgd->wgd_size, 0);
2374 	if (nvl == NULL) {
2375 		err = EBADMSG;
2376 		goto out;
2377 	}
2378 	sx_xlock(&sc->sc_lock);
2379 	if (nvlist_exists_bool(nvl, "replace-peers") &&
2380 		nvlist_get_bool(nvl, "replace-peers"))
2381 		wg_peer_destroy_all(sc);
2382 	if (nvlist_exists_number(nvl, "listen-port")) {
2383 		uint64_t new_port = nvlist_get_number(nvl, "listen-port");
2384 		if (new_port > UINT16_MAX) {
2385 			err = EINVAL;
2386 			goto out_locked;
2387 		}
2388 		if (new_port != sc->sc_socket.so_port) {
2389 			if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
2390 				if ((err = wg_socket_init(sc, new_port)) != 0)
2391 					goto out_locked;
2392 			} else
2393 				sc->sc_socket.so_port = new_port;
2394 		}
2395 	}
2396 	if (nvlist_exists_binary(nvl, "private-key")) {
2397 		const void *key = nvlist_get_binary(nvl, "private-key", &size);
2398 		if (size != WG_KEY_SIZE) {
2399 			err = EINVAL;
2400 			goto out_locked;
2401 		}
2402 
2403 		if (noise_local_keys(sc->sc_local, NULL, private) != 0 ||
2404 		    timingsafe_bcmp(private, key, WG_KEY_SIZE) != 0) {
2405 			struct wg_peer *peer;
2406 
2407 			if (curve25519_generate_public(public, key)) {
2408 				/* Peer conflict: remove conflicting peer. */
2409 				struct noise_remote *remote;
2410 				if ((remote = noise_remote_lookup(sc->sc_local,
2411 				    public)) != NULL) {
2412 					peer = noise_remote_arg(remote);
2413 					wg_peer_destroy(peer);
2414 					noise_remote_put(remote);
2415 				}
2416 			}
2417 
2418 			/*
2419 			 * Set the private key and invalidate all existing
2420 			 * handshakes.
2421 			 */
2422 			/* Note: we might be removing the private key. */
2423 			noise_local_private(sc->sc_local, key);
2424 			if (noise_local_keys(sc->sc_local, NULL, NULL) == 0)
2425 				cookie_checker_update(&sc->sc_cookie, public);
2426 			else
2427 				cookie_checker_update(&sc->sc_cookie, NULL);
2428 		}
2429 	}
2430 	if (nvlist_exists_number(nvl, "user-cookie")) {
2431 		uint64_t user_cookie = nvlist_get_number(nvl, "user-cookie");
2432 		if (user_cookie > UINT32_MAX) {
2433 			err = EINVAL;
2434 			goto out_locked;
2435 		}
2436 		err = wg_socket_set_cookie(sc, user_cookie);
2437 		if (err)
2438 			goto out_locked;
2439 	}
2440 	if (nvlist_exists_nvlist_array(nvl, "peers")) {
2441 		size_t peercount;
2442 		const nvlist_t * const*nvl_peers;
2443 
2444 		nvl_peers = nvlist_get_nvlist_array(nvl, "peers", &peercount);
2445 		for (int i = 0; i < peercount; i++) {
2446 			err = wg_peer_add(sc, nvl_peers[i]);
2447 			if (err != 0)
2448 				goto out_locked;
2449 		}
2450 	}
2451 
2452 out_locked:
2453 	sx_xunlock(&sc->sc_lock);
2454 	nvlist_destroy(nvl);
2455 out:
2456 	zfree(nvlpacked, M_TEMP);
2457 	return (err);
2458 }
2459 
2460 static int
2461 wgc_get(struct wg_softc *sc, struct wg_data_io *wgd)
2462 {
2463 	uint8_t public_key[WG_KEY_SIZE] = { 0 };
2464 	uint8_t private_key[WG_KEY_SIZE] = { 0 };
2465 	uint8_t preshared_key[NOISE_SYMMETRIC_KEY_LEN] = { 0 };
2466 	nvlist_t *nvl, *nvl_peer, *nvl_aip, **nvl_peers, **nvl_aips;
2467 	size_t size, peer_count, aip_count, i, j;
2468 	struct wg_timespec64 ts64;
2469 	struct wg_peer *peer;
2470 	struct wg_aip *aip;
2471 	void *packed;
2472 	int err = 0;
2473 
2474 	nvl = nvlist_create(0);
2475 	if (!nvl)
2476 		return (ENOMEM);
2477 
2478 	sx_slock(&sc->sc_lock);
2479 
2480 	if (sc->sc_socket.so_port != 0)
2481 		nvlist_add_number(nvl, "listen-port", sc->sc_socket.so_port);
2482 	if (sc->sc_socket.so_user_cookie != 0)
2483 		nvlist_add_number(nvl, "user-cookie", sc->sc_socket.so_user_cookie);
2484 	if (noise_local_keys(sc->sc_local, public_key, private_key) == 0) {
2485 		nvlist_add_binary(nvl, "public-key", public_key, WG_KEY_SIZE);
2486 		if (wgc_privileged(sc))
2487 			nvlist_add_binary(nvl, "private-key", private_key, WG_KEY_SIZE);
2488 		explicit_bzero(private_key, sizeof(private_key));
2489 	}
2490 	peer_count = sc->sc_peers_num;
2491 	if (peer_count) {
2492 		nvl_peers = mallocarray(peer_count, sizeof(void *), M_NVLIST, M_WAITOK | M_ZERO);
2493 		i = 0;
2494 		TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) {
2495 			if (i >= peer_count)
2496 				panic("peers changed from under us");
2497 
2498 			nvl_peers[i++] = nvl_peer = nvlist_create(0);
2499 			if (!nvl_peer) {
2500 				err = ENOMEM;
2501 				goto err_peer;
2502 			}
2503 
2504 			(void)noise_remote_keys(peer->p_remote, public_key, preshared_key);
2505 			nvlist_add_binary(nvl_peer, "public-key", public_key, sizeof(public_key));
2506 			if (wgc_privileged(sc))
2507 				nvlist_add_binary(nvl_peer, "preshared-key", preshared_key, sizeof(preshared_key));
2508 			explicit_bzero(preshared_key, sizeof(preshared_key));
2509 			if (peer->p_endpoint.e_remote.r_sa.sa_family == AF_INET)
2510 				nvlist_add_binary(nvl_peer, "endpoint", &peer->p_endpoint.e_remote, sizeof(struct sockaddr_in));
2511 			else if (peer->p_endpoint.e_remote.r_sa.sa_family == AF_INET6)
2512 				nvlist_add_binary(nvl_peer, "endpoint", &peer->p_endpoint.e_remote, sizeof(struct sockaddr_in6));
2513 			wg_timers_get_last_handshake(peer, &ts64);
2514 			nvlist_add_binary(nvl_peer, "last-handshake-time", &ts64, sizeof(ts64));
2515 			nvlist_add_number(nvl_peer, "persistent-keepalive-interval", peer->p_persistent_keepalive_interval);
2516 			nvlist_add_number(nvl_peer, "rx-bytes", counter_u64_fetch(peer->p_rx_bytes));
2517 			nvlist_add_number(nvl_peer, "tx-bytes", counter_u64_fetch(peer->p_tx_bytes));
2518 
2519 			aip_count = peer->p_aips_num;
2520 			if (aip_count) {
2521 				nvl_aips = mallocarray(aip_count, sizeof(void *), M_NVLIST, M_WAITOK | M_ZERO);
2522 				j = 0;
2523 				LIST_FOREACH(aip, &peer->p_aips, a_entry) {
2524 					if (j >= aip_count)
2525 						panic("aips changed from under us");
2526 
2527 					nvl_aips[j++] = nvl_aip = nvlist_create(0);
2528 					if (!nvl_aip) {
2529 						err = ENOMEM;
2530 						goto err_aip;
2531 					}
2532 					if (aip->a_af == AF_INET) {
2533 						nvlist_add_binary(nvl_aip, "ipv4", &aip->a_addr.in, sizeof(aip->a_addr.in));
2534 						nvlist_add_number(nvl_aip, "cidr", bitcount32(aip->a_mask.ip));
2535 					}
2536 #ifdef INET6
2537 					else if (aip->a_af == AF_INET6) {
2538 						nvlist_add_binary(nvl_aip, "ipv6", &aip->a_addr.in6, sizeof(aip->a_addr.in6));
2539 						nvlist_add_number(nvl_aip, "cidr", in6_mask2len(&aip->a_mask.in6, NULL));
2540 					}
2541 #endif
2542 				}
2543 				nvlist_add_nvlist_array(nvl_peer, "allowed-ips", (const nvlist_t *const *)nvl_aips, aip_count);
2544 			err_aip:
2545 				for (j = 0; j < aip_count; ++j)
2546 					nvlist_destroy(nvl_aips[j]);
2547 				free(nvl_aips, M_NVLIST);
2548 				if (err)
2549 					goto err_peer;
2550 			}
2551 		}
2552 		nvlist_add_nvlist_array(nvl, "peers", (const nvlist_t * const *)nvl_peers, peer_count);
2553 	err_peer:
2554 		for (i = 0; i < peer_count; ++i)
2555 			nvlist_destroy(nvl_peers[i]);
2556 		free(nvl_peers, M_NVLIST);
2557 		if (err) {
2558 			sx_sunlock(&sc->sc_lock);
2559 			goto err;
2560 		}
2561 	}
2562 	sx_sunlock(&sc->sc_lock);
2563 	packed = nvlist_pack(nvl, &size);
2564 	if (!packed) {
2565 		err = ENOMEM;
2566 		goto err;
2567 	}
2568 	if (!wgd->wgd_size) {
2569 		wgd->wgd_size = size;
2570 		goto out;
2571 	}
2572 	if (wgd->wgd_size < size) {
2573 		err = ENOSPC;
2574 		goto out;
2575 	}
2576 	err = copyout(packed, wgd->wgd_data, size);
2577 	wgd->wgd_size = size;
2578 
2579 out:
2580 	zfree(packed, M_NVLIST);
2581 err:
2582 	nvlist_destroy(nvl);
2583 	return (err);
2584 }
2585 
2586 static int
2587 wg_ioctl(if_t ifp, u_long cmd, caddr_t data)
2588 {
2589 	struct wg_data_io *wgd = (struct wg_data_io *)data;
2590 	struct ifreq *ifr = (struct ifreq *)data;
2591 	struct wg_softc *sc;
2592 	int ret = 0;
2593 
2594 	sx_slock(&wg_sx);
2595 	sc = if_getsoftc(ifp);
2596 	if (!sc) {
2597 		ret = ENXIO;
2598 		goto out;
2599 	}
2600 
2601 	switch (cmd) {
2602 	case SIOCSWG:
2603 		ret = priv_check(curthread, PRIV_NET_WG);
2604 		if (ret == 0)
2605 			ret = wgc_set(sc, wgd);
2606 		break;
2607 	case SIOCGWG:
2608 		ret = wgc_get(sc, wgd);
2609 		break;
2610 	/* Interface IOCTLs */
2611 	case SIOCSIFADDR:
2612 		/*
2613 		 * This differs from *BSD norms, but is more uniform with how
2614 		 * WireGuard behaves elsewhere.
2615 		 */
2616 		break;
2617 	case SIOCSIFFLAGS:
2618 		if (if_getflags(ifp) & IFF_UP)
2619 			ret = wg_up(sc);
2620 		else
2621 			wg_down(sc);
2622 		break;
2623 	case SIOCSIFMTU:
2624 		if (ifr->ifr_mtu <= 0 || ifr->ifr_mtu > MAX_MTU)
2625 			ret = EINVAL;
2626 		else
2627 			if_setmtu(ifp, ifr->ifr_mtu);
2628 		break;
2629 	case SIOCADDMULTI:
2630 	case SIOCDELMULTI:
2631 		break;
2632 	case SIOCGTUNFIB:
2633 		ifr->ifr_fib = sc->sc_socket.so_fibnum;
2634 		break;
2635 	case SIOCSTUNFIB:
2636 		ret = priv_check(curthread, PRIV_NET_WG);
2637 		if (ret)
2638 			break;
2639 		ret = priv_check(curthread, PRIV_NET_SETIFFIB);
2640 		if (ret)
2641 			break;
2642 		sx_xlock(&sc->sc_lock);
2643 		ret = wg_socket_set_fibnum(sc, ifr->ifr_fib);
2644 		sx_xunlock(&sc->sc_lock);
2645 		break;
2646 	default:
2647 		ret = ENOTTY;
2648 	}
2649 
2650 out:
2651 	sx_sunlock(&wg_sx);
2652 	return (ret);
2653 }
2654 
2655 static int
2656 wg_up(struct wg_softc *sc)
2657 {
2658 	if_t ifp = sc->sc_ifp;
2659 	struct wg_peer *peer;
2660 	int rc = EBUSY;
2661 
2662 	sx_xlock(&sc->sc_lock);
2663 	/* Jail's being removed, no more wg_up(). */
2664 	if ((sc->sc_flags & WGF_DYING) != 0)
2665 		goto out;
2666 
2667 	/* Silent success if we're already running. */
2668 	rc = 0;
2669 	if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
2670 		goto out;
2671 	if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0);
2672 
2673 	rc = wg_socket_init(sc, sc->sc_socket.so_port);
2674 	if (rc == 0) {
2675 		TAILQ_FOREACH(peer, &sc->sc_peers, p_entry)
2676 			wg_timers_enable(peer);
2677 		if_link_state_change(sc->sc_ifp, LINK_STATE_UP);
2678 	} else {
2679 		if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
2680 		DPRINTF(sc, "Unable to initialize sockets: %d\n", rc);
2681 	}
2682 out:
2683 	sx_xunlock(&sc->sc_lock);
2684 	return (rc);
2685 }
2686 
2687 static void
2688 wg_down(struct wg_softc *sc)
2689 {
2690 	if_t ifp = sc->sc_ifp;
2691 	struct wg_peer *peer;
2692 
2693 	sx_xlock(&sc->sc_lock);
2694 	if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) {
2695 		sx_xunlock(&sc->sc_lock);
2696 		return;
2697 	}
2698 	if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
2699 
2700 	TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) {
2701 		wg_queue_purge(&peer->p_stage_queue);
2702 		wg_timers_disable(peer);
2703 	}
2704 
2705 	wg_queue_purge(&sc->sc_handshake_queue);
2706 
2707 	TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) {
2708 		noise_remote_handshake_clear(peer->p_remote);
2709 		noise_remote_keypairs_clear(peer->p_remote);
2710 	}
2711 
2712 	if_link_state_change(sc->sc_ifp, LINK_STATE_DOWN);
2713 	wg_socket_uninit(sc);
2714 
2715 	sx_xunlock(&sc->sc_lock);
2716 }
2717 
2718 static int
2719 wg_clone_create(struct if_clone *ifc, char *name, size_t len,
2720     struct ifc_data *ifd, struct ifnet **ifpp)
2721 {
2722 	struct wg_softc *sc;
2723 	if_t ifp;
2724 
2725 	sc = malloc(sizeof(*sc), M_WG, M_WAITOK | M_ZERO);
2726 
2727 	sc->sc_local = noise_local_alloc(sc);
2728 
2729 	sc->sc_encrypt = mallocarray(sizeof(struct grouptask), mp_ncpus, M_WG, M_WAITOK | M_ZERO);
2730 
2731 	sc->sc_decrypt = mallocarray(sizeof(struct grouptask), mp_ncpus, M_WG, M_WAITOK | M_ZERO);
2732 
2733 	if (!rn_inithead((void **)&sc->sc_aip4, offsetof(struct aip_addr, in) * NBBY))
2734 		goto free_decrypt;
2735 
2736 	if (!rn_inithead((void **)&sc->sc_aip6, offsetof(struct aip_addr, in6) * NBBY))
2737 		goto free_aip4;
2738 
2739 	atomic_add_int(&clone_count, 1);
2740 	ifp = sc->sc_ifp = if_alloc(IFT_WIREGUARD);
2741 
2742 	sc->sc_ucred = crhold(curthread->td_ucred);
2743 	sc->sc_socket.so_fibnum = curthread->td_proc->p_fibnum;
2744 	sc->sc_socket.so_port = 0;
2745 
2746 	TAILQ_INIT(&sc->sc_peers);
2747 	sc->sc_peers_num = 0;
2748 
2749 	cookie_checker_init(&sc->sc_cookie);
2750 
2751 	RADIX_NODE_HEAD_LOCK_INIT(sc->sc_aip4);
2752 	RADIX_NODE_HEAD_LOCK_INIT(sc->sc_aip6);
2753 
2754 	GROUPTASK_INIT(&sc->sc_handshake, 0, (gtask_fn_t *)wg_softc_handshake_receive, sc);
2755 	taskqgroup_attach(qgroup_wg_tqg, &sc->sc_handshake, sc, NULL, NULL, "wg tx initiation");
2756 	wg_queue_init(&sc->sc_handshake_queue, "hsq");
2757 
2758 	for (int i = 0; i < mp_ncpus; i++) {
2759 		GROUPTASK_INIT(&sc->sc_encrypt[i], 0,
2760 		     (gtask_fn_t *)wg_softc_encrypt, sc);
2761 		taskqgroup_attach_cpu(qgroup_wg_tqg, &sc->sc_encrypt[i], sc, i, NULL, NULL, "wg encrypt");
2762 		GROUPTASK_INIT(&sc->sc_decrypt[i], 0,
2763 		    (gtask_fn_t *)wg_softc_decrypt, sc);
2764 		taskqgroup_attach_cpu(qgroup_wg_tqg, &sc->sc_decrypt[i], sc, i, NULL, NULL, "wg decrypt");
2765 	}
2766 
2767 	wg_queue_init(&sc->sc_encrypt_parallel, "encp");
2768 	wg_queue_init(&sc->sc_decrypt_parallel, "decp");
2769 
2770 	sx_init(&sc->sc_lock, "wg softc lock");
2771 
2772 	if_setsoftc(ifp, sc);
2773 	if_setcapabilities(ifp, WG_CAPS);
2774 	if_setcapenable(ifp, WG_CAPS);
2775 	if_initname(ifp, wgname, ifd->unit);
2776 
2777 	if_setmtu(ifp, DEFAULT_MTU);
2778 	if_setflags(ifp, IFF_NOARP | IFF_MULTICAST);
2779 	if_setinitfn(ifp, wg_init);
2780 	if_setreassignfn(ifp, wg_reassign);
2781 	if_setqflushfn(ifp, wg_qflush);
2782 	if_settransmitfn(ifp, wg_transmit);
2783 	if_setoutputfn(ifp, wg_output);
2784 	if_setioctlfn(ifp, wg_ioctl);
2785 	if_attach(ifp);
2786 	bpfattach(ifp, DLT_NULL, sizeof(uint32_t));
2787 #ifdef INET6
2788 	ND_IFINFO(ifp)->flags &= ~ND6_IFF_AUTO_LINKLOCAL;
2789 	ND_IFINFO(ifp)->flags |= ND6_IFF_NO_DAD;
2790 #endif
2791 	sx_xlock(&wg_sx);
2792 	LIST_INSERT_HEAD(&wg_list, sc, sc_entry);
2793 	sx_xunlock(&wg_sx);
2794 	*ifpp = ifp;
2795 	return (0);
2796 free_aip4:
2797 	RADIX_NODE_HEAD_DESTROY(sc->sc_aip4);
2798 	free(sc->sc_aip4, M_RTABLE);
2799 free_decrypt:
2800 	free(sc->sc_decrypt, M_WG);
2801 	free(sc->sc_encrypt, M_WG);
2802 	noise_local_free(sc->sc_local, NULL);
2803 	free(sc, M_WG);
2804 	return (ENOMEM);
2805 }
2806 
2807 static void
2808 wg_clone_deferred_free(struct noise_local *l)
2809 {
2810 	struct wg_softc *sc = noise_local_arg(l);
2811 
2812 	free(sc, M_WG);
2813 	atomic_add_int(&clone_count, -1);
2814 }
2815 
2816 static int
2817 wg_clone_destroy(struct if_clone *ifc, if_t ifp, uint32_t flags)
2818 {
2819 	struct wg_softc *sc = if_getsoftc(ifp);
2820 	struct ucred *cred;
2821 
2822 	sx_xlock(&wg_sx);
2823 	if_setsoftc(ifp, NULL);
2824 	sx_xlock(&sc->sc_lock);
2825 	sc->sc_flags |= WGF_DYING;
2826 	cred = sc->sc_ucred;
2827 	sc->sc_ucred = NULL;
2828 	sx_xunlock(&sc->sc_lock);
2829 	LIST_REMOVE(sc, sc_entry);
2830 	sx_xunlock(&wg_sx);
2831 
2832 	if_link_state_change(sc->sc_ifp, LINK_STATE_DOWN);
2833 	CURVNET_SET(if_getvnet(sc->sc_ifp));
2834 	if_purgeaddrs(sc->sc_ifp);
2835 	CURVNET_RESTORE();
2836 
2837 	sx_xlock(&sc->sc_lock);
2838 	wg_socket_uninit(sc);
2839 	sx_xunlock(&sc->sc_lock);
2840 
2841 	/*
2842 	 * No guarantees that all traffic have passed until the epoch has
2843 	 * elapsed with the socket closed.
2844 	 */
2845 	NET_EPOCH_WAIT();
2846 
2847 	taskqgroup_drain_all(qgroup_wg_tqg);
2848 	sx_xlock(&sc->sc_lock);
2849 	wg_peer_destroy_all(sc);
2850 	NET_EPOCH_DRAIN_CALLBACKS();
2851 	sx_xunlock(&sc->sc_lock);
2852 	sx_destroy(&sc->sc_lock);
2853 	taskqgroup_detach(qgroup_wg_tqg, &sc->sc_handshake);
2854 	for (int i = 0; i < mp_ncpus; i++) {
2855 		taskqgroup_detach(qgroup_wg_tqg, &sc->sc_encrypt[i]);
2856 		taskqgroup_detach(qgroup_wg_tqg, &sc->sc_decrypt[i]);
2857 	}
2858 	free(sc->sc_encrypt, M_WG);
2859 	free(sc->sc_decrypt, M_WG);
2860 	wg_queue_deinit(&sc->sc_handshake_queue);
2861 	wg_queue_deinit(&sc->sc_encrypt_parallel);
2862 	wg_queue_deinit(&sc->sc_decrypt_parallel);
2863 
2864 	RADIX_NODE_HEAD_DESTROY(sc->sc_aip4);
2865 	RADIX_NODE_HEAD_DESTROY(sc->sc_aip6);
2866 	rn_detachhead((void **)&sc->sc_aip4);
2867 	rn_detachhead((void **)&sc->sc_aip6);
2868 
2869 	cookie_checker_free(&sc->sc_cookie);
2870 
2871 	if (cred != NULL)
2872 		crfree(cred);
2873 	if_detach(sc->sc_ifp);
2874 	if_free(sc->sc_ifp);
2875 
2876 	noise_local_free(sc->sc_local, wg_clone_deferred_free);
2877 
2878 	return (0);
2879 }
2880 
2881 static void
2882 wg_qflush(if_t ifp __unused)
2883 {
2884 }
2885 
2886 /*
2887  * Privileged information (private-key, preshared-key) are only exported for
2888  * root and jailed root by default.
2889  */
2890 static bool
2891 wgc_privileged(struct wg_softc *sc)
2892 {
2893 	struct thread *td;
2894 
2895 	td = curthread;
2896 	return (priv_check(td, PRIV_NET_WG) == 0);
2897 }
2898 
2899 static void
2900 wg_reassign(if_t ifp, struct vnet *new_vnet __unused,
2901     char *unused __unused)
2902 {
2903 	struct wg_softc *sc;
2904 
2905 	sc = if_getsoftc(ifp);
2906 	wg_down(sc);
2907 }
2908 
2909 static void
2910 wg_init(void *xsc)
2911 {
2912 	struct wg_softc *sc;
2913 
2914 	sc = xsc;
2915 	wg_up(sc);
2916 }
2917 
2918 static void
2919 vnet_wg_init(const void *unused __unused)
2920 {
2921 	struct if_clone_addreq req = {
2922 		.create_f = wg_clone_create,
2923 		.destroy_f = wg_clone_destroy,
2924 		.flags = IFC_F_AUTOUNIT,
2925 	};
2926 	V_wg_cloner = ifc_attach_cloner(wgname, &req);
2927 }
2928 VNET_SYSINIT(vnet_wg_init, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY,
2929 	     vnet_wg_init, NULL);
2930 
2931 static void
2932 vnet_wg_uninit(const void *unused __unused)
2933 {
2934 	if (V_wg_cloner)
2935 		ifc_detach_cloner(V_wg_cloner);
2936 }
2937 VNET_SYSUNINIT(vnet_wg_uninit, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY,
2938 	       vnet_wg_uninit, NULL);
2939 
2940 static int
2941 wg_prison_remove(void *obj, void *data __unused)
2942 {
2943 	const struct prison *pr = obj;
2944 	struct wg_softc *sc;
2945 
2946 	/*
2947 	 * Do a pass through all if_wg interfaces and release creds on any from
2948 	 * the jail that are supposed to be going away.  This will, in turn, let
2949 	 * the jail die so that we don't end up with Schrödinger's jail.
2950 	 */
2951 	sx_slock(&wg_sx);
2952 	LIST_FOREACH(sc, &wg_list, sc_entry) {
2953 		sx_xlock(&sc->sc_lock);
2954 		if (!(sc->sc_flags & WGF_DYING) && sc->sc_ucred && sc->sc_ucred->cr_prison == pr) {
2955 			struct ucred *cred = sc->sc_ucred;
2956 			DPRINTF(sc, "Creating jail exiting\n");
2957 			if_link_state_change(sc->sc_ifp, LINK_STATE_DOWN);
2958 			wg_socket_uninit(sc);
2959 			sc->sc_ucred = NULL;
2960 			crfree(cred);
2961 			sc->sc_flags |= WGF_DYING;
2962 		}
2963 		sx_xunlock(&sc->sc_lock);
2964 	}
2965 	sx_sunlock(&wg_sx);
2966 
2967 	return (0);
2968 }
2969 
2970 #ifdef SELFTESTS
2971 #include "selftest/allowedips.c"
2972 static bool wg_run_selftests(void)
2973 {
2974 	bool ret = true;
2975 	ret &= wg_allowedips_selftest();
2976 	ret &= noise_counter_selftest();
2977 	ret &= cookie_selftest();
2978 	return ret;
2979 }
2980 #else
2981 static inline bool wg_run_selftests(void) { return true; }
2982 #endif
2983 
2984 static int
2985 wg_module_init(void)
2986 {
2987 	int ret = ENOMEM;
2988 
2989 	osd_method_t methods[PR_MAXMETHOD] = {
2990 		[PR_METHOD_REMOVE] = wg_prison_remove,
2991 	};
2992 
2993 	if ((wg_packet_zone = uma_zcreate("wg packet", sizeof(struct wg_packet),
2994 	     NULL, NULL, NULL, NULL, 0, 0)) == NULL)
2995 		goto free_none;
2996 	ret = crypto_init();
2997 	if (ret != 0)
2998 		goto free_zone;
2999 	if (cookie_init() != 0)
3000 		goto free_crypto;
3001 
3002 	wg_osd_jail_slot = osd_jail_register(NULL, methods);
3003 
3004 	ret = ENOTRECOVERABLE;
3005 	if (!wg_run_selftests())
3006 		goto free_all;
3007 
3008 	return (0);
3009 
3010 free_all:
3011 	osd_jail_deregister(wg_osd_jail_slot);
3012 	cookie_deinit();
3013 free_crypto:
3014 	crypto_deinit();
3015 free_zone:
3016 	uma_zdestroy(wg_packet_zone);
3017 free_none:
3018 	return (ret);
3019 }
3020 
3021 static void
3022 wg_module_deinit(void)
3023 {
3024 	VNET_ITERATOR_DECL(vnet_iter);
3025 	VNET_LIST_RLOCK();
3026 	VNET_FOREACH(vnet_iter) {
3027 		struct if_clone *clone = VNET_VNET(vnet_iter, wg_cloner);
3028 		if (clone) {
3029 			ifc_detach_cloner(clone);
3030 			VNET_VNET(vnet_iter, wg_cloner) = NULL;
3031 		}
3032 	}
3033 	VNET_LIST_RUNLOCK();
3034 	NET_EPOCH_WAIT();
3035 	MPASS(LIST_EMPTY(&wg_list));
3036 	osd_jail_deregister(wg_osd_jail_slot);
3037 	cookie_deinit();
3038 	crypto_deinit();
3039 	uma_zdestroy(wg_packet_zone);
3040 }
3041 
3042 static int
3043 wg_module_event_handler(module_t mod, int what, void *arg)
3044 {
3045 	switch (what) {
3046 		case MOD_LOAD:
3047 			return wg_module_init();
3048 		case MOD_UNLOAD:
3049 			wg_module_deinit();
3050 			break;
3051 		default:
3052 			return (EOPNOTSUPP);
3053 	}
3054 	return (0);
3055 }
3056 
3057 static moduledata_t wg_moduledata = {
3058 	"if_wg",
3059 	wg_module_event_handler,
3060 	NULL
3061 };
3062 
3063 DECLARE_MODULE(if_wg, wg_moduledata, SI_SUB_PSEUDO, SI_ORDER_ANY);
3064 MODULE_VERSION(if_wg, WIREGUARD_VERSION);
3065 MODULE_DEPEND(if_wg, crypto, 1, 1, 1);
3066