1 /* SPDX-License-Identifier: ISC 2 * 3 * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 4 * Copyright (C) 2019-2021 Matt Dunwoodie <ncon@noconroy.net> 5 * Copyright (c) 2019-2020 Rubicon Communications, LLC (Netgate) 6 * Copyright (c) 2021 Kyle Evans <kevans@FreeBSD.org> 7 * Copyright (c) 2022 The FreeBSD Foundation 8 */ 9 10 #include "opt_inet.h" 11 #include "opt_inet6.h" 12 13 #include <sys/param.h> 14 #include <sys/systm.h> 15 #include <sys/counter.h> 16 #include <sys/gtaskqueue.h> 17 #include <sys/jail.h> 18 #include <sys/kernel.h> 19 #include <sys/lock.h> 20 #include <sys/mbuf.h> 21 #include <sys/module.h> 22 #include <sys/nv.h> 23 #include <sys/priv.h> 24 #include <sys/protosw.h> 25 #include <sys/rmlock.h> 26 #include <sys/rwlock.h> 27 #include <sys/smp.h> 28 #include <sys/socket.h> 29 #include <sys/socketvar.h> 30 #include <sys/sockio.h> 31 #include <sys/sysctl.h> 32 #include <sys/sx.h> 33 #include <machine/_inttypes.h> 34 #include <net/bpf.h> 35 #include <net/ethernet.h> 36 #include <net/if.h> 37 #include <net/if_clone.h> 38 #include <net/if_types.h> 39 #include <net/if_var.h> 40 #include <net/netisr.h> 41 #include <net/radix.h> 42 #include <netinet/in.h> 43 #include <netinet6/in6_var.h> 44 #include <netinet/ip.h> 45 #include <netinet/ip6.h> 46 #include <netinet/ip_encap.h> 47 #include <netinet/ip_icmp.h> 48 #include <netinet/icmp6.h> 49 #include <netinet/udp_var.h> 50 #include <netinet6/nd6.h> 51 52 #include "wg_noise.h" 53 #include "wg_cookie.h" 54 #include "version.h" 55 #include "if_wg.h" 56 57 #define DEFAULT_MTU (ETHERMTU - 80) 58 #define MAX_MTU (IF_MAXMTU - 80) 59 60 #define MAX_STAGED_PKT 128 61 #define MAX_QUEUED_PKT 1024 62 #define MAX_QUEUED_PKT_MASK (MAX_QUEUED_PKT - 1) 63 64 #define MAX_QUEUED_HANDSHAKES 4096 65 66 #define REKEY_TIMEOUT_JITTER 334 /* 1/3 sec, round for arc4random_uniform */ 67 #define MAX_TIMER_HANDSHAKES (90 / REKEY_TIMEOUT) 68 #define NEW_HANDSHAKE_TIMEOUT (REKEY_TIMEOUT + KEEPALIVE_TIMEOUT) 69 #define UNDERLOAD_TIMEOUT 1 70 71 #define DPRINTF(sc, ...) if (if_getflags(sc->sc_ifp) & IFF_DEBUG) if_printf(sc->sc_ifp, ##__VA_ARGS__) 72 73 /* First byte indicating packet type on the wire */ 74 #define WG_PKT_INITIATION htole32(1) 75 #define WG_PKT_RESPONSE htole32(2) 76 #define WG_PKT_COOKIE htole32(3) 77 #define WG_PKT_DATA htole32(4) 78 79 #define WG_PKT_PADDING 16 80 #define WG_KEY_SIZE 32 81 82 struct wg_pkt_initiation { 83 uint32_t t; 84 uint32_t s_idx; 85 uint8_t ue[NOISE_PUBLIC_KEY_LEN]; 86 uint8_t es[NOISE_PUBLIC_KEY_LEN + NOISE_AUTHTAG_LEN]; 87 uint8_t ets[NOISE_TIMESTAMP_LEN + NOISE_AUTHTAG_LEN]; 88 struct cookie_macs m; 89 }; 90 91 struct wg_pkt_response { 92 uint32_t t; 93 uint32_t s_idx; 94 uint32_t r_idx; 95 uint8_t ue[NOISE_PUBLIC_KEY_LEN]; 96 uint8_t en[0 + NOISE_AUTHTAG_LEN]; 97 struct cookie_macs m; 98 }; 99 100 struct wg_pkt_cookie { 101 uint32_t t; 102 uint32_t r_idx; 103 uint8_t nonce[COOKIE_NONCE_SIZE]; 104 uint8_t ec[COOKIE_ENCRYPTED_SIZE]; 105 }; 106 107 struct wg_pkt_data { 108 uint32_t t; 109 uint32_t r_idx; 110 uint64_t nonce; 111 uint8_t buf[]; 112 }; 113 114 struct wg_endpoint { 115 union { 116 struct sockaddr r_sa; 117 struct sockaddr_in r_sin; 118 #ifdef INET6 119 struct sockaddr_in6 r_sin6; 120 #endif 121 } e_remote; 122 union { 123 struct in_addr l_in; 124 #ifdef INET6 125 struct in6_pktinfo l_pktinfo6; 126 #define l_in6 l_pktinfo6.ipi6_addr 127 #endif 128 } e_local; 129 }; 130 131 struct aip_addr { 132 uint8_t length; 133 union { 134 uint8_t bytes[16]; 135 uint32_t ip; 136 uint32_t ip6[4]; 137 struct in_addr in; 138 struct in6_addr in6; 139 }; 140 }; 141 142 struct wg_aip { 143 struct radix_node a_nodes[2]; 144 LIST_ENTRY(wg_aip) a_entry; 145 struct aip_addr a_addr; 146 struct aip_addr a_mask; 147 struct wg_peer *a_peer; 148 sa_family_t a_af; 149 }; 150 151 struct wg_packet { 152 STAILQ_ENTRY(wg_packet) p_serial; 153 STAILQ_ENTRY(wg_packet) p_parallel; 154 struct wg_endpoint p_endpoint; 155 struct noise_keypair *p_keypair; 156 uint64_t p_nonce; 157 struct mbuf *p_mbuf; 158 int p_mtu; 159 sa_family_t p_af; 160 enum wg_ring_state { 161 WG_PACKET_UNCRYPTED, 162 WG_PACKET_CRYPTED, 163 WG_PACKET_DEAD, 164 } p_state; 165 }; 166 167 STAILQ_HEAD(wg_packet_list, wg_packet); 168 169 struct wg_queue { 170 struct mtx q_mtx; 171 struct wg_packet_list q_queue; 172 size_t q_len; 173 }; 174 175 struct wg_peer { 176 TAILQ_ENTRY(wg_peer) p_entry; 177 uint64_t p_id; 178 struct wg_softc *p_sc; 179 180 struct noise_remote *p_remote; 181 struct cookie_maker p_cookie; 182 183 struct rwlock p_endpoint_lock; 184 struct wg_endpoint p_endpoint; 185 186 struct wg_queue p_stage_queue; 187 struct wg_queue p_encrypt_serial; 188 struct wg_queue p_decrypt_serial; 189 190 bool p_enabled; 191 bool p_need_another_keepalive; 192 uint16_t p_persistent_keepalive_interval; 193 struct callout p_new_handshake; 194 struct callout p_send_keepalive; 195 struct callout p_retry_handshake; 196 struct callout p_zero_key_material; 197 struct callout p_persistent_keepalive; 198 199 struct mtx p_handshake_mtx; 200 struct timespec p_handshake_complete; /* nanotime */ 201 int p_handshake_retries; 202 203 struct grouptask p_send; 204 struct grouptask p_recv; 205 206 counter_u64_t p_tx_bytes; 207 counter_u64_t p_rx_bytes; 208 209 LIST_HEAD(, wg_aip) p_aips; 210 size_t p_aips_num; 211 }; 212 213 struct wg_socket { 214 struct socket *so_so4; 215 struct socket *so_so6; 216 uint32_t so_user_cookie; 217 int so_fibnum; 218 in_port_t so_port; 219 }; 220 221 struct wg_softc { 222 LIST_ENTRY(wg_softc) sc_entry; 223 if_t sc_ifp; 224 int sc_flags; 225 226 struct ucred *sc_ucred; 227 struct wg_socket sc_socket; 228 229 TAILQ_HEAD(,wg_peer) sc_peers; 230 size_t sc_peers_num; 231 232 struct noise_local *sc_local; 233 struct cookie_checker sc_cookie; 234 235 struct radix_node_head *sc_aip4; 236 struct radix_node_head *sc_aip6; 237 238 struct grouptask sc_handshake; 239 struct wg_queue sc_handshake_queue; 240 241 struct grouptask *sc_encrypt; 242 struct grouptask *sc_decrypt; 243 struct wg_queue sc_encrypt_parallel; 244 struct wg_queue sc_decrypt_parallel; 245 u_int sc_encrypt_last_cpu; 246 u_int sc_decrypt_last_cpu; 247 248 struct sx sc_lock; 249 }; 250 251 #define WGF_DYING 0x0001 252 253 #define MAX_LOOPS 8 254 #define MTAG_WGLOOP 0x77676c70 /* wglp */ 255 256 #define GROUPTASK_DRAIN(gtask) \ 257 gtaskqueue_drain((gtask)->gt_taskqueue, &(gtask)->gt_task) 258 259 #define BPF_MTAP2_AF(ifp, m, af) do { \ 260 uint32_t __bpf_tap_af = (af); \ 261 BPF_MTAP2(ifp, &__bpf_tap_af, sizeof(__bpf_tap_af), m); \ 262 } while (0) 263 264 static int clone_count; 265 static uma_zone_t wg_packet_zone; 266 static volatile unsigned long peer_counter = 0; 267 static const char wgname[] = "wg"; 268 static unsigned wg_osd_jail_slot; 269 270 static struct sx wg_sx; 271 SX_SYSINIT(wg_sx, &wg_sx, "wg_sx"); 272 273 static LIST_HEAD(, wg_softc) wg_list = LIST_HEAD_INITIALIZER(wg_list); 274 275 static TASKQGROUP_DEFINE(wg_tqg, mp_ncpus, 1); 276 277 MALLOC_DEFINE(M_WG, "WG", "wireguard"); 278 279 VNET_DEFINE_STATIC(struct if_clone *, wg_cloner); 280 281 #define V_wg_cloner VNET(wg_cloner) 282 #define WG_CAPS IFCAP_LINKSTATE 283 284 struct wg_timespec64 { 285 uint64_t tv_sec; 286 uint64_t tv_nsec; 287 }; 288 289 static int wg_socket_init(struct wg_softc *, in_port_t); 290 static int wg_socket_bind(struct socket **, struct socket **, in_port_t *); 291 static void wg_socket_set(struct wg_softc *, struct socket *, struct socket *); 292 static void wg_socket_uninit(struct wg_softc *); 293 static int wg_socket_set_sockopt(struct socket *, struct socket *, int, void *, size_t); 294 static int wg_socket_set_cookie(struct wg_softc *, uint32_t); 295 static int wg_socket_set_fibnum(struct wg_softc *, int); 296 static int wg_send(struct wg_softc *, struct wg_endpoint *, struct mbuf *); 297 static void wg_timers_enable(struct wg_peer *); 298 static void wg_timers_disable(struct wg_peer *); 299 static void wg_timers_set_persistent_keepalive(struct wg_peer *, uint16_t); 300 static void wg_timers_get_last_handshake(struct wg_peer *, struct wg_timespec64 *); 301 static void wg_timers_event_data_sent(struct wg_peer *); 302 static void wg_timers_event_data_received(struct wg_peer *); 303 static void wg_timers_event_any_authenticated_packet_sent(struct wg_peer *); 304 static void wg_timers_event_any_authenticated_packet_received(struct wg_peer *); 305 static void wg_timers_event_any_authenticated_packet_traversal(struct wg_peer *); 306 static void wg_timers_event_handshake_initiated(struct wg_peer *); 307 static void wg_timers_event_handshake_complete(struct wg_peer *); 308 static void wg_timers_event_session_derived(struct wg_peer *); 309 static void wg_timers_event_want_initiation(struct wg_peer *); 310 static void wg_timers_run_send_initiation(struct wg_peer *, bool); 311 static void wg_timers_run_retry_handshake(void *); 312 static void wg_timers_run_send_keepalive(void *); 313 static void wg_timers_run_new_handshake(void *); 314 static void wg_timers_run_zero_key_material(void *); 315 static void wg_timers_run_persistent_keepalive(void *); 316 static int wg_aip_add(struct wg_softc *, struct wg_peer *, sa_family_t, 317 const void *, uint8_t); 318 static int wg_aip_del(struct wg_softc *, struct wg_peer *, sa_family_t, 319 const void *, uint8_t); 320 static struct wg_peer *wg_aip_lookup(struct wg_softc *, sa_family_t, void *); 321 static void wg_aip_remove_all(struct wg_softc *, struct wg_peer *); 322 static struct wg_peer *wg_peer_create(struct wg_softc *, 323 const uint8_t [WG_KEY_SIZE], int *); 324 static void wg_peer_free_deferred(struct noise_remote *); 325 static void wg_peer_destroy(struct wg_peer *); 326 static void wg_peer_destroy_all(struct wg_softc *); 327 static void wg_peer_send_buf(struct wg_peer *, uint8_t *, size_t); 328 static void wg_send_initiation(struct wg_peer *); 329 static void wg_send_response(struct wg_peer *); 330 static void wg_send_cookie(struct wg_softc *, struct cookie_macs *, uint32_t, struct wg_endpoint *); 331 static void wg_peer_set_endpoint(struct wg_peer *, struct wg_endpoint *); 332 static void wg_peer_clear_src(struct wg_peer *); 333 static void wg_peer_get_endpoint(struct wg_peer *, struct wg_endpoint *); 334 static void wg_send_buf(struct wg_softc *, struct wg_endpoint *, uint8_t *, size_t); 335 static void wg_send_keepalive(struct wg_peer *); 336 static void wg_handshake(struct wg_softc *, struct wg_packet *); 337 static void wg_encrypt(struct wg_softc *, struct wg_packet *); 338 static void wg_decrypt(struct wg_softc *, struct wg_packet *); 339 static void wg_softc_handshake_receive(struct wg_softc *); 340 static void wg_softc_decrypt(struct wg_softc *); 341 static void wg_softc_encrypt(struct wg_softc *); 342 static void wg_encrypt_dispatch(struct wg_softc *); 343 static void wg_decrypt_dispatch(struct wg_softc *); 344 static void wg_deliver_out(struct wg_peer *); 345 static void wg_deliver_in(struct wg_peer *); 346 static struct wg_packet *wg_packet_alloc(struct mbuf *); 347 static void wg_packet_free(struct wg_packet *); 348 static void wg_queue_init(struct wg_queue *, const char *); 349 static void wg_queue_deinit(struct wg_queue *); 350 static size_t wg_queue_len(struct wg_queue *); 351 static int wg_queue_enqueue_handshake(struct wg_queue *, struct wg_packet *); 352 static struct wg_packet *wg_queue_dequeue_handshake(struct wg_queue *); 353 static void wg_queue_push_staged(struct wg_queue *, struct wg_packet *); 354 static void wg_queue_enlist_staged(struct wg_queue *, struct wg_packet_list *); 355 static void wg_queue_delist_staged(struct wg_queue *, struct wg_packet_list *); 356 static void wg_queue_purge(struct wg_queue *); 357 static int wg_queue_both(struct wg_queue *, struct wg_queue *, struct wg_packet *); 358 static struct wg_packet *wg_queue_dequeue_serial(struct wg_queue *); 359 static struct wg_packet *wg_queue_dequeue_parallel(struct wg_queue *); 360 static bool wg_input(struct mbuf *, int, struct inpcb *, const struct sockaddr *, void *); 361 static void wg_peer_send_staged(struct wg_peer *); 362 static int wg_clone_create(struct if_clone *ifc, char *name, size_t len, 363 struct ifc_data *ifd, if_t *ifpp); 364 static void wg_qflush(if_t); 365 static inline int determine_af_and_pullup(struct mbuf **m, sa_family_t *af); 366 static int wg_xmit(if_t, struct mbuf *, sa_family_t, uint32_t); 367 static int wg_transmit(if_t, struct mbuf *); 368 static int wg_output(if_t, struct mbuf *, const struct sockaddr *, struct route *); 369 static int wg_clone_destroy(struct if_clone *ifc, if_t ifp, 370 uint32_t flags); 371 static bool wgc_privileged(struct wg_softc *); 372 static int wgc_get(struct wg_softc *, struct wg_data_io *); 373 static int wgc_set(struct wg_softc *, struct wg_data_io *); 374 static int wg_up(struct wg_softc *); 375 static void wg_down(struct wg_softc *); 376 static void wg_reassign(if_t, struct vnet *, char *unused); 377 static void wg_init(void *); 378 static int wg_ioctl(if_t, u_long, caddr_t); 379 static void vnet_wg_init(const void *); 380 static void vnet_wg_uninit(const void *); 381 static int wg_module_init(void); 382 static void wg_module_deinit(void); 383 384 /* TODO Peer */ 385 static struct wg_peer * 386 wg_peer_create(struct wg_softc *sc, const uint8_t pub_key[WG_KEY_SIZE], 387 int *errp) 388 { 389 struct wg_peer *peer; 390 391 sx_assert(&sc->sc_lock, SX_XLOCKED); 392 393 peer = malloc(sizeof(*peer), M_WG, M_WAITOK | M_ZERO); 394 395 peer->p_remote = noise_remote_alloc(sc->sc_local, peer, pub_key); 396 if ((*errp = noise_remote_enable(peer->p_remote)) != 0) { 397 noise_remote_free(peer->p_remote, NULL); 398 free(peer, M_WG); 399 return (NULL); 400 } 401 402 peer->p_id = peer_counter++; 403 peer->p_sc = sc; 404 peer->p_tx_bytes = counter_u64_alloc(M_WAITOK); 405 peer->p_rx_bytes = counter_u64_alloc(M_WAITOK); 406 407 cookie_maker_init(&peer->p_cookie, pub_key); 408 409 rw_init(&peer->p_endpoint_lock, "wg_peer_endpoint"); 410 411 wg_queue_init(&peer->p_stage_queue, "stageq"); 412 wg_queue_init(&peer->p_encrypt_serial, "txq"); 413 wg_queue_init(&peer->p_decrypt_serial, "rxq"); 414 415 peer->p_enabled = false; 416 peer->p_need_another_keepalive = false; 417 peer->p_persistent_keepalive_interval = 0; 418 callout_init(&peer->p_new_handshake, true); 419 callout_init(&peer->p_send_keepalive, true); 420 callout_init(&peer->p_retry_handshake, true); 421 callout_init(&peer->p_persistent_keepalive, true); 422 callout_init(&peer->p_zero_key_material, true); 423 424 mtx_init(&peer->p_handshake_mtx, "peer handshake", NULL, MTX_DEF); 425 bzero(&peer->p_handshake_complete, sizeof(peer->p_handshake_complete)); 426 peer->p_handshake_retries = 0; 427 428 GROUPTASK_INIT(&peer->p_send, 0, (gtask_fn_t *)wg_deliver_out, peer); 429 taskqgroup_attach(qgroup_wg_tqg, &peer->p_send, peer, NULL, NULL, "wg send"); 430 GROUPTASK_INIT(&peer->p_recv, 0, (gtask_fn_t *)wg_deliver_in, peer); 431 taskqgroup_attach(qgroup_wg_tqg, &peer->p_recv, peer, NULL, NULL, "wg recv"); 432 433 LIST_INIT(&peer->p_aips); 434 peer->p_aips_num = 0; 435 436 TAILQ_INSERT_TAIL(&sc->sc_peers, peer, p_entry); 437 sc->sc_peers_num++; 438 439 if (if_getlinkstate(sc->sc_ifp) == LINK_STATE_UP) 440 wg_timers_enable(peer); 441 442 DPRINTF(sc, "Peer %" PRIu64 " created\n", peer->p_id); 443 return (peer); 444 } 445 446 static void 447 wg_peer_free_deferred(struct noise_remote *r) 448 { 449 struct wg_peer *peer = noise_remote_arg(r); 450 451 /* While there are no references remaining, we may still have 452 * p_{send,recv} executing (think empty queue, but wg_deliver_{in,out} 453 * needs to check the queue. We should wait for them and then free. */ 454 GROUPTASK_DRAIN(&peer->p_recv); 455 GROUPTASK_DRAIN(&peer->p_send); 456 taskqgroup_detach(qgroup_wg_tqg, &peer->p_recv); 457 taskqgroup_detach(qgroup_wg_tqg, &peer->p_send); 458 459 wg_queue_deinit(&peer->p_decrypt_serial); 460 wg_queue_deinit(&peer->p_encrypt_serial); 461 wg_queue_deinit(&peer->p_stage_queue); 462 463 counter_u64_free(peer->p_tx_bytes); 464 counter_u64_free(peer->p_rx_bytes); 465 rw_destroy(&peer->p_endpoint_lock); 466 mtx_destroy(&peer->p_handshake_mtx); 467 468 cookie_maker_free(&peer->p_cookie); 469 470 free(peer, M_WG); 471 } 472 473 static void 474 wg_peer_destroy(struct wg_peer *peer) 475 { 476 struct wg_softc *sc = peer->p_sc; 477 sx_assert(&sc->sc_lock, SX_XLOCKED); 478 479 /* Disable remote and timers. This will prevent any new handshakes 480 * occuring. */ 481 noise_remote_disable(peer->p_remote); 482 wg_timers_disable(peer); 483 484 /* Now we can remove all allowed IPs so no more packets will be routed 485 * to the peer. */ 486 wg_aip_remove_all(sc, peer); 487 488 /* Remove peer from the interface, then free. Some references may still 489 * exist to p_remote, so noise_remote_free will wait until they're all 490 * put to call wg_peer_free_deferred. */ 491 sc->sc_peers_num--; 492 TAILQ_REMOVE(&sc->sc_peers, peer, p_entry); 493 DPRINTF(sc, "Peer %" PRIu64 " destroyed\n", peer->p_id); 494 noise_remote_free(peer->p_remote, wg_peer_free_deferred); 495 } 496 497 static void 498 wg_peer_destroy_all(struct wg_softc *sc) 499 { 500 struct wg_peer *peer, *tpeer; 501 TAILQ_FOREACH_SAFE(peer, &sc->sc_peers, p_entry, tpeer) 502 wg_peer_destroy(peer); 503 } 504 505 static void 506 wg_peer_set_endpoint(struct wg_peer *peer, struct wg_endpoint *e) 507 { 508 MPASS(e->e_remote.r_sa.sa_family != 0); 509 if (memcmp(e, &peer->p_endpoint, sizeof(*e)) == 0) 510 return; 511 512 rw_wlock(&peer->p_endpoint_lock); 513 peer->p_endpoint = *e; 514 rw_wunlock(&peer->p_endpoint_lock); 515 } 516 517 static void 518 wg_peer_clear_src(struct wg_peer *peer) 519 { 520 rw_wlock(&peer->p_endpoint_lock); 521 bzero(&peer->p_endpoint.e_local, sizeof(peer->p_endpoint.e_local)); 522 rw_wunlock(&peer->p_endpoint_lock); 523 } 524 525 static void 526 wg_peer_get_endpoint(struct wg_peer *peer, struct wg_endpoint *e) 527 { 528 rw_rlock(&peer->p_endpoint_lock); 529 *e = peer->p_endpoint; 530 rw_runlock(&peer->p_endpoint_lock); 531 } 532 533 static int 534 wg_aip_addrinfo(struct wg_aip *aip, const void *baddr, uint8_t cidr) 535 { 536 #if defined(INET) || defined(INET6) 537 struct aip_addr *addr, *mask; 538 539 addr = &aip->a_addr; 540 mask = &aip->a_mask; 541 #endif 542 switch (aip->a_af) { 543 #ifdef INET 544 case AF_INET: 545 if (cidr > 32) cidr = 32; 546 addr->in = *(const struct in_addr *)baddr; 547 mask->ip = htonl(~((1LL << (32 - cidr)) - 1) & 0xffffffff); 548 addr->ip &= mask->ip; 549 addr->length = mask->length = offsetof(struct aip_addr, in) + sizeof(struct in_addr); 550 break; 551 #endif 552 #ifdef INET6 553 case AF_INET6: 554 if (cidr > 128) cidr = 128; 555 addr->in6 = *(const struct in6_addr *)baddr; 556 in6_prefixlen2mask(&mask->in6, cidr); 557 for (int i = 0; i < 4; i++) 558 addr->ip6[i] &= mask->ip6[i]; 559 addr->length = mask->length = offsetof(struct aip_addr, in6) + sizeof(struct in6_addr); 560 break; 561 #endif 562 default: 563 return (EAFNOSUPPORT); 564 } 565 566 return (0); 567 } 568 569 /* Allowed IP */ 570 static int 571 wg_aip_add(struct wg_softc *sc, struct wg_peer *peer, sa_family_t af, 572 const void *baddr, uint8_t cidr) 573 { 574 struct radix_node_head *root = NULL; 575 struct radix_node *node; 576 struct wg_aip *aip; 577 int ret = 0; 578 579 aip = malloc(sizeof(*aip), M_WG, M_WAITOK | M_ZERO); 580 aip->a_peer = peer; 581 aip->a_af = af; 582 583 ret = wg_aip_addrinfo(aip, baddr, cidr); 584 if (ret != 0) { 585 free(aip, M_WG); 586 return (ret); 587 } 588 589 root = af == AF_INET ? sc->sc_aip4 : sc->sc_aip6; 590 MPASS(root != NULL); 591 RADIX_NODE_HEAD_LOCK(root); 592 node = root->rnh_addaddr(&aip->a_addr, &aip->a_mask, &root->rh, aip->a_nodes); 593 if (node == aip->a_nodes) { 594 LIST_INSERT_HEAD(&peer->p_aips, aip, a_entry); 595 peer->p_aips_num++; 596 } else if (!node) 597 node = root->rnh_lookup(&aip->a_addr, &aip->a_mask, &root->rh); 598 if (!node) { 599 free(aip, M_WG); 600 ret = ENOMEM; 601 } else if (node != aip->a_nodes) { 602 free(aip, M_WG); 603 aip = (struct wg_aip *)node; 604 if (aip->a_peer != peer) { 605 LIST_REMOVE(aip, a_entry); 606 aip->a_peer->p_aips_num--; 607 aip->a_peer = peer; 608 LIST_INSERT_HEAD(&peer->p_aips, aip, a_entry); 609 aip->a_peer->p_aips_num++; 610 } 611 } 612 RADIX_NODE_HEAD_UNLOCK(root); 613 return (ret); 614 } 615 616 static int 617 wg_aip_del(struct wg_softc *sc, struct wg_peer *peer, sa_family_t af, 618 const void *baddr, uint8_t cidr) 619 { 620 struct radix_node_head *root = NULL; 621 struct radix_node *dnode __diagused, *node; 622 struct wg_aip *aip, addr; 623 int ret = 0; 624 625 /* 626 * We need to be sure that all padding is cleared, as it is above when 627 * new AllowedIPs are added, since we want to do a direct comparison. 628 */ 629 memset(&addr, 0, sizeof(addr)); 630 addr.a_af = af; 631 632 ret = wg_aip_addrinfo(&addr, baddr, cidr); 633 if (ret != 0) 634 return (ret); 635 636 root = af == AF_INET ? sc->sc_aip4 : sc->sc_aip6; 637 638 MPASS(root != NULL); 639 RADIX_NODE_HEAD_LOCK(root); 640 641 node = root->rnh_lookup(&addr.a_addr, &addr.a_mask, &root->rh); 642 if (node == NULL) { 643 RADIX_NODE_HEAD_UNLOCK(root); 644 return (0); 645 } 646 647 aip = (struct wg_aip *)node; 648 if (aip->a_peer != peer) { 649 /* 650 * They could have specified an allowed-ip that belonged to a 651 * different peer, in which case our job is done because the 652 * AllowedIP has been removed. 653 */ 654 RADIX_NODE_HEAD_UNLOCK(root); 655 return (0); 656 } 657 658 dnode = root->rnh_deladdr(&aip->a_addr, &aip->a_mask, &root->rh); 659 MPASS(dnode == node); 660 RADIX_NODE_HEAD_UNLOCK(root); 661 662 LIST_REMOVE(aip, a_entry); 663 peer->p_aips_num--; 664 free(aip, M_WG); 665 return (0); 666 } 667 668 static struct wg_peer * 669 wg_aip_lookup(struct wg_softc *sc, sa_family_t af, void *a) 670 { 671 struct radix_node_head *root; 672 struct radix_node *node; 673 struct wg_peer *peer; 674 struct aip_addr addr; 675 RADIX_NODE_HEAD_RLOCK_TRACKER; 676 677 switch (af) { 678 case AF_INET: 679 root = sc->sc_aip4; 680 memcpy(&addr.in, a, sizeof(addr.in)); 681 addr.length = offsetof(struct aip_addr, in) + sizeof(struct in_addr); 682 break; 683 case AF_INET6: 684 root = sc->sc_aip6; 685 memcpy(&addr.in6, a, sizeof(addr.in6)); 686 addr.length = offsetof(struct aip_addr, in6) + sizeof(struct in6_addr); 687 break; 688 default: 689 return NULL; 690 } 691 692 RADIX_NODE_HEAD_RLOCK(root); 693 node = root->rnh_matchaddr(&addr, &root->rh); 694 if (node != NULL) { 695 peer = ((struct wg_aip *)node)->a_peer; 696 noise_remote_ref(peer->p_remote); 697 } else { 698 peer = NULL; 699 } 700 RADIX_NODE_HEAD_RUNLOCK(root); 701 702 return (peer); 703 } 704 705 static void 706 wg_aip_remove_all(struct wg_softc *sc, struct wg_peer *peer) 707 { 708 struct wg_aip *aip, *taip; 709 710 RADIX_NODE_HEAD_LOCK(sc->sc_aip4); 711 LIST_FOREACH_SAFE(aip, &peer->p_aips, a_entry, taip) { 712 if (aip->a_af == AF_INET) { 713 if (sc->sc_aip4->rnh_deladdr(&aip->a_addr, &aip->a_mask, &sc->sc_aip4->rh) == NULL) 714 panic("failed to delete aip %p", aip); 715 LIST_REMOVE(aip, a_entry); 716 peer->p_aips_num--; 717 free(aip, M_WG); 718 } 719 } 720 RADIX_NODE_HEAD_UNLOCK(sc->sc_aip4); 721 722 RADIX_NODE_HEAD_LOCK(sc->sc_aip6); 723 LIST_FOREACH_SAFE(aip, &peer->p_aips, a_entry, taip) { 724 if (aip->a_af == AF_INET6) { 725 if (sc->sc_aip6->rnh_deladdr(&aip->a_addr, &aip->a_mask, &sc->sc_aip6->rh) == NULL) 726 panic("failed to delete aip %p", aip); 727 LIST_REMOVE(aip, a_entry); 728 peer->p_aips_num--; 729 free(aip, M_WG); 730 } 731 } 732 RADIX_NODE_HEAD_UNLOCK(sc->sc_aip6); 733 734 if (!LIST_EMPTY(&peer->p_aips) || peer->p_aips_num != 0) 735 panic("wg_aip_remove_all could not delete all %p", peer); 736 } 737 738 static int 739 wg_socket_init(struct wg_softc *sc, in_port_t port) 740 { 741 struct ucred *cred = sc->sc_ucred; 742 struct socket *so4 = NULL, *so6 = NULL; 743 int rc; 744 745 sx_assert(&sc->sc_lock, SX_XLOCKED); 746 747 if (!cred) 748 return (EBUSY); 749 750 /* 751 * For socket creation, we use the creds of the thread that created the 752 * tunnel rather than the current thread to maintain the semantics that 753 * WireGuard has on Linux with network namespaces -- that the sockets 754 * are created in their home vnet so that they can be configured and 755 * functionally attached to a foreign vnet as the jail's only interface 756 * to the network. 757 */ 758 #ifdef INET 759 rc = socreate(AF_INET, &so4, SOCK_DGRAM, IPPROTO_UDP, cred, curthread); 760 if (rc) 761 goto out; 762 763 rc = udp_set_kernel_tunneling(so4, wg_input, NULL, sc); 764 /* 765 * udp_set_kernel_tunneling can only fail if there is already a tunneling function set. 766 * This should never happen with a new socket. 767 */ 768 MPASS(rc == 0); 769 #endif 770 771 #ifdef INET6 772 rc = socreate(AF_INET6, &so6, SOCK_DGRAM, IPPROTO_UDP, cred, curthread); 773 if (rc) 774 goto out; 775 rc = udp_set_kernel_tunneling(so6, wg_input, NULL, sc); 776 MPASS(rc == 0); 777 #endif 778 779 if (sc->sc_socket.so_user_cookie) { 780 rc = wg_socket_set_sockopt(so4, so6, SO_USER_COOKIE, &sc->sc_socket.so_user_cookie, sizeof(sc->sc_socket.so_user_cookie)); 781 if (rc) 782 goto out; 783 } 784 rc = wg_socket_set_sockopt(so4, so6, SO_SETFIB, &sc->sc_socket.so_fibnum, sizeof(sc->sc_socket.so_fibnum)); 785 if (rc) 786 goto out; 787 788 rc = wg_socket_bind(&so4, &so6, &port); 789 if (!rc) { 790 sc->sc_socket.so_port = port; 791 wg_socket_set(sc, so4, so6); 792 } 793 out: 794 if (rc) { 795 if (so4 != NULL) 796 soclose(so4); 797 if (so6 != NULL) 798 soclose(so6); 799 } 800 return (rc); 801 } 802 803 static int wg_socket_set_sockopt(struct socket *so4, struct socket *so6, int name, void *val, size_t len) 804 { 805 int ret4 = 0, ret6 = 0; 806 struct sockopt sopt = { 807 .sopt_dir = SOPT_SET, 808 .sopt_level = SOL_SOCKET, 809 .sopt_name = name, 810 .sopt_val = val, 811 .sopt_valsize = len 812 }; 813 814 if (so4) 815 ret4 = sosetopt(so4, &sopt); 816 if (so6) 817 ret6 = sosetopt(so6, &sopt); 818 return (ret4 ?: ret6); 819 } 820 821 static int wg_socket_set_cookie(struct wg_softc *sc, uint32_t user_cookie) 822 { 823 struct wg_socket *so = &sc->sc_socket; 824 int ret; 825 826 sx_assert(&sc->sc_lock, SX_XLOCKED); 827 ret = wg_socket_set_sockopt(so->so_so4, so->so_so6, SO_USER_COOKIE, &user_cookie, sizeof(user_cookie)); 828 if (!ret) 829 so->so_user_cookie = user_cookie; 830 return (ret); 831 } 832 833 static int wg_socket_set_fibnum(struct wg_softc *sc, int fibnum) 834 { 835 struct wg_socket *so = &sc->sc_socket; 836 int ret; 837 838 sx_assert(&sc->sc_lock, SX_XLOCKED); 839 840 ret = wg_socket_set_sockopt(so->so_so4, so->so_so6, SO_SETFIB, &fibnum, sizeof(fibnum)); 841 if (!ret) 842 so->so_fibnum = fibnum; 843 return (ret); 844 } 845 846 static void 847 wg_socket_uninit(struct wg_softc *sc) 848 { 849 wg_socket_set(sc, NULL, NULL); 850 } 851 852 static void 853 wg_socket_set(struct wg_softc *sc, struct socket *new_so4, struct socket *new_so6) 854 { 855 struct wg_socket *so = &sc->sc_socket; 856 struct socket *so4, *so6; 857 858 sx_assert(&sc->sc_lock, SX_XLOCKED); 859 860 so4 = atomic_load_ptr(&so->so_so4); 861 so6 = atomic_load_ptr(&so->so_so6); 862 atomic_store_ptr(&so->so_so4, new_so4); 863 atomic_store_ptr(&so->so_so6, new_so6); 864 865 if (!so4 && !so6) 866 return; 867 NET_EPOCH_WAIT(); 868 if (so4) 869 soclose(so4); 870 if (so6) 871 soclose(so6); 872 } 873 874 static int 875 wg_socket_bind(struct socket **in_so4, struct socket **in_so6, in_port_t *requested_port) 876 { 877 struct socket *so4 = *in_so4, *so6 = *in_so6; 878 int ret4 = 0, ret6 = 0; 879 in_port_t port = *requested_port; 880 struct sockaddr_in sin = { 881 .sin_len = sizeof(struct sockaddr_in), 882 .sin_family = AF_INET, 883 .sin_port = htons(port) 884 }; 885 struct sockaddr_in6 sin6 = { 886 .sin6_len = sizeof(struct sockaddr_in6), 887 .sin6_family = AF_INET6, 888 .sin6_port = htons(port) 889 }; 890 891 if (so4) { 892 ret4 = sobind(so4, (struct sockaddr *)&sin, curthread); 893 if (ret4 && ret4 != EADDRNOTAVAIL) 894 return (ret4); 895 if (!ret4 && !sin.sin_port) { 896 struct sockaddr_in bound_sin = 897 { .sin_len = sizeof(bound_sin) }; 898 int ret; 899 900 ret = sosockaddr(so4, (struct sockaddr *)&bound_sin); 901 if (ret) 902 return (ret); 903 port = ntohs(bound_sin.sin_port); 904 sin6.sin6_port = bound_sin.sin_port; 905 } 906 } 907 908 if (so6) { 909 ret6 = sobind(so6, (struct sockaddr *)&sin6, curthread); 910 if (ret6 && ret6 != EADDRNOTAVAIL) 911 return (ret6); 912 if (!ret6 && !sin6.sin6_port) { 913 struct sockaddr_in6 bound_sin6 = 914 { .sin6_len = sizeof(bound_sin6) }; 915 int ret; 916 917 ret = sosockaddr(so6, (struct sockaddr *)&bound_sin6); 918 if (ret) 919 return (ret); 920 port = ntohs(bound_sin6.sin6_port); 921 } 922 } 923 924 if (ret4 && ret6) 925 return (ret4); 926 *requested_port = port; 927 if (ret4 && !ret6 && so4) { 928 soclose(so4); 929 *in_so4 = NULL; 930 } else if (ret6 && !ret4 && so6) { 931 soclose(so6); 932 *in_so6 = NULL; 933 } 934 return (0); 935 } 936 937 static int 938 wg_send(struct wg_softc *sc, struct wg_endpoint *e, struct mbuf *m) 939 { 940 struct epoch_tracker et; 941 struct sockaddr *sa; 942 struct wg_socket *so = &sc->sc_socket; 943 struct socket *so4, *so6; 944 struct mbuf *control = NULL; 945 int ret = 0; 946 size_t len = m->m_pkthdr.len; 947 948 /* Get local control address before locking */ 949 if (e->e_remote.r_sa.sa_family == AF_INET) { 950 if (e->e_local.l_in.s_addr != INADDR_ANY) 951 control = sbcreatecontrol((caddr_t)&e->e_local.l_in, 952 sizeof(struct in_addr), IP_SENDSRCADDR, 953 IPPROTO_IP, M_NOWAIT); 954 #ifdef INET6 955 } else if (e->e_remote.r_sa.sa_family == AF_INET6) { 956 if (!IN6_IS_ADDR_UNSPECIFIED(&e->e_local.l_in6)) 957 control = sbcreatecontrol((caddr_t)&e->e_local.l_pktinfo6, 958 sizeof(struct in6_pktinfo), IPV6_PKTINFO, 959 IPPROTO_IPV6, M_NOWAIT); 960 #endif 961 } else { 962 m_freem(m); 963 return (EAFNOSUPPORT); 964 } 965 966 /* Get remote address */ 967 sa = &e->e_remote.r_sa; 968 969 NET_EPOCH_ENTER(et); 970 so4 = atomic_load_ptr(&so->so_so4); 971 so6 = atomic_load_ptr(&so->so_so6); 972 if (e->e_remote.r_sa.sa_family == AF_INET && so4 != NULL) 973 ret = sosend(so4, sa, NULL, m, control, 0, curthread); 974 else if (e->e_remote.r_sa.sa_family == AF_INET6 && so6 != NULL) 975 ret = sosend(so6, sa, NULL, m, control, 0, curthread); 976 else { 977 ret = ENOTCONN; 978 m_freem(control); 979 m_freem(m); 980 } 981 NET_EPOCH_EXIT(et); 982 if (ret == 0) { 983 if_inc_counter(sc->sc_ifp, IFCOUNTER_OPACKETS, 1); 984 if_inc_counter(sc->sc_ifp, IFCOUNTER_OBYTES, len); 985 } 986 return (ret); 987 } 988 989 static void 990 wg_send_buf(struct wg_softc *sc, struct wg_endpoint *e, uint8_t *buf, size_t len) 991 { 992 struct mbuf *m; 993 int ret = 0; 994 bool retried = false; 995 996 retry: 997 m = m_get2(len, M_NOWAIT, MT_DATA, M_PKTHDR); 998 if (!m) { 999 ret = ENOMEM; 1000 goto out; 1001 } 1002 m_copyback(m, 0, len, buf); 1003 1004 if (ret == 0) { 1005 ret = wg_send(sc, e, m); 1006 /* Retry if we couldn't bind to e->e_local */ 1007 if (ret == EADDRNOTAVAIL && !retried) { 1008 bzero(&e->e_local, sizeof(e->e_local)); 1009 retried = true; 1010 goto retry; 1011 } 1012 } else { 1013 ret = wg_send(sc, e, m); 1014 } 1015 out: 1016 if (ret) 1017 DPRINTF(sc, "Unable to send packet: %d\n", ret); 1018 } 1019 1020 /* Timers */ 1021 static void 1022 wg_timers_enable(struct wg_peer *peer) 1023 { 1024 atomic_store_bool(&peer->p_enabled, true); 1025 wg_timers_run_persistent_keepalive(peer); 1026 } 1027 1028 static void 1029 wg_timers_disable(struct wg_peer *peer) 1030 { 1031 /* By setting p_enabled = false, then calling NET_EPOCH_WAIT, we can be 1032 * sure no new handshakes are created after the wait. This is because 1033 * all callout_resets (scheduling the callout) are guarded by 1034 * p_enabled. We can be sure all sections that read p_enabled and then 1035 * optionally call callout_reset are finished as they are surrounded by 1036 * NET_EPOCH_{ENTER,EXIT}. 1037 * 1038 * However, as new callouts may be scheduled during NET_EPOCH_WAIT (but 1039 * not after), we stop all callouts leaving no callouts active. 1040 * 1041 * We should also pull NET_EPOCH_WAIT out of the FOREACH(peer) loops, but the 1042 * performance impact is acceptable for the time being. */ 1043 atomic_store_bool(&peer->p_enabled, false); 1044 NET_EPOCH_WAIT(); 1045 atomic_store_bool(&peer->p_need_another_keepalive, false); 1046 1047 callout_stop(&peer->p_new_handshake); 1048 callout_stop(&peer->p_send_keepalive); 1049 callout_stop(&peer->p_retry_handshake); 1050 callout_stop(&peer->p_persistent_keepalive); 1051 callout_stop(&peer->p_zero_key_material); 1052 } 1053 1054 static void 1055 wg_timers_set_persistent_keepalive(struct wg_peer *peer, uint16_t interval) 1056 { 1057 struct epoch_tracker et; 1058 if (interval != peer->p_persistent_keepalive_interval) { 1059 atomic_store_16(&peer->p_persistent_keepalive_interval, interval); 1060 NET_EPOCH_ENTER(et); 1061 if (atomic_load_bool(&peer->p_enabled)) 1062 wg_timers_run_persistent_keepalive(peer); 1063 NET_EPOCH_EXIT(et); 1064 } 1065 } 1066 1067 static void 1068 wg_timers_get_last_handshake(struct wg_peer *peer, struct wg_timespec64 *time) 1069 { 1070 mtx_lock(&peer->p_handshake_mtx); 1071 time->tv_sec = peer->p_handshake_complete.tv_sec; 1072 time->tv_nsec = peer->p_handshake_complete.tv_nsec; 1073 mtx_unlock(&peer->p_handshake_mtx); 1074 } 1075 1076 static void 1077 wg_timers_event_data_sent(struct wg_peer *peer) 1078 { 1079 struct epoch_tracker et; 1080 NET_EPOCH_ENTER(et); 1081 if (atomic_load_bool(&peer->p_enabled) && 1082 !callout_pending(&peer->p_new_handshake)) 1083 callout_reset(&peer->p_new_handshake, MSEC_2_TICKS( 1084 NEW_HANDSHAKE_TIMEOUT * 1000 + 1085 arc4random_uniform(REKEY_TIMEOUT_JITTER)), 1086 wg_timers_run_new_handshake, peer); 1087 NET_EPOCH_EXIT(et); 1088 } 1089 1090 static void 1091 wg_timers_event_data_received(struct wg_peer *peer) 1092 { 1093 struct epoch_tracker et; 1094 NET_EPOCH_ENTER(et); 1095 if (atomic_load_bool(&peer->p_enabled)) { 1096 if (!callout_pending(&peer->p_send_keepalive)) 1097 callout_reset(&peer->p_send_keepalive, 1098 MSEC_2_TICKS(KEEPALIVE_TIMEOUT * 1000), 1099 wg_timers_run_send_keepalive, peer); 1100 else 1101 atomic_store_bool(&peer->p_need_another_keepalive, 1102 true); 1103 } 1104 NET_EPOCH_EXIT(et); 1105 } 1106 1107 static void 1108 wg_timers_event_any_authenticated_packet_sent(struct wg_peer *peer) 1109 { 1110 callout_stop(&peer->p_send_keepalive); 1111 } 1112 1113 static void 1114 wg_timers_event_any_authenticated_packet_received(struct wg_peer *peer) 1115 { 1116 callout_stop(&peer->p_new_handshake); 1117 } 1118 1119 static void 1120 wg_timers_event_any_authenticated_packet_traversal(struct wg_peer *peer) 1121 { 1122 struct epoch_tracker et; 1123 uint16_t interval; 1124 NET_EPOCH_ENTER(et); 1125 interval = atomic_load_16(&peer->p_persistent_keepalive_interval); 1126 if (atomic_load_bool(&peer->p_enabled) && interval > 0) 1127 callout_reset(&peer->p_persistent_keepalive, 1128 MSEC_2_TICKS(interval * 1000), 1129 wg_timers_run_persistent_keepalive, peer); 1130 NET_EPOCH_EXIT(et); 1131 } 1132 1133 static void 1134 wg_timers_event_handshake_initiated(struct wg_peer *peer) 1135 { 1136 struct epoch_tracker et; 1137 NET_EPOCH_ENTER(et); 1138 if (atomic_load_bool(&peer->p_enabled)) 1139 callout_reset(&peer->p_retry_handshake, MSEC_2_TICKS( 1140 REKEY_TIMEOUT * 1000 + 1141 arc4random_uniform(REKEY_TIMEOUT_JITTER)), 1142 wg_timers_run_retry_handshake, peer); 1143 NET_EPOCH_EXIT(et); 1144 } 1145 1146 static void 1147 wg_timers_event_handshake_complete(struct wg_peer *peer) 1148 { 1149 struct epoch_tracker et; 1150 NET_EPOCH_ENTER(et); 1151 if (atomic_load_bool(&peer->p_enabled)) { 1152 mtx_lock(&peer->p_handshake_mtx); 1153 callout_stop(&peer->p_retry_handshake); 1154 peer->p_handshake_retries = 0; 1155 getnanotime(&peer->p_handshake_complete); 1156 mtx_unlock(&peer->p_handshake_mtx); 1157 wg_timers_run_send_keepalive(peer); 1158 } 1159 NET_EPOCH_EXIT(et); 1160 } 1161 1162 static void 1163 wg_timers_event_session_derived(struct wg_peer *peer) 1164 { 1165 struct epoch_tracker et; 1166 NET_EPOCH_ENTER(et); 1167 if (atomic_load_bool(&peer->p_enabled)) 1168 callout_reset(&peer->p_zero_key_material, 1169 MSEC_2_TICKS(REJECT_AFTER_TIME * 3 * 1000), 1170 wg_timers_run_zero_key_material, peer); 1171 NET_EPOCH_EXIT(et); 1172 } 1173 1174 static void 1175 wg_timers_event_want_initiation(struct wg_peer *peer) 1176 { 1177 struct epoch_tracker et; 1178 NET_EPOCH_ENTER(et); 1179 if (atomic_load_bool(&peer->p_enabled)) 1180 wg_timers_run_send_initiation(peer, false); 1181 NET_EPOCH_EXIT(et); 1182 } 1183 1184 static void 1185 wg_timers_run_send_initiation(struct wg_peer *peer, bool is_retry) 1186 { 1187 if (!is_retry) 1188 peer->p_handshake_retries = 0; 1189 if (noise_remote_initiation_expired(peer->p_remote) == ETIMEDOUT) 1190 wg_send_initiation(peer); 1191 } 1192 1193 static void 1194 wg_timers_run_retry_handshake(void *_peer) 1195 { 1196 struct epoch_tracker et; 1197 struct wg_peer *peer = _peer; 1198 1199 mtx_lock(&peer->p_handshake_mtx); 1200 if (peer->p_handshake_retries <= MAX_TIMER_HANDSHAKES) { 1201 peer->p_handshake_retries++; 1202 mtx_unlock(&peer->p_handshake_mtx); 1203 1204 DPRINTF(peer->p_sc, "Handshake for peer %" PRIu64 " did not complete " 1205 "after %d seconds, retrying (try %d)\n", peer->p_id, 1206 REKEY_TIMEOUT, peer->p_handshake_retries + 1); 1207 wg_peer_clear_src(peer); 1208 wg_timers_run_send_initiation(peer, true); 1209 } else { 1210 mtx_unlock(&peer->p_handshake_mtx); 1211 1212 DPRINTF(peer->p_sc, "Handshake for peer %" PRIu64 " did not complete " 1213 "after %d retries, giving up\n", peer->p_id, 1214 MAX_TIMER_HANDSHAKES + 2); 1215 1216 callout_stop(&peer->p_send_keepalive); 1217 wg_queue_purge(&peer->p_stage_queue); 1218 NET_EPOCH_ENTER(et); 1219 if (atomic_load_bool(&peer->p_enabled) && 1220 !callout_pending(&peer->p_zero_key_material)) 1221 callout_reset(&peer->p_zero_key_material, 1222 MSEC_2_TICKS(REJECT_AFTER_TIME * 3 * 1000), 1223 wg_timers_run_zero_key_material, peer); 1224 NET_EPOCH_EXIT(et); 1225 } 1226 } 1227 1228 static void 1229 wg_timers_run_send_keepalive(void *_peer) 1230 { 1231 struct epoch_tracker et; 1232 struct wg_peer *peer = _peer; 1233 1234 wg_send_keepalive(peer); 1235 NET_EPOCH_ENTER(et); 1236 if (atomic_load_bool(&peer->p_enabled) && 1237 atomic_load_bool(&peer->p_need_another_keepalive)) { 1238 atomic_store_bool(&peer->p_need_another_keepalive, false); 1239 callout_reset(&peer->p_send_keepalive, 1240 MSEC_2_TICKS(KEEPALIVE_TIMEOUT * 1000), 1241 wg_timers_run_send_keepalive, peer); 1242 } 1243 NET_EPOCH_EXIT(et); 1244 } 1245 1246 static void 1247 wg_timers_run_new_handshake(void *_peer) 1248 { 1249 struct wg_peer *peer = _peer; 1250 1251 DPRINTF(peer->p_sc, "Retrying handshake with peer %" PRIu64 " because we " 1252 "stopped hearing back after %d seconds\n", 1253 peer->p_id, NEW_HANDSHAKE_TIMEOUT); 1254 1255 wg_peer_clear_src(peer); 1256 wg_timers_run_send_initiation(peer, false); 1257 } 1258 1259 static void 1260 wg_timers_run_zero_key_material(void *_peer) 1261 { 1262 struct wg_peer *peer = _peer; 1263 1264 DPRINTF(peer->p_sc, "Zeroing out keys for peer %" PRIu64 ", since we " 1265 "haven't received a new one in %d seconds\n", 1266 peer->p_id, REJECT_AFTER_TIME * 3); 1267 noise_remote_keypairs_clear(peer->p_remote); 1268 } 1269 1270 static void 1271 wg_timers_run_persistent_keepalive(void *_peer) 1272 { 1273 struct wg_peer *peer = _peer; 1274 1275 if (atomic_load_16(&peer->p_persistent_keepalive_interval) > 0) 1276 wg_send_keepalive(peer); 1277 } 1278 1279 /* TODO Handshake */ 1280 static void 1281 wg_peer_send_buf(struct wg_peer *peer, uint8_t *buf, size_t len) 1282 { 1283 struct wg_endpoint endpoint; 1284 1285 counter_u64_add(peer->p_tx_bytes, len); 1286 wg_timers_event_any_authenticated_packet_traversal(peer); 1287 wg_timers_event_any_authenticated_packet_sent(peer); 1288 wg_peer_get_endpoint(peer, &endpoint); 1289 wg_send_buf(peer->p_sc, &endpoint, buf, len); 1290 } 1291 1292 static void 1293 wg_send_initiation(struct wg_peer *peer) 1294 { 1295 struct wg_pkt_initiation pkt; 1296 1297 if (noise_create_initiation(peer->p_remote, &pkt.s_idx, pkt.ue, 1298 pkt.es, pkt.ets) != 0) 1299 return; 1300 1301 DPRINTF(peer->p_sc, "Sending handshake initiation to peer %" PRIu64 "\n", peer->p_id); 1302 1303 pkt.t = WG_PKT_INITIATION; 1304 cookie_maker_mac(&peer->p_cookie, &pkt.m, &pkt, 1305 sizeof(pkt) - sizeof(pkt.m)); 1306 wg_peer_send_buf(peer, (uint8_t *)&pkt, sizeof(pkt)); 1307 wg_timers_event_handshake_initiated(peer); 1308 } 1309 1310 static void 1311 wg_send_response(struct wg_peer *peer) 1312 { 1313 struct wg_pkt_response pkt; 1314 1315 if (noise_create_response(peer->p_remote, &pkt.s_idx, &pkt.r_idx, 1316 pkt.ue, pkt.en) != 0) 1317 return; 1318 1319 DPRINTF(peer->p_sc, "Sending handshake response to peer %" PRIu64 "\n", peer->p_id); 1320 1321 wg_timers_event_session_derived(peer); 1322 pkt.t = WG_PKT_RESPONSE; 1323 cookie_maker_mac(&peer->p_cookie, &pkt.m, &pkt, 1324 sizeof(pkt)-sizeof(pkt.m)); 1325 wg_peer_send_buf(peer, (uint8_t*)&pkt, sizeof(pkt)); 1326 } 1327 1328 static void 1329 wg_send_cookie(struct wg_softc *sc, struct cookie_macs *cm, uint32_t idx, 1330 struct wg_endpoint *e) 1331 { 1332 struct wg_pkt_cookie pkt; 1333 1334 DPRINTF(sc, "Sending cookie response for denied handshake message\n"); 1335 1336 pkt.t = WG_PKT_COOKIE; 1337 pkt.r_idx = idx; 1338 1339 cookie_checker_create_payload(&sc->sc_cookie, cm, pkt.nonce, 1340 pkt.ec, &e->e_remote.r_sa); 1341 wg_send_buf(sc, e, (uint8_t *)&pkt, sizeof(pkt)); 1342 } 1343 1344 static void 1345 wg_send_keepalive(struct wg_peer *peer) 1346 { 1347 struct wg_packet *pkt; 1348 struct mbuf *m; 1349 1350 if (wg_queue_len(&peer->p_stage_queue) > 0) 1351 goto send; 1352 if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) 1353 return; 1354 if ((pkt = wg_packet_alloc(m)) == NULL) { 1355 m_freem(m); 1356 return; 1357 } 1358 wg_queue_push_staged(&peer->p_stage_queue, pkt); 1359 DPRINTF(peer->p_sc, "Sending keepalive packet to peer %" PRIu64 "\n", peer->p_id); 1360 send: 1361 wg_peer_send_staged(peer); 1362 } 1363 1364 static void 1365 wg_handshake(struct wg_softc *sc, struct wg_packet *pkt) 1366 { 1367 struct wg_pkt_initiation *init; 1368 struct wg_pkt_response *resp; 1369 struct wg_pkt_cookie *cook; 1370 struct wg_endpoint *e; 1371 struct wg_peer *peer; 1372 struct mbuf *m; 1373 struct noise_remote *remote = NULL; 1374 int res; 1375 bool underload = false; 1376 static sbintime_t wg_last_underload; /* sbinuptime */ 1377 1378 underload = wg_queue_len(&sc->sc_handshake_queue) >= MAX_QUEUED_HANDSHAKES / 8; 1379 if (underload) { 1380 wg_last_underload = getsbinuptime(); 1381 } else if (wg_last_underload) { 1382 underload = wg_last_underload + UNDERLOAD_TIMEOUT * SBT_1S > getsbinuptime(); 1383 if (!underload) 1384 wg_last_underload = 0; 1385 } 1386 1387 m = pkt->p_mbuf; 1388 e = &pkt->p_endpoint; 1389 1390 if ((pkt->p_mbuf = m = m_pullup(m, m->m_pkthdr.len)) == NULL) 1391 goto error; 1392 1393 switch (*mtod(m, uint32_t *)) { 1394 case WG_PKT_INITIATION: 1395 init = mtod(m, struct wg_pkt_initiation *); 1396 1397 res = cookie_checker_validate_macs(&sc->sc_cookie, &init->m, 1398 init, sizeof(*init) - sizeof(init->m), 1399 underload, &e->e_remote.r_sa, 1400 if_getvnet(sc->sc_ifp)); 1401 1402 if (res == EINVAL) { 1403 DPRINTF(sc, "Invalid initiation MAC\n"); 1404 goto error; 1405 } else if (res == ECONNREFUSED) { 1406 DPRINTF(sc, "Handshake ratelimited\n"); 1407 goto error; 1408 } else if (res == EAGAIN) { 1409 wg_send_cookie(sc, &init->m, init->s_idx, e); 1410 goto error; 1411 } else if (res != 0) { 1412 panic("unexpected response: %d\n", res); 1413 } 1414 1415 if (noise_consume_initiation(sc->sc_local, &remote, 1416 init->s_idx, init->ue, init->es, init->ets) != 0) { 1417 DPRINTF(sc, "Invalid handshake initiation\n"); 1418 goto error; 1419 } 1420 1421 peer = noise_remote_arg(remote); 1422 1423 DPRINTF(sc, "Receiving handshake initiation from peer %" PRIu64 "\n", peer->p_id); 1424 1425 wg_peer_set_endpoint(peer, e); 1426 wg_send_response(peer); 1427 break; 1428 case WG_PKT_RESPONSE: 1429 resp = mtod(m, struct wg_pkt_response *); 1430 1431 res = cookie_checker_validate_macs(&sc->sc_cookie, &resp->m, 1432 resp, sizeof(*resp) - sizeof(resp->m), 1433 underload, &e->e_remote.r_sa, 1434 if_getvnet(sc->sc_ifp)); 1435 1436 if (res == EINVAL) { 1437 DPRINTF(sc, "Invalid response MAC\n"); 1438 goto error; 1439 } else if (res == ECONNREFUSED) { 1440 DPRINTF(sc, "Handshake ratelimited\n"); 1441 goto error; 1442 } else if (res == EAGAIN) { 1443 wg_send_cookie(sc, &resp->m, resp->s_idx, e); 1444 goto error; 1445 } else if (res != 0) { 1446 panic("unexpected response: %d\n", res); 1447 } 1448 1449 if (noise_consume_response(sc->sc_local, &remote, 1450 resp->s_idx, resp->r_idx, resp->ue, resp->en) != 0) { 1451 DPRINTF(sc, "Invalid handshake response\n"); 1452 goto error; 1453 } 1454 1455 peer = noise_remote_arg(remote); 1456 DPRINTF(sc, "Receiving handshake response from peer %" PRIu64 "\n", peer->p_id); 1457 1458 wg_peer_set_endpoint(peer, e); 1459 wg_timers_event_session_derived(peer); 1460 wg_timers_event_handshake_complete(peer); 1461 break; 1462 case WG_PKT_COOKIE: 1463 cook = mtod(m, struct wg_pkt_cookie *); 1464 1465 if ((remote = noise_remote_index(sc->sc_local, cook->r_idx)) == NULL) { 1466 DPRINTF(sc, "Unknown cookie index\n"); 1467 goto error; 1468 } 1469 1470 peer = noise_remote_arg(remote); 1471 1472 if (cookie_maker_consume_payload(&peer->p_cookie, 1473 cook->nonce, cook->ec) == 0) { 1474 DPRINTF(sc, "Receiving cookie response\n"); 1475 } else { 1476 DPRINTF(sc, "Could not decrypt cookie response\n"); 1477 goto error; 1478 } 1479 1480 goto not_authenticated; 1481 default: 1482 panic("invalid packet in handshake queue"); 1483 } 1484 1485 wg_timers_event_any_authenticated_packet_received(peer); 1486 wg_timers_event_any_authenticated_packet_traversal(peer); 1487 1488 not_authenticated: 1489 counter_u64_add(peer->p_rx_bytes, m->m_pkthdr.len); 1490 if_inc_counter(sc->sc_ifp, IFCOUNTER_IPACKETS, 1); 1491 if_inc_counter(sc->sc_ifp, IFCOUNTER_IBYTES, m->m_pkthdr.len); 1492 error: 1493 if (remote != NULL) 1494 noise_remote_put(remote); 1495 wg_packet_free(pkt); 1496 } 1497 1498 static void 1499 wg_softc_handshake_receive(struct wg_softc *sc) 1500 { 1501 struct wg_packet *pkt; 1502 while ((pkt = wg_queue_dequeue_handshake(&sc->sc_handshake_queue)) != NULL) 1503 wg_handshake(sc, pkt); 1504 } 1505 1506 static void 1507 wg_mbuf_reset(struct mbuf *m) 1508 { 1509 1510 struct m_tag *t, *tmp; 1511 1512 /* 1513 * We want to reset the mbuf to a newly allocated state, containing 1514 * just the packet contents. Unfortunately FreeBSD doesn't seem to 1515 * offer this anywhere, so we have to make it up as we go. If we can 1516 * get this in kern/kern_mbuf.c, that would be best. 1517 * 1518 * Notice: this may break things unexpectedly but it is better to fail 1519 * closed in the extreme case than leak informtion in every 1520 * case. 1521 * 1522 * With that said, all this attempts to do is remove any extraneous 1523 * information that could be present. 1524 */ 1525 1526 M_ASSERTPKTHDR(m); 1527 1528 m->m_flags &= ~(M_BCAST|M_MCAST|M_VLANTAG|M_PROMISC|M_PROTOFLAGS); 1529 1530 M_HASHTYPE_CLEAR(m); 1531 #ifdef NUMA 1532 m->m_pkthdr.numa_domain = M_NODOM; 1533 #endif 1534 SLIST_FOREACH_SAFE(t, &m->m_pkthdr.tags, m_tag_link, tmp) { 1535 if ((t->m_tag_id != 0 || t->m_tag_cookie != MTAG_WGLOOP) && 1536 t->m_tag_id != PACKET_TAG_MACLABEL) 1537 m_tag_delete(m, t); 1538 } 1539 1540 KASSERT((m->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0, 1541 ("%s: mbuf %p has a send tag", __func__, m)); 1542 1543 m->m_pkthdr.csum_flags = 0; 1544 m->m_pkthdr.PH_per.sixtyfour[0] = 0; 1545 m->m_pkthdr.PH_loc.sixtyfour[0] = 0; 1546 } 1547 1548 static inline unsigned int 1549 calculate_padding(struct wg_packet *pkt) 1550 { 1551 unsigned int padded_size, last_unit = pkt->p_mbuf->m_pkthdr.len; 1552 1553 /* Keepalive packets don't set p_mtu, but also have a length of zero. */ 1554 if (__predict_false(pkt->p_mtu == 0)) { 1555 padded_size = (last_unit + (WG_PKT_PADDING - 1)) & 1556 ~(WG_PKT_PADDING - 1); 1557 return (padded_size - last_unit); 1558 } 1559 1560 if (__predict_false(last_unit > pkt->p_mtu)) 1561 last_unit %= pkt->p_mtu; 1562 1563 padded_size = (last_unit + (WG_PKT_PADDING - 1)) & ~(WG_PKT_PADDING - 1); 1564 if (pkt->p_mtu < padded_size) 1565 padded_size = pkt->p_mtu; 1566 return (padded_size - last_unit); 1567 } 1568 1569 static void 1570 wg_encrypt(struct wg_softc *sc, struct wg_packet *pkt) 1571 { 1572 static const uint8_t padding[WG_PKT_PADDING] = { 0 }; 1573 struct wg_pkt_data *data; 1574 struct wg_peer *peer; 1575 struct noise_remote *remote; 1576 struct mbuf *m; 1577 uint32_t idx; 1578 unsigned int padlen; 1579 enum wg_ring_state state = WG_PACKET_DEAD; 1580 1581 remote = noise_keypair_remote(pkt->p_keypair); 1582 peer = noise_remote_arg(remote); 1583 m = pkt->p_mbuf; 1584 1585 /* Pad the packet */ 1586 padlen = calculate_padding(pkt); 1587 if (padlen != 0 && !m_append(m, padlen, padding)) 1588 goto out; 1589 1590 /* Do encryption */ 1591 if (noise_keypair_encrypt(pkt->p_keypair, &idx, pkt->p_nonce, m) != 0) 1592 goto out; 1593 1594 /* Put header into packet */ 1595 M_PREPEND(m, sizeof(struct wg_pkt_data), M_NOWAIT); 1596 if (m == NULL) 1597 goto out; 1598 data = mtod(m, struct wg_pkt_data *); 1599 data->t = WG_PKT_DATA; 1600 data->r_idx = idx; 1601 data->nonce = htole64(pkt->p_nonce); 1602 1603 wg_mbuf_reset(m); 1604 state = WG_PACKET_CRYPTED; 1605 out: 1606 pkt->p_mbuf = m; 1607 atomic_store_rel_int(&pkt->p_state, state); 1608 GROUPTASK_ENQUEUE(&peer->p_send); 1609 noise_remote_put(remote); 1610 } 1611 1612 static void 1613 wg_decrypt(struct wg_softc *sc, struct wg_packet *pkt) 1614 { 1615 struct wg_peer *peer, *allowed_peer; 1616 struct noise_remote *remote; 1617 struct mbuf *m; 1618 int len; 1619 enum wg_ring_state state = WG_PACKET_DEAD; 1620 1621 remote = noise_keypair_remote(pkt->p_keypair); 1622 peer = noise_remote_arg(remote); 1623 m = pkt->p_mbuf; 1624 1625 /* Read nonce and then adjust to remove the header. */ 1626 pkt->p_nonce = le64toh(mtod(m, struct wg_pkt_data *)->nonce); 1627 m_adj(m, sizeof(struct wg_pkt_data)); 1628 1629 if (noise_keypair_decrypt(pkt->p_keypair, pkt->p_nonce, m) != 0) 1630 goto out; 1631 1632 /* A packet with length 0 is a keepalive packet */ 1633 if (__predict_false(m->m_pkthdr.len == 0)) { 1634 DPRINTF(sc, "Receiving keepalive packet from peer " 1635 "%" PRIu64 "\n", peer->p_id); 1636 state = WG_PACKET_CRYPTED; 1637 goto out; 1638 } 1639 1640 /* 1641 * We can let the network stack handle the intricate validation of the 1642 * IP header, we just worry about the sizeof and the version, so we can 1643 * read the source address in wg_aip_lookup. 1644 */ 1645 1646 if (determine_af_and_pullup(&m, &pkt->p_af) == 0) { 1647 if (pkt->p_af == AF_INET) { 1648 struct ip *ip = mtod(m, struct ip *); 1649 allowed_peer = wg_aip_lookup(sc, AF_INET, &ip->ip_src); 1650 len = ntohs(ip->ip_len); 1651 if (len >= sizeof(struct ip) && len < m->m_pkthdr.len) 1652 m_adj(m, len - m->m_pkthdr.len); 1653 } else if (pkt->p_af == AF_INET6) { 1654 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); 1655 allowed_peer = wg_aip_lookup(sc, AF_INET6, &ip6->ip6_src); 1656 len = ntohs(ip6->ip6_plen) + sizeof(struct ip6_hdr); 1657 if (len < m->m_pkthdr.len) 1658 m_adj(m, len - m->m_pkthdr.len); 1659 } else 1660 panic("determine_af_and_pullup returned unexpected value"); 1661 } else { 1662 DPRINTF(sc, "Packet is neither ipv4 nor ipv6 from peer %" PRIu64 "\n", peer->p_id); 1663 goto out; 1664 } 1665 1666 /* We only want to compare the address, not dereference, so drop the ref. */ 1667 if (allowed_peer != NULL) 1668 noise_remote_put(allowed_peer->p_remote); 1669 1670 if (__predict_false(peer != allowed_peer)) { 1671 DPRINTF(sc, "Packet has unallowed src IP from peer %" PRIu64 "\n", peer->p_id); 1672 goto out; 1673 } 1674 1675 wg_mbuf_reset(m); 1676 state = WG_PACKET_CRYPTED; 1677 out: 1678 pkt->p_mbuf = m; 1679 atomic_store_rel_int(&pkt->p_state, state); 1680 GROUPTASK_ENQUEUE(&peer->p_recv); 1681 noise_remote_put(remote); 1682 } 1683 1684 static void 1685 wg_softc_decrypt(struct wg_softc *sc) 1686 { 1687 struct wg_packet *pkt; 1688 1689 while ((pkt = wg_queue_dequeue_parallel(&sc->sc_decrypt_parallel)) != NULL) 1690 wg_decrypt(sc, pkt); 1691 } 1692 1693 static void 1694 wg_softc_encrypt(struct wg_softc *sc) 1695 { 1696 struct wg_packet *pkt; 1697 1698 while ((pkt = wg_queue_dequeue_parallel(&sc->sc_encrypt_parallel)) != NULL) 1699 wg_encrypt(sc, pkt); 1700 } 1701 1702 static void 1703 wg_encrypt_dispatch(struct wg_softc *sc) 1704 { 1705 /* 1706 * The update to encrypt_last_cpu is racey such that we may 1707 * reschedule the task for the same CPU multiple times, but 1708 * the race doesn't really matter. 1709 */ 1710 u_int cpu = (sc->sc_encrypt_last_cpu + 1) % mp_ncpus; 1711 sc->sc_encrypt_last_cpu = cpu; 1712 GROUPTASK_ENQUEUE(&sc->sc_encrypt[cpu]); 1713 } 1714 1715 static void 1716 wg_decrypt_dispatch(struct wg_softc *sc) 1717 { 1718 u_int cpu = (sc->sc_decrypt_last_cpu + 1) % mp_ncpus; 1719 sc->sc_decrypt_last_cpu = cpu; 1720 GROUPTASK_ENQUEUE(&sc->sc_decrypt[cpu]); 1721 } 1722 1723 static void 1724 wg_deliver_out(struct wg_peer *peer) 1725 { 1726 struct wg_endpoint endpoint; 1727 struct wg_softc *sc = peer->p_sc; 1728 struct wg_packet *pkt; 1729 struct mbuf *m; 1730 int rc, len; 1731 1732 wg_peer_get_endpoint(peer, &endpoint); 1733 1734 while ((pkt = wg_queue_dequeue_serial(&peer->p_encrypt_serial)) != NULL) { 1735 if (atomic_load_acq_int(&pkt->p_state) != WG_PACKET_CRYPTED) 1736 goto error; 1737 1738 m = pkt->p_mbuf; 1739 pkt->p_mbuf = NULL; 1740 1741 len = m->m_pkthdr.len; 1742 1743 wg_timers_event_any_authenticated_packet_traversal(peer); 1744 wg_timers_event_any_authenticated_packet_sent(peer); 1745 rc = wg_send(sc, &endpoint, m); 1746 if (rc == 0) { 1747 if (len > (sizeof(struct wg_pkt_data) + NOISE_AUTHTAG_LEN)) 1748 wg_timers_event_data_sent(peer); 1749 counter_u64_add(peer->p_tx_bytes, len); 1750 } else if (rc == EADDRNOTAVAIL) { 1751 wg_peer_clear_src(peer); 1752 wg_peer_get_endpoint(peer, &endpoint); 1753 goto error; 1754 } else { 1755 goto error; 1756 } 1757 wg_packet_free(pkt); 1758 if (noise_keep_key_fresh_send(peer->p_remote)) 1759 wg_timers_event_want_initiation(peer); 1760 continue; 1761 error: 1762 if_inc_counter(sc->sc_ifp, IFCOUNTER_OERRORS, 1); 1763 wg_packet_free(pkt); 1764 } 1765 } 1766 1767 #ifdef DEV_NETMAP 1768 /* 1769 * Hand a packet to the netmap RX ring, via netmap's 1770 * freebsd_generic_rx_handler(). 1771 */ 1772 static void 1773 wg_deliver_netmap(if_t ifp, struct mbuf *m, int af) 1774 { 1775 struct ether_header *eh; 1776 1777 M_PREPEND(m, ETHER_HDR_LEN, M_NOWAIT); 1778 if (__predict_false(m == NULL)) { 1779 if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); 1780 return; 1781 } 1782 1783 eh = mtod(m, struct ether_header *); 1784 eh->ether_type = af == AF_INET ? 1785 htons(ETHERTYPE_IP) : htons(ETHERTYPE_IPV6); 1786 memcpy(eh->ether_shost, "\x02\x02\x02\x02\x02\x02", ETHER_ADDR_LEN); 1787 memcpy(eh->ether_dhost, "\xff\xff\xff\xff\xff\xff", ETHER_ADDR_LEN); 1788 if_input(ifp, m); 1789 } 1790 #endif 1791 1792 static void 1793 wg_deliver_in(struct wg_peer *peer) 1794 { 1795 struct wg_softc *sc = peer->p_sc; 1796 if_t ifp = sc->sc_ifp; 1797 struct wg_packet *pkt; 1798 struct mbuf *m; 1799 struct epoch_tracker et; 1800 int af; 1801 1802 while ((pkt = wg_queue_dequeue_serial(&peer->p_decrypt_serial)) != NULL) { 1803 if (atomic_load_acq_int(&pkt->p_state) != WG_PACKET_CRYPTED) 1804 goto error; 1805 1806 m = pkt->p_mbuf; 1807 if (noise_keypair_nonce_check(pkt->p_keypair, pkt->p_nonce) != 0) 1808 goto error; 1809 1810 if (noise_keypair_received_with(pkt->p_keypair) == ECONNRESET) 1811 wg_timers_event_handshake_complete(peer); 1812 1813 wg_timers_event_any_authenticated_packet_received(peer); 1814 wg_timers_event_any_authenticated_packet_traversal(peer); 1815 wg_peer_set_endpoint(peer, &pkt->p_endpoint); 1816 1817 counter_u64_add(peer->p_rx_bytes, m->m_pkthdr.len + 1818 sizeof(struct wg_pkt_data) + NOISE_AUTHTAG_LEN); 1819 if_inc_counter(sc->sc_ifp, IFCOUNTER_IPACKETS, 1); 1820 if_inc_counter(sc->sc_ifp, IFCOUNTER_IBYTES, m->m_pkthdr.len + 1821 sizeof(struct wg_pkt_data) + NOISE_AUTHTAG_LEN); 1822 1823 if (m->m_pkthdr.len == 0) 1824 goto done; 1825 1826 af = pkt->p_af; 1827 MPASS(af == AF_INET || af == AF_INET6); 1828 pkt->p_mbuf = NULL; 1829 1830 m->m_pkthdr.rcvif = ifp; 1831 1832 NET_EPOCH_ENTER(et); 1833 BPF_MTAP2_AF(ifp, m, af); 1834 1835 CURVNET_SET(if_getvnet(ifp)); 1836 M_SETFIB(m, if_getfib(ifp)); 1837 if (V_encap_hash_inbound) 1838 encap_set_flowid(m, af); 1839 #ifdef DEV_NETMAP 1840 if ((if_getcapenable(ifp) & IFCAP_NETMAP) != 0) 1841 wg_deliver_netmap(ifp, m, af); 1842 else 1843 #endif 1844 if (af == AF_INET) 1845 netisr_dispatch(NETISR_IP, m); 1846 else if (af == AF_INET6) 1847 netisr_dispatch(NETISR_IPV6, m); 1848 CURVNET_RESTORE(); 1849 NET_EPOCH_EXIT(et); 1850 1851 wg_timers_event_data_received(peer); 1852 1853 done: 1854 if (noise_keep_key_fresh_recv(peer->p_remote)) 1855 wg_timers_event_want_initiation(peer); 1856 wg_packet_free(pkt); 1857 continue; 1858 error: 1859 if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); 1860 wg_packet_free(pkt); 1861 } 1862 } 1863 1864 static struct wg_packet * 1865 wg_packet_alloc(struct mbuf *m) 1866 { 1867 struct wg_packet *pkt; 1868 1869 if ((pkt = uma_zalloc(wg_packet_zone, M_NOWAIT | M_ZERO)) == NULL) 1870 return (NULL); 1871 pkt->p_mbuf = m; 1872 return (pkt); 1873 } 1874 1875 static void 1876 wg_packet_free(struct wg_packet *pkt) 1877 { 1878 if (pkt->p_keypair != NULL) 1879 noise_keypair_put(pkt->p_keypair); 1880 if (pkt->p_mbuf != NULL) 1881 m_freem(pkt->p_mbuf); 1882 uma_zfree(wg_packet_zone, pkt); 1883 } 1884 1885 static void 1886 wg_queue_init(struct wg_queue *queue, const char *name) 1887 { 1888 mtx_init(&queue->q_mtx, name, NULL, MTX_DEF); 1889 STAILQ_INIT(&queue->q_queue); 1890 queue->q_len = 0; 1891 } 1892 1893 static void 1894 wg_queue_deinit(struct wg_queue *queue) 1895 { 1896 wg_queue_purge(queue); 1897 mtx_destroy(&queue->q_mtx); 1898 } 1899 1900 static size_t 1901 wg_queue_len(struct wg_queue *queue) 1902 { 1903 return (queue->q_len); 1904 } 1905 1906 static int 1907 wg_queue_enqueue_handshake(struct wg_queue *hs, struct wg_packet *pkt) 1908 { 1909 int ret = 0; 1910 mtx_lock(&hs->q_mtx); 1911 if (hs->q_len < MAX_QUEUED_HANDSHAKES) { 1912 STAILQ_INSERT_TAIL(&hs->q_queue, pkt, p_parallel); 1913 hs->q_len++; 1914 } else { 1915 ret = ENOBUFS; 1916 } 1917 mtx_unlock(&hs->q_mtx); 1918 if (ret != 0) 1919 wg_packet_free(pkt); 1920 return (ret); 1921 } 1922 1923 static struct wg_packet * 1924 wg_queue_dequeue_handshake(struct wg_queue *hs) 1925 { 1926 struct wg_packet *pkt; 1927 mtx_lock(&hs->q_mtx); 1928 if ((pkt = STAILQ_FIRST(&hs->q_queue)) != NULL) { 1929 STAILQ_REMOVE_HEAD(&hs->q_queue, p_parallel); 1930 hs->q_len--; 1931 } 1932 mtx_unlock(&hs->q_mtx); 1933 return (pkt); 1934 } 1935 1936 static void 1937 wg_queue_push_staged(struct wg_queue *staged, struct wg_packet *pkt) 1938 { 1939 struct wg_packet *old = NULL; 1940 1941 mtx_lock(&staged->q_mtx); 1942 if (staged->q_len >= MAX_STAGED_PKT) { 1943 old = STAILQ_FIRST(&staged->q_queue); 1944 STAILQ_REMOVE_HEAD(&staged->q_queue, p_parallel); 1945 staged->q_len--; 1946 } 1947 STAILQ_INSERT_TAIL(&staged->q_queue, pkt, p_parallel); 1948 staged->q_len++; 1949 mtx_unlock(&staged->q_mtx); 1950 1951 if (old != NULL) 1952 wg_packet_free(old); 1953 } 1954 1955 static void 1956 wg_queue_enlist_staged(struct wg_queue *staged, struct wg_packet_list *list) 1957 { 1958 struct wg_packet *pkt, *tpkt; 1959 STAILQ_FOREACH_SAFE(pkt, list, p_parallel, tpkt) 1960 wg_queue_push_staged(staged, pkt); 1961 } 1962 1963 static void 1964 wg_queue_delist_staged(struct wg_queue *staged, struct wg_packet_list *list) 1965 { 1966 STAILQ_INIT(list); 1967 mtx_lock(&staged->q_mtx); 1968 STAILQ_CONCAT(list, &staged->q_queue); 1969 staged->q_len = 0; 1970 mtx_unlock(&staged->q_mtx); 1971 } 1972 1973 static void 1974 wg_queue_purge(struct wg_queue *staged) 1975 { 1976 struct wg_packet_list list; 1977 struct wg_packet *pkt, *tpkt; 1978 wg_queue_delist_staged(staged, &list); 1979 STAILQ_FOREACH_SAFE(pkt, &list, p_parallel, tpkt) 1980 wg_packet_free(pkt); 1981 } 1982 1983 static int 1984 wg_queue_both(struct wg_queue *parallel, struct wg_queue *serial, struct wg_packet *pkt) 1985 { 1986 pkt->p_state = WG_PACKET_UNCRYPTED; 1987 1988 mtx_lock(&serial->q_mtx); 1989 if (serial->q_len < MAX_QUEUED_PKT) { 1990 serial->q_len++; 1991 STAILQ_INSERT_TAIL(&serial->q_queue, pkt, p_serial); 1992 } else { 1993 mtx_unlock(&serial->q_mtx); 1994 wg_packet_free(pkt); 1995 return (ENOBUFS); 1996 } 1997 mtx_unlock(&serial->q_mtx); 1998 1999 mtx_lock(¶llel->q_mtx); 2000 if (parallel->q_len < MAX_QUEUED_PKT) { 2001 parallel->q_len++; 2002 STAILQ_INSERT_TAIL(¶llel->q_queue, pkt, p_parallel); 2003 } else { 2004 mtx_unlock(¶llel->q_mtx); 2005 pkt->p_state = WG_PACKET_DEAD; 2006 return (ENOBUFS); 2007 } 2008 mtx_unlock(¶llel->q_mtx); 2009 2010 return (0); 2011 } 2012 2013 static struct wg_packet * 2014 wg_queue_dequeue_serial(struct wg_queue *serial) 2015 { 2016 struct wg_packet *pkt = NULL; 2017 mtx_lock(&serial->q_mtx); 2018 if (serial->q_len > 0 && STAILQ_FIRST(&serial->q_queue)->p_state != WG_PACKET_UNCRYPTED) { 2019 serial->q_len--; 2020 pkt = STAILQ_FIRST(&serial->q_queue); 2021 STAILQ_REMOVE_HEAD(&serial->q_queue, p_serial); 2022 } 2023 mtx_unlock(&serial->q_mtx); 2024 return (pkt); 2025 } 2026 2027 static struct wg_packet * 2028 wg_queue_dequeue_parallel(struct wg_queue *parallel) 2029 { 2030 struct wg_packet *pkt = NULL; 2031 mtx_lock(¶llel->q_mtx); 2032 if (parallel->q_len > 0) { 2033 parallel->q_len--; 2034 pkt = STAILQ_FIRST(¶llel->q_queue); 2035 STAILQ_REMOVE_HEAD(¶llel->q_queue, p_parallel); 2036 } 2037 mtx_unlock(¶llel->q_mtx); 2038 return (pkt); 2039 } 2040 2041 static bool 2042 wg_input(struct mbuf *m, int offset, struct inpcb *inpcb, 2043 const struct sockaddr *sa, void *_sc) 2044 { 2045 #ifdef INET 2046 const struct sockaddr_in *sin; 2047 #endif 2048 #ifdef INET6 2049 const struct sockaddr_in6 *sin6; 2050 #endif 2051 struct noise_remote *remote; 2052 struct wg_pkt_data *data; 2053 struct wg_packet *pkt; 2054 struct wg_peer *peer; 2055 struct wg_softc *sc = _sc; 2056 struct mbuf *defragged; 2057 2058 defragged = m_defrag(m, M_NOWAIT); 2059 if (defragged) 2060 m = defragged; 2061 m = m_unshare(m, M_NOWAIT); 2062 if (!m) { 2063 if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1); 2064 return true; 2065 } 2066 2067 /* Caller provided us with `sa`, no need for this header. */ 2068 m_adj(m, offset + sizeof(struct udphdr)); 2069 2070 /* Pullup enough to read packet type */ 2071 if ((m = m_pullup(m, sizeof(uint32_t))) == NULL) { 2072 if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1); 2073 return true; 2074 } 2075 2076 if ((pkt = wg_packet_alloc(m)) == NULL) { 2077 if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1); 2078 m_freem(m); 2079 return true; 2080 } 2081 2082 /* Save send/recv address and port for later. */ 2083 switch (sa->sa_family) { 2084 #ifdef INET 2085 case AF_INET: 2086 sin = (const struct sockaddr_in *)sa; 2087 pkt->p_endpoint.e_remote.r_sin = sin[0]; 2088 pkt->p_endpoint.e_local.l_in = sin[1].sin_addr; 2089 break; 2090 #endif 2091 #ifdef INET6 2092 case AF_INET6: 2093 sin6 = (const struct sockaddr_in6 *)sa; 2094 pkt->p_endpoint.e_remote.r_sin6 = sin6[0]; 2095 pkt->p_endpoint.e_local.l_in6 = sin6[1].sin6_addr; 2096 break; 2097 #endif 2098 default: 2099 goto error; 2100 } 2101 2102 if ((m->m_pkthdr.len == sizeof(struct wg_pkt_initiation) && 2103 *mtod(m, uint32_t *) == WG_PKT_INITIATION) || 2104 (m->m_pkthdr.len == sizeof(struct wg_pkt_response) && 2105 *mtod(m, uint32_t *) == WG_PKT_RESPONSE) || 2106 (m->m_pkthdr.len == sizeof(struct wg_pkt_cookie) && 2107 *mtod(m, uint32_t *) == WG_PKT_COOKIE)) { 2108 2109 if (wg_queue_enqueue_handshake(&sc->sc_handshake_queue, pkt) != 0) { 2110 if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1); 2111 DPRINTF(sc, "Dropping handshake packet\n"); 2112 } 2113 GROUPTASK_ENQUEUE(&sc->sc_handshake); 2114 } else if (m->m_pkthdr.len >= sizeof(struct wg_pkt_data) + 2115 NOISE_AUTHTAG_LEN && *mtod(m, uint32_t *) == WG_PKT_DATA) { 2116 2117 /* Pullup whole header to read r_idx below. */ 2118 if ((pkt->p_mbuf = m_pullup(m, sizeof(struct wg_pkt_data))) == NULL) 2119 goto error; 2120 2121 data = mtod(pkt->p_mbuf, struct wg_pkt_data *); 2122 if ((pkt->p_keypair = noise_keypair_lookup(sc->sc_local, data->r_idx)) == NULL) 2123 goto error; 2124 2125 remote = noise_keypair_remote(pkt->p_keypair); 2126 peer = noise_remote_arg(remote); 2127 if (wg_queue_both(&sc->sc_decrypt_parallel, &peer->p_decrypt_serial, pkt) != 0) 2128 if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1); 2129 wg_decrypt_dispatch(sc); 2130 noise_remote_put(remote); 2131 } else { 2132 goto error; 2133 } 2134 return true; 2135 error: 2136 if_inc_counter(sc->sc_ifp, IFCOUNTER_IERRORS, 1); 2137 wg_packet_free(pkt); 2138 return true; 2139 } 2140 2141 static void 2142 wg_peer_send_staged(struct wg_peer *peer) 2143 { 2144 struct wg_packet_list list; 2145 struct noise_keypair *keypair; 2146 struct wg_packet *pkt, *tpkt; 2147 struct wg_softc *sc = peer->p_sc; 2148 2149 wg_queue_delist_staged(&peer->p_stage_queue, &list); 2150 2151 if (STAILQ_EMPTY(&list)) 2152 return; 2153 2154 if ((keypair = noise_keypair_current(peer->p_remote)) == NULL) 2155 goto error; 2156 2157 STAILQ_FOREACH(pkt, &list, p_parallel) { 2158 if (noise_keypair_nonce_next(keypair, &pkt->p_nonce) != 0) 2159 goto error_keypair; 2160 } 2161 STAILQ_FOREACH_SAFE(pkt, &list, p_parallel, tpkt) { 2162 pkt->p_keypair = noise_keypair_ref(keypair); 2163 if (wg_queue_both(&sc->sc_encrypt_parallel, &peer->p_encrypt_serial, pkt) != 0) 2164 if_inc_counter(sc->sc_ifp, IFCOUNTER_OQDROPS, 1); 2165 } 2166 wg_encrypt_dispatch(sc); 2167 noise_keypair_put(keypair); 2168 return; 2169 2170 error_keypair: 2171 noise_keypair_put(keypair); 2172 error: 2173 wg_queue_enlist_staged(&peer->p_stage_queue, &list); 2174 wg_timers_event_want_initiation(peer); 2175 } 2176 2177 static inline void 2178 xmit_err(if_t ifp, struct mbuf *m, struct wg_packet *pkt, sa_family_t af) 2179 { 2180 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 2181 switch (af) { 2182 #ifdef INET 2183 case AF_INET: 2184 icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_HOST, 0, 0); 2185 if (pkt) 2186 pkt->p_mbuf = NULL; 2187 m = NULL; 2188 break; 2189 #endif 2190 #ifdef INET6 2191 case AF_INET6: 2192 icmp6_error(m, ICMP6_DST_UNREACH, 0, 0); 2193 if (pkt) 2194 pkt->p_mbuf = NULL; 2195 m = NULL; 2196 break; 2197 #endif 2198 } 2199 if (pkt) 2200 wg_packet_free(pkt); 2201 else if (m) 2202 m_freem(m); 2203 } 2204 2205 static int 2206 wg_xmit(if_t ifp, struct mbuf *m, sa_family_t af, uint32_t mtu) 2207 { 2208 struct wg_packet *pkt = NULL; 2209 struct wg_softc *sc = if_getsoftc(ifp); 2210 struct wg_peer *peer; 2211 int rc = 0; 2212 sa_family_t peer_af; 2213 2214 /* Work around lifetime issue in the ipv6 mld code. */ 2215 if (__predict_false((if_getflags(ifp) & IFF_DYING) || !sc)) { 2216 rc = ENXIO; 2217 goto err_xmit; 2218 } 2219 2220 if ((pkt = wg_packet_alloc(m)) == NULL) { 2221 rc = ENOBUFS; 2222 goto err_xmit; 2223 } 2224 pkt->p_mtu = mtu; 2225 pkt->p_af = af; 2226 2227 if (af == AF_INET) { 2228 peer = wg_aip_lookup(sc, AF_INET, &mtod(m, struct ip *)->ip_dst); 2229 } else if (af == AF_INET6) { 2230 peer = wg_aip_lookup(sc, AF_INET6, &mtod(m, struct ip6_hdr *)->ip6_dst); 2231 } else { 2232 rc = EAFNOSUPPORT; 2233 goto err_xmit; 2234 } 2235 2236 BPF_MTAP2_AF(ifp, m, pkt->p_af); 2237 2238 if (__predict_false(peer == NULL)) { 2239 rc = ENETUNREACH; 2240 goto err_xmit; 2241 } 2242 2243 if (__predict_false(if_tunnel_check_nesting(ifp, m, MTAG_WGLOOP, MAX_LOOPS))) { 2244 DPRINTF(sc, "Packet looped"); 2245 rc = ELOOP; 2246 goto err_peer; 2247 } 2248 2249 peer_af = peer->p_endpoint.e_remote.r_sa.sa_family; 2250 if (__predict_false(peer_af != AF_INET && peer_af != AF_INET6)) { 2251 DPRINTF(sc, "No valid endpoint has been configured or " 2252 "discovered for peer %" PRIu64 "\n", peer->p_id); 2253 rc = EHOSTUNREACH; 2254 goto err_peer; 2255 } 2256 2257 wg_queue_push_staged(&peer->p_stage_queue, pkt); 2258 wg_peer_send_staged(peer); 2259 noise_remote_put(peer->p_remote); 2260 return (0); 2261 2262 err_peer: 2263 noise_remote_put(peer->p_remote); 2264 err_xmit: 2265 xmit_err(ifp, m, pkt, af); 2266 return (rc); 2267 } 2268 2269 static inline int 2270 determine_af_and_pullup(struct mbuf **m, sa_family_t *af) 2271 { 2272 u_char ipv; 2273 if ((*m)->m_pkthdr.len >= sizeof(struct ip6_hdr)) 2274 *m = m_pullup(*m, sizeof(struct ip6_hdr)); 2275 else if ((*m)->m_pkthdr.len >= sizeof(struct ip)) 2276 *m = m_pullup(*m, sizeof(struct ip)); 2277 else 2278 return (EAFNOSUPPORT); 2279 if (*m == NULL) 2280 return (ENOBUFS); 2281 ipv = mtod(*m, struct ip *)->ip_v; 2282 if (ipv == 4) 2283 *af = AF_INET; 2284 else if (ipv == 6 && (*m)->m_pkthdr.len >= sizeof(struct ip6_hdr)) 2285 *af = AF_INET6; 2286 else 2287 return (EAFNOSUPPORT); 2288 return (0); 2289 } 2290 2291 static int 2292 determine_ethertype_and_pullup(struct mbuf **m, int *etp) 2293 { 2294 struct ether_header *eh; 2295 2296 *m = m_pullup(*m, sizeof(struct ether_header)); 2297 if (__predict_false(*m == NULL)) 2298 return (ENOBUFS); 2299 eh = mtod(*m, struct ether_header *); 2300 *etp = ntohs(eh->ether_type); 2301 if (*etp != ETHERTYPE_IP && *etp != ETHERTYPE_IPV6) 2302 return (EAFNOSUPPORT); 2303 return (0); 2304 } 2305 2306 /* 2307 * This should only be invoked by netmap, via nm_os_generic_xmit_frame(), to 2308 * transmit packets from the netmap TX ring. 2309 */ 2310 static int 2311 wg_transmit(if_t ifp, struct mbuf *m) 2312 { 2313 sa_family_t af; 2314 int et, ret; 2315 struct mbuf *defragged; 2316 2317 KASSERT((if_getcapenable(ifp) & IFCAP_NETMAP) != 0, 2318 ("%s: ifp %p is not in netmap mode", __func__, ifp)); 2319 2320 defragged = m_defrag(m, M_NOWAIT); 2321 if (defragged) 2322 m = defragged; 2323 m = m_unshare(m, M_NOWAIT); 2324 if (!m) { 2325 xmit_err(ifp, m, NULL, AF_UNSPEC); 2326 return (ENOBUFS); 2327 } 2328 2329 ret = determine_ethertype_and_pullup(&m, &et); 2330 if (ret) { 2331 xmit_err(ifp, m, NULL, AF_UNSPEC); 2332 return (ret); 2333 } 2334 m_adj(m, sizeof(struct ether_header)); 2335 2336 ret = determine_af_and_pullup(&m, &af); 2337 if (ret) { 2338 xmit_err(ifp, m, NULL, AF_UNSPEC); 2339 return (ret); 2340 } 2341 2342 /* 2343 * netmap only gets to see transient errors, since it handles errors by 2344 * refusing to advance the transmit ring and retrying later. 2345 */ 2346 ret = wg_xmit(ifp, m, af, if_getmtu(ifp)); 2347 if (ret == ENOBUFS) 2348 return (ret); 2349 return (0); 2350 } 2351 2352 #ifdef DEV_NETMAP 2353 /* 2354 * This should only be invoked by netmap, via nm_os_send_up(), to process 2355 * packets from the host TX ring. 2356 */ 2357 static void 2358 wg_if_input(if_t ifp, struct mbuf *m) 2359 { 2360 int et; 2361 2362 KASSERT((if_getcapenable(ifp) & IFCAP_NETMAP) != 0, 2363 ("%s: ifp %p is not in netmap mode", __func__, ifp)); 2364 2365 if (determine_ethertype_and_pullup(&m, &et) != 0) { 2366 if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); 2367 m_freem(m); 2368 return; 2369 } 2370 CURVNET_SET(if_getvnet(ifp)); 2371 switch (et) { 2372 case ETHERTYPE_IP: 2373 m_adj(m, sizeof(struct ether_header)); 2374 netisr_dispatch(NETISR_IP, m); 2375 break; 2376 case ETHERTYPE_IPV6: 2377 m_adj(m, sizeof(struct ether_header)); 2378 netisr_dispatch(NETISR_IPV6, m); 2379 break; 2380 default: 2381 __assert_unreachable(); 2382 } 2383 CURVNET_RESTORE(); 2384 } 2385 2386 /* 2387 * Deliver a packet to the host RX ring. Because the interface is in netmap 2388 * mode, the if_transmit() call should pass the packet to netmap_transmit(). 2389 */ 2390 static int 2391 wg_xmit_netmap(if_t ifp, struct mbuf *m, int af) 2392 { 2393 struct ether_header *eh; 2394 2395 if (__predict_false(if_tunnel_check_nesting(ifp, m, MTAG_WGLOOP, 2396 MAX_LOOPS))) { 2397 printf("%s:%d\n", __func__, __LINE__); 2398 if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); 2399 m_freem(m); 2400 return (ELOOP); 2401 } 2402 2403 M_PREPEND(m, ETHER_HDR_LEN, M_NOWAIT); 2404 if (__predict_false(m == NULL)) { 2405 if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); 2406 return (ENOBUFS); 2407 } 2408 2409 eh = mtod(m, struct ether_header *); 2410 eh->ether_type = af == AF_INET ? 2411 htons(ETHERTYPE_IP) : htons(ETHERTYPE_IPV6); 2412 memcpy(eh->ether_shost, "\x06\x06\x06\x06\x06\x06", ETHER_ADDR_LEN); 2413 memcpy(eh->ether_dhost, "\xff\xff\xff\xff\xff\xff", ETHER_ADDR_LEN); 2414 return (if_transmit(ifp, m)); 2415 } 2416 #endif /* DEV_NETMAP */ 2417 2418 static int 2419 wg_output(if_t ifp, struct mbuf *m, const struct sockaddr *dst, struct route *ro) 2420 { 2421 sa_family_t parsed_af; 2422 uint32_t af, mtu; 2423 int ret; 2424 struct mbuf *defragged; 2425 2426 /* BPF writes need to be handled specially. */ 2427 if (dst->sa_family == AF_UNSPEC || dst->sa_family == pseudo_AF_HDRCMPLT) 2428 memcpy(&af, dst->sa_data, sizeof(af)); 2429 else 2430 af = RO_GET_FAMILY(ro, dst); 2431 if (af == AF_UNSPEC) { 2432 xmit_err(ifp, m, NULL, af); 2433 return (EAFNOSUPPORT); 2434 } 2435 2436 #ifdef DEV_NETMAP 2437 if ((if_getcapenable(ifp) & IFCAP_NETMAP) != 0) 2438 return (wg_xmit_netmap(ifp, m, af)); 2439 #endif 2440 2441 defragged = m_defrag(m, M_NOWAIT); 2442 if (defragged) 2443 m = defragged; 2444 m = m_unshare(m, M_NOWAIT); 2445 if (!m) { 2446 xmit_err(ifp, m, NULL, AF_UNSPEC); 2447 return (ENOBUFS); 2448 } 2449 2450 ret = determine_af_and_pullup(&m, &parsed_af); 2451 if (ret) { 2452 xmit_err(ifp, m, NULL, AF_UNSPEC); 2453 return (ret); 2454 } 2455 2456 MPASS(parsed_af == af); 2457 mtu = (ro != NULL && ro->ro_mtu > 0) ? ro->ro_mtu : if_getmtu(ifp); 2458 return (wg_xmit(ifp, m, parsed_af, mtu)); 2459 } 2460 2461 static int 2462 wg_peer_add(struct wg_softc *sc, const nvlist_t *nvl) 2463 { 2464 uint8_t public[WG_KEY_SIZE]; 2465 const void *pub_key, *preshared_key = NULL; 2466 const struct sockaddr *endpoint; 2467 int err; 2468 size_t size; 2469 struct noise_remote *remote; 2470 struct wg_peer *peer = NULL; 2471 bool need_cleanup = false; 2472 2473 sx_assert(&sc->sc_lock, SX_XLOCKED); 2474 2475 if (!nvlist_exists_binary(nvl, "public-key")) { 2476 return (EINVAL); 2477 } 2478 pub_key = nvlist_get_binary(nvl, "public-key", &size); 2479 if (size != WG_KEY_SIZE) { 2480 return (EINVAL); 2481 } 2482 if (noise_local_keys(sc->sc_local, public, NULL) == 0 && 2483 bcmp(public, pub_key, WG_KEY_SIZE) == 0) { 2484 return (0); // Silently ignored; not actually a failure. 2485 } 2486 if ((remote = noise_remote_lookup(sc->sc_local, pub_key)) != NULL) 2487 peer = noise_remote_arg(remote); 2488 if (nvlist_exists_bool(nvl, "remove") && 2489 nvlist_get_bool(nvl, "remove")) { 2490 if (remote != NULL) { 2491 wg_peer_destroy(peer); 2492 noise_remote_put(remote); 2493 } 2494 return (0); 2495 } 2496 if (nvlist_exists_bool(nvl, "replace-allowedips") && 2497 nvlist_get_bool(nvl, "replace-allowedips") && 2498 peer != NULL) { 2499 2500 wg_aip_remove_all(sc, peer); 2501 } 2502 if (peer == NULL) { 2503 peer = wg_peer_create(sc, pub_key, &err); 2504 if (peer == NULL) 2505 goto out; 2506 need_cleanup = true; 2507 } 2508 if (nvlist_exists_binary(nvl, "endpoint")) { 2509 endpoint = nvlist_get_binary(nvl, "endpoint", &size); 2510 if (size > sizeof(peer->p_endpoint.e_remote)) { 2511 err = EINVAL; 2512 goto out; 2513 } 2514 memcpy(&peer->p_endpoint.e_remote, endpoint, size); 2515 } 2516 if (nvlist_exists_binary(nvl, "preshared-key")) { 2517 preshared_key = nvlist_get_binary(nvl, "preshared-key", &size); 2518 if (size != WG_KEY_SIZE) { 2519 err = EINVAL; 2520 goto out; 2521 } 2522 noise_remote_set_psk(peer->p_remote, preshared_key); 2523 } 2524 if (nvlist_exists_number(nvl, "persistent-keepalive-interval")) { 2525 uint64_t pki = nvlist_get_number(nvl, "persistent-keepalive-interval"); 2526 if (pki > UINT16_MAX) { 2527 err = EINVAL; 2528 goto out; 2529 } 2530 wg_timers_set_persistent_keepalive(peer, pki); 2531 } 2532 if (nvlist_exists_nvlist_array(nvl, "allowed-ips")) { 2533 const void *addr; 2534 uint64_t cidr; 2535 const nvlist_t * const * aipl; 2536 size_t allowedip_count; 2537 2538 aipl = nvlist_get_nvlist_array(nvl, "allowed-ips", &allowedip_count); 2539 for (size_t idx = 0; idx < allowedip_count; idx++) { 2540 sa_family_t ipaf; 2541 int ipflags; 2542 2543 if (!nvlist_exists_number(aipl[idx], "cidr")) 2544 continue; 2545 2546 ipaf = AF_UNSPEC; 2547 ipflags = 0; 2548 if (nvlist_exists_number(aipl[idx], "flags")) 2549 ipflags = nvlist_get_number(aipl[idx], "flags"); 2550 if ((ipflags & ~WGALLOWEDIP_VALID_FLAGS) != 0) { 2551 err = EOPNOTSUPP; 2552 goto out; 2553 } 2554 cidr = nvlist_get_number(aipl[idx], "cidr"); 2555 if (nvlist_exists_binary(aipl[idx], "ipv4")) { 2556 addr = nvlist_get_binary(aipl[idx], "ipv4", &size); 2557 if (addr == NULL || cidr > 32 || size != sizeof(struct in_addr)) { 2558 err = EINVAL; 2559 goto out; 2560 } 2561 2562 ipaf = AF_INET; 2563 } else if (nvlist_exists_binary(aipl[idx], "ipv6")) { 2564 addr = nvlist_get_binary(aipl[idx], "ipv6", &size); 2565 if (addr == NULL || cidr > 128 || size != sizeof(struct in6_addr)) { 2566 err = EINVAL; 2567 goto out; 2568 } 2569 2570 ipaf = AF_INET6; 2571 } else { 2572 continue; 2573 } 2574 2575 MPASS(ipaf != AF_UNSPEC); 2576 if ((ipflags & WGALLOWEDIP_REMOVE_ME) != 0) { 2577 err = wg_aip_del(sc, peer, ipaf, addr, cidr); 2578 } else { 2579 err = wg_aip_add(sc, peer, ipaf, addr, cidr); 2580 } 2581 2582 if (err != 0) 2583 goto out; 2584 } 2585 } 2586 if (remote != NULL) 2587 noise_remote_put(remote); 2588 return (0); 2589 out: 2590 if (need_cleanup) /* If we fail, only destroy if it was new. */ 2591 wg_peer_destroy(peer); 2592 if (remote != NULL) 2593 noise_remote_put(remote); 2594 return (err); 2595 } 2596 2597 static int 2598 wgc_set(struct wg_softc *sc, struct wg_data_io *wgd) 2599 { 2600 uint8_t public[WG_KEY_SIZE], private[WG_KEY_SIZE]; 2601 if_t ifp; 2602 void *nvlpacked; 2603 nvlist_t *nvl; 2604 ssize_t size; 2605 int err; 2606 2607 ifp = sc->sc_ifp; 2608 if (wgd->wgd_size == 0 || wgd->wgd_data == NULL) 2609 return (EFAULT); 2610 2611 /* Can nvlists be streamed in? It's not nice to impose arbitrary limits like that but 2612 * there needs to be _some_ limitation. */ 2613 if (wgd->wgd_size >= UINT32_MAX / 2) 2614 return (E2BIG); 2615 2616 nvlpacked = malloc(wgd->wgd_size, M_TEMP, M_WAITOK | M_ZERO); 2617 2618 err = copyin(wgd->wgd_data, nvlpacked, wgd->wgd_size); 2619 if (err) 2620 goto out; 2621 nvl = nvlist_unpack(nvlpacked, wgd->wgd_size, 0); 2622 if (nvl == NULL) { 2623 err = EBADMSG; 2624 goto out; 2625 } 2626 sx_xlock(&sc->sc_lock); 2627 if (nvlist_exists_bool(nvl, "replace-peers") && 2628 nvlist_get_bool(nvl, "replace-peers")) 2629 wg_peer_destroy_all(sc); 2630 if (nvlist_exists_number(nvl, "listen-port")) { 2631 uint64_t new_port = nvlist_get_number(nvl, "listen-port"); 2632 if (new_port > UINT16_MAX) { 2633 err = EINVAL; 2634 goto out_locked; 2635 } 2636 if (new_port != sc->sc_socket.so_port) { 2637 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) { 2638 if ((err = wg_socket_init(sc, new_port)) != 0) 2639 goto out_locked; 2640 } else 2641 sc->sc_socket.so_port = new_port; 2642 } 2643 } 2644 if (nvlist_exists_binary(nvl, "private-key")) { 2645 const void *key = nvlist_get_binary(nvl, "private-key", &size); 2646 if (size != WG_KEY_SIZE) { 2647 err = EINVAL; 2648 goto out_locked; 2649 } 2650 2651 if (noise_local_keys(sc->sc_local, NULL, private) != 0 || 2652 timingsafe_bcmp(private, key, WG_KEY_SIZE) != 0) { 2653 struct wg_peer *peer; 2654 2655 if (curve25519_generate_public(public, key)) { 2656 /* Peer conflict: remove conflicting peer. */ 2657 struct noise_remote *remote; 2658 if ((remote = noise_remote_lookup(sc->sc_local, 2659 public)) != NULL) { 2660 peer = noise_remote_arg(remote); 2661 wg_peer_destroy(peer); 2662 noise_remote_put(remote); 2663 } 2664 } 2665 2666 /* 2667 * Set the private key and invalidate all existing 2668 * handshakes. 2669 */ 2670 /* Note: we might be removing the private key. */ 2671 noise_local_private(sc->sc_local, key); 2672 if (noise_local_keys(sc->sc_local, NULL, NULL) == 0) 2673 cookie_checker_update(&sc->sc_cookie, public); 2674 else 2675 cookie_checker_update(&sc->sc_cookie, NULL); 2676 } 2677 } 2678 if (nvlist_exists_number(nvl, "user-cookie")) { 2679 uint64_t user_cookie = nvlist_get_number(nvl, "user-cookie"); 2680 if (user_cookie > UINT32_MAX) { 2681 err = EINVAL; 2682 goto out_locked; 2683 } 2684 err = wg_socket_set_cookie(sc, user_cookie); 2685 if (err) 2686 goto out_locked; 2687 } 2688 if (nvlist_exists_nvlist_array(nvl, "peers")) { 2689 size_t peercount; 2690 const nvlist_t * const*nvl_peers; 2691 2692 nvl_peers = nvlist_get_nvlist_array(nvl, "peers", &peercount); 2693 for (int i = 0; i < peercount; i++) { 2694 err = wg_peer_add(sc, nvl_peers[i]); 2695 if (err != 0) 2696 goto out_locked; 2697 } 2698 } 2699 2700 out_locked: 2701 sx_xunlock(&sc->sc_lock); 2702 nvlist_destroy(nvl); 2703 out: 2704 zfree(nvlpacked, M_TEMP); 2705 return (err); 2706 } 2707 2708 static int 2709 wgc_get(struct wg_softc *sc, struct wg_data_io *wgd) 2710 { 2711 uint8_t public_key[WG_KEY_SIZE] = { 0 }; 2712 uint8_t private_key[WG_KEY_SIZE] = { 0 }; 2713 uint8_t preshared_key[NOISE_SYMMETRIC_KEY_LEN] = { 0 }; 2714 nvlist_t *nvl, *nvl_peer, *nvl_aip, **nvl_peers, **nvl_aips; 2715 size_t size, peer_count, aip_count, i, j; 2716 struct wg_timespec64 ts64; 2717 struct wg_peer *peer; 2718 struct wg_aip *aip; 2719 void *packed; 2720 int err = 0; 2721 2722 nvl = nvlist_create(0); 2723 if (!nvl) 2724 return (ENOMEM); 2725 2726 sx_slock(&sc->sc_lock); 2727 2728 if (sc->sc_socket.so_port != 0) 2729 nvlist_add_number(nvl, "listen-port", sc->sc_socket.so_port); 2730 if (sc->sc_socket.so_user_cookie != 0) 2731 nvlist_add_number(nvl, "user-cookie", sc->sc_socket.so_user_cookie); 2732 if (noise_local_keys(sc->sc_local, public_key, private_key) == 0) { 2733 nvlist_add_binary(nvl, "public-key", public_key, WG_KEY_SIZE); 2734 if (wgc_privileged(sc)) 2735 nvlist_add_binary(nvl, "private-key", private_key, WG_KEY_SIZE); 2736 explicit_bzero(private_key, sizeof(private_key)); 2737 } 2738 peer_count = sc->sc_peers_num; 2739 if (peer_count) { 2740 nvl_peers = mallocarray(peer_count, sizeof(void *), M_NVLIST, M_WAITOK | M_ZERO); 2741 i = 0; 2742 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) { 2743 if (i >= peer_count) 2744 panic("peers changed from under us"); 2745 2746 nvl_peers[i++] = nvl_peer = nvlist_create(0); 2747 if (!nvl_peer) { 2748 err = ENOMEM; 2749 goto err_peer; 2750 } 2751 2752 (void)noise_remote_keys(peer->p_remote, public_key, preshared_key); 2753 nvlist_add_binary(nvl_peer, "public-key", public_key, sizeof(public_key)); 2754 if (wgc_privileged(sc)) 2755 nvlist_add_binary(nvl_peer, "preshared-key", preshared_key, sizeof(preshared_key)); 2756 explicit_bzero(preshared_key, sizeof(preshared_key)); 2757 if (peer->p_endpoint.e_remote.r_sa.sa_family == AF_INET) 2758 nvlist_add_binary(nvl_peer, "endpoint", &peer->p_endpoint.e_remote, sizeof(struct sockaddr_in)); 2759 else if (peer->p_endpoint.e_remote.r_sa.sa_family == AF_INET6) 2760 nvlist_add_binary(nvl_peer, "endpoint", &peer->p_endpoint.e_remote, sizeof(struct sockaddr_in6)); 2761 wg_timers_get_last_handshake(peer, &ts64); 2762 nvlist_add_binary(nvl_peer, "last-handshake-time", &ts64, sizeof(ts64)); 2763 nvlist_add_number(nvl_peer, "persistent-keepalive-interval", peer->p_persistent_keepalive_interval); 2764 nvlist_add_number(nvl_peer, "rx-bytes", counter_u64_fetch(peer->p_rx_bytes)); 2765 nvlist_add_number(nvl_peer, "tx-bytes", counter_u64_fetch(peer->p_tx_bytes)); 2766 2767 aip_count = peer->p_aips_num; 2768 if (aip_count) { 2769 nvl_aips = mallocarray(aip_count, sizeof(void *), M_NVLIST, M_WAITOK | M_ZERO); 2770 j = 0; 2771 LIST_FOREACH(aip, &peer->p_aips, a_entry) { 2772 if (j >= aip_count) 2773 panic("aips changed from under us"); 2774 2775 nvl_aips[j++] = nvl_aip = nvlist_create(0); 2776 if (!nvl_aip) { 2777 err = ENOMEM; 2778 goto err_aip; 2779 } 2780 if (aip->a_af == AF_INET) { 2781 nvlist_add_binary(nvl_aip, "ipv4", &aip->a_addr.in, sizeof(aip->a_addr.in)); 2782 nvlist_add_number(nvl_aip, "cidr", bitcount32(aip->a_mask.ip)); 2783 } 2784 #ifdef INET6 2785 else if (aip->a_af == AF_INET6) { 2786 nvlist_add_binary(nvl_aip, "ipv6", &aip->a_addr.in6, sizeof(aip->a_addr.in6)); 2787 nvlist_add_number(nvl_aip, "cidr", in6_mask2len(&aip->a_mask.in6, NULL)); 2788 } 2789 #endif 2790 } 2791 nvlist_add_nvlist_array(nvl_peer, "allowed-ips", (const nvlist_t *const *)nvl_aips, aip_count); 2792 err_aip: 2793 for (j = 0; j < aip_count; ++j) 2794 nvlist_destroy(nvl_aips[j]); 2795 free(nvl_aips, M_NVLIST); 2796 if (err) 2797 goto err_peer; 2798 } 2799 } 2800 nvlist_add_nvlist_array(nvl, "peers", (const nvlist_t * const *)nvl_peers, peer_count); 2801 err_peer: 2802 for (i = 0; i < peer_count; ++i) 2803 nvlist_destroy(nvl_peers[i]); 2804 free(nvl_peers, M_NVLIST); 2805 if (err) { 2806 sx_sunlock(&sc->sc_lock); 2807 goto err; 2808 } 2809 } 2810 sx_sunlock(&sc->sc_lock); 2811 packed = nvlist_pack(nvl, &size); 2812 if (!packed) { 2813 err = ENOMEM; 2814 goto err; 2815 } 2816 if (!wgd->wgd_size) { 2817 wgd->wgd_size = size; 2818 goto out; 2819 } 2820 if (wgd->wgd_size < size) { 2821 err = ENOSPC; 2822 goto out; 2823 } 2824 err = copyout(packed, wgd->wgd_data, size); 2825 wgd->wgd_size = size; 2826 2827 out: 2828 zfree(packed, M_NVLIST); 2829 err: 2830 nvlist_destroy(nvl); 2831 return (err); 2832 } 2833 2834 static int 2835 wg_ioctl(if_t ifp, u_long cmd, caddr_t data) 2836 { 2837 struct wg_data_io *wgd = (struct wg_data_io *)data; 2838 struct ifreq *ifr = (struct ifreq *)data; 2839 struct wg_softc *sc; 2840 int ret = 0; 2841 2842 sx_slock(&wg_sx); 2843 sc = if_getsoftc(ifp); 2844 if (!sc) { 2845 ret = ENXIO; 2846 goto out; 2847 } 2848 2849 switch (cmd) { 2850 case SIOCSWG: 2851 ret = priv_check(curthread, PRIV_NET_WG); 2852 if (ret == 0) 2853 ret = wgc_set(sc, wgd); 2854 break; 2855 case SIOCGWG: 2856 ret = wgc_get(sc, wgd); 2857 break; 2858 /* Interface IOCTLs */ 2859 case SIOCSIFADDR: 2860 /* 2861 * This differs from *BSD norms, but is more uniform with how 2862 * WireGuard behaves elsewhere. 2863 */ 2864 break; 2865 case SIOCSIFFLAGS: 2866 if (if_getflags(ifp) & IFF_UP) 2867 ret = wg_up(sc); 2868 else 2869 wg_down(sc); 2870 break; 2871 case SIOCSIFMTU: 2872 if (ifr->ifr_mtu <= 0 || ifr->ifr_mtu > MAX_MTU) 2873 ret = EINVAL; 2874 else 2875 if_setmtu(ifp, ifr->ifr_mtu); 2876 break; 2877 case SIOCADDMULTI: 2878 case SIOCDELMULTI: 2879 break; 2880 case SIOCGTUNFIB: 2881 ifr->ifr_fib = sc->sc_socket.so_fibnum; 2882 break; 2883 case SIOCSTUNFIB: 2884 ret = priv_check(curthread, PRIV_NET_WG); 2885 if (ret) 2886 break; 2887 ret = priv_check(curthread, PRIV_NET_SETIFFIB); 2888 if (ret) 2889 break; 2890 sx_xlock(&sc->sc_lock); 2891 ret = wg_socket_set_fibnum(sc, ifr->ifr_fib); 2892 sx_xunlock(&sc->sc_lock); 2893 break; 2894 default: 2895 ret = ENOTTY; 2896 } 2897 2898 out: 2899 sx_sunlock(&wg_sx); 2900 return (ret); 2901 } 2902 2903 static int 2904 wg_up(struct wg_softc *sc) 2905 { 2906 if_t ifp = sc->sc_ifp; 2907 struct wg_peer *peer; 2908 int rc = EBUSY; 2909 2910 sx_xlock(&sc->sc_lock); 2911 /* Jail's being removed, no more wg_up(). */ 2912 if ((sc->sc_flags & WGF_DYING) != 0) 2913 goto out; 2914 2915 /* Silent success if we're already running. */ 2916 rc = 0; 2917 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 2918 goto out; 2919 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0); 2920 2921 rc = wg_socket_init(sc, sc->sc_socket.so_port); 2922 if (rc == 0) { 2923 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) 2924 wg_timers_enable(peer); 2925 if_link_state_change(sc->sc_ifp, LINK_STATE_UP); 2926 } else { 2927 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); 2928 DPRINTF(sc, "Unable to initialize sockets: %d\n", rc); 2929 } 2930 out: 2931 sx_xunlock(&sc->sc_lock); 2932 return (rc); 2933 } 2934 2935 static void 2936 wg_down(struct wg_softc *sc) 2937 { 2938 if_t ifp = sc->sc_ifp; 2939 struct wg_peer *peer; 2940 2941 sx_xlock(&sc->sc_lock); 2942 if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) { 2943 sx_xunlock(&sc->sc_lock); 2944 return; 2945 } 2946 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); 2947 2948 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) { 2949 wg_queue_purge(&peer->p_stage_queue); 2950 wg_timers_disable(peer); 2951 } 2952 2953 wg_queue_purge(&sc->sc_handshake_queue); 2954 2955 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) { 2956 noise_remote_handshake_clear(peer->p_remote); 2957 noise_remote_keypairs_clear(peer->p_remote); 2958 } 2959 2960 if_link_state_change(sc->sc_ifp, LINK_STATE_DOWN); 2961 wg_socket_uninit(sc); 2962 2963 sx_xunlock(&sc->sc_lock); 2964 } 2965 2966 static int 2967 wg_clone_create(struct if_clone *ifc, char *name, size_t len, 2968 struct ifc_data *ifd, struct ifnet **ifpp) 2969 { 2970 struct wg_softc *sc; 2971 if_t ifp; 2972 2973 sc = malloc(sizeof(*sc), M_WG, M_WAITOK | M_ZERO); 2974 2975 sc->sc_local = noise_local_alloc(sc); 2976 2977 sc->sc_encrypt = mallocarray(sizeof(struct grouptask), mp_ncpus, M_WG, M_WAITOK | M_ZERO); 2978 2979 sc->sc_decrypt = mallocarray(sizeof(struct grouptask), mp_ncpus, M_WG, M_WAITOK | M_ZERO); 2980 2981 if (!rn_inithead((void **)&sc->sc_aip4, offsetof(struct aip_addr, in) * NBBY)) 2982 goto free_decrypt; 2983 2984 if (!rn_inithead((void **)&sc->sc_aip6, offsetof(struct aip_addr, in6) * NBBY)) 2985 goto free_aip4; 2986 2987 atomic_add_int(&clone_count, 1); 2988 ifp = sc->sc_ifp = if_alloc(IFT_WIREGUARD); 2989 2990 sc->sc_ucred = crhold(curthread->td_ucred); 2991 sc->sc_socket.so_fibnum = curthread->td_proc->p_fibnum; 2992 sc->sc_socket.so_port = 0; 2993 2994 TAILQ_INIT(&sc->sc_peers); 2995 sc->sc_peers_num = 0; 2996 2997 cookie_checker_init(&sc->sc_cookie); 2998 2999 RADIX_NODE_HEAD_LOCK_INIT(sc->sc_aip4); 3000 RADIX_NODE_HEAD_LOCK_INIT(sc->sc_aip6); 3001 3002 GROUPTASK_INIT(&sc->sc_handshake, 0, (gtask_fn_t *)wg_softc_handshake_receive, sc); 3003 taskqgroup_attach(qgroup_wg_tqg, &sc->sc_handshake, sc, NULL, NULL, "wg tx initiation"); 3004 wg_queue_init(&sc->sc_handshake_queue, "hsq"); 3005 3006 for (int i = 0; i < mp_ncpus; i++) { 3007 GROUPTASK_INIT(&sc->sc_encrypt[i], 0, 3008 (gtask_fn_t *)wg_softc_encrypt, sc); 3009 taskqgroup_attach_cpu(qgroup_wg_tqg, &sc->sc_encrypt[i], sc, i, NULL, NULL, "wg encrypt"); 3010 GROUPTASK_INIT(&sc->sc_decrypt[i], 0, 3011 (gtask_fn_t *)wg_softc_decrypt, sc); 3012 taskqgroup_attach_cpu(qgroup_wg_tqg, &sc->sc_decrypt[i], sc, i, NULL, NULL, "wg decrypt"); 3013 } 3014 3015 wg_queue_init(&sc->sc_encrypt_parallel, "encp"); 3016 wg_queue_init(&sc->sc_decrypt_parallel, "decp"); 3017 3018 sx_init(&sc->sc_lock, "wg softc lock"); 3019 3020 if_setsoftc(ifp, sc); 3021 if_setcapabilities(ifp, WG_CAPS); 3022 if_setcapenable(ifp, WG_CAPS); 3023 if_initname(ifp, wgname, ifd->unit); 3024 3025 if_setmtu(ifp, DEFAULT_MTU); 3026 if_setflags(ifp, IFF_NOARP | IFF_MULTICAST); 3027 if_setinitfn(ifp, wg_init); 3028 if_setreassignfn(ifp, wg_reassign); 3029 if_setqflushfn(ifp, wg_qflush); 3030 if_settransmitfn(ifp, wg_transmit); 3031 #ifdef DEV_NETMAP 3032 if_setinputfn(ifp, wg_if_input); 3033 #endif 3034 if_setoutputfn(ifp, wg_output); 3035 if_setioctlfn(ifp, wg_ioctl); 3036 if_attach(ifp); 3037 bpfattach(ifp, DLT_NULL, sizeof(uint32_t)); 3038 #ifdef INET6 3039 if_getinet6(ifp)->nd_flags &= ~ND6_IFF_AUTO_LINKLOCAL; 3040 if_getinet6(ifp)->nd_flags |= ND6_IFF_NO_DAD; 3041 #endif 3042 sx_xlock(&wg_sx); 3043 LIST_INSERT_HEAD(&wg_list, sc, sc_entry); 3044 sx_xunlock(&wg_sx); 3045 *ifpp = ifp; 3046 return (0); 3047 free_aip4: 3048 RADIX_NODE_HEAD_DESTROY(sc->sc_aip4); 3049 free(sc->sc_aip4, M_RTABLE); 3050 free_decrypt: 3051 free(sc->sc_decrypt, M_WG); 3052 free(sc->sc_encrypt, M_WG); 3053 noise_local_free(sc->sc_local, NULL); 3054 free(sc, M_WG); 3055 return (ENOMEM); 3056 } 3057 3058 static void 3059 wg_clone_deferred_free(struct noise_local *l) 3060 { 3061 struct wg_softc *sc = noise_local_arg(l); 3062 3063 free(sc, M_WG); 3064 atomic_add_int(&clone_count, -1); 3065 } 3066 3067 static int 3068 wg_clone_destroy(struct if_clone *ifc, if_t ifp, uint32_t flags) 3069 { 3070 struct wg_softc *sc = if_getsoftc(ifp); 3071 struct ucred *cred; 3072 3073 sx_xlock(&wg_sx); 3074 if_setsoftc(ifp, NULL); 3075 sx_xlock(&sc->sc_lock); 3076 sc->sc_flags |= WGF_DYING; 3077 cred = sc->sc_ucred; 3078 sc->sc_ucred = NULL; 3079 sx_xunlock(&sc->sc_lock); 3080 LIST_REMOVE(sc, sc_entry); 3081 sx_xunlock(&wg_sx); 3082 3083 if_link_state_change(sc->sc_ifp, LINK_STATE_DOWN); 3084 CURVNET_SET(if_getvnet(sc->sc_ifp)); 3085 if_purgeaddrs(sc->sc_ifp); 3086 CURVNET_RESTORE(); 3087 3088 sx_xlock(&sc->sc_lock); 3089 wg_socket_uninit(sc); 3090 sx_xunlock(&sc->sc_lock); 3091 3092 /* 3093 * No guarantees that all traffic have passed until the epoch has 3094 * elapsed with the socket closed. 3095 */ 3096 NET_EPOCH_WAIT(); 3097 3098 taskqgroup_drain_all(qgroup_wg_tqg); 3099 sx_xlock(&sc->sc_lock); 3100 wg_peer_destroy_all(sc); 3101 NET_EPOCH_DRAIN_CALLBACKS(); 3102 sx_xunlock(&sc->sc_lock); 3103 sx_destroy(&sc->sc_lock); 3104 taskqgroup_detach(qgroup_wg_tqg, &sc->sc_handshake); 3105 for (int i = 0; i < mp_ncpus; i++) { 3106 taskqgroup_detach(qgroup_wg_tqg, &sc->sc_encrypt[i]); 3107 taskqgroup_detach(qgroup_wg_tqg, &sc->sc_decrypt[i]); 3108 } 3109 free(sc->sc_encrypt, M_WG); 3110 free(sc->sc_decrypt, M_WG); 3111 wg_queue_deinit(&sc->sc_handshake_queue); 3112 wg_queue_deinit(&sc->sc_encrypt_parallel); 3113 wg_queue_deinit(&sc->sc_decrypt_parallel); 3114 3115 RADIX_NODE_HEAD_DESTROY(sc->sc_aip4); 3116 RADIX_NODE_HEAD_DESTROY(sc->sc_aip6); 3117 rn_detachhead((void **)&sc->sc_aip4); 3118 rn_detachhead((void **)&sc->sc_aip6); 3119 3120 cookie_checker_free(&sc->sc_cookie); 3121 3122 if (cred != NULL) 3123 crfree(cred); 3124 bpfdetach(sc->sc_ifp); 3125 if_detach(sc->sc_ifp); 3126 if_free(sc->sc_ifp); 3127 3128 noise_local_free(sc->sc_local, wg_clone_deferred_free); 3129 3130 return (0); 3131 } 3132 3133 static void 3134 wg_qflush(if_t ifp __unused) 3135 { 3136 } 3137 3138 /* 3139 * Privileged information (private-key, preshared-key) are only exported for 3140 * root and jailed root by default. 3141 */ 3142 static bool 3143 wgc_privileged(struct wg_softc *sc) 3144 { 3145 struct thread *td; 3146 3147 td = curthread; 3148 return (priv_check(td, PRIV_NET_WG) == 0); 3149 } 3150 3151 static void 3152 wg_reassign(if_t ifp, struct vnet *new_vnet __unused, 3153 char *unused __unused) 3154 { 3155 struct wg_softc *sc; 3156 3157 sc = if_getsoftc(ifp); 3158 wg_down(sc); 3159 } 3160 3161 static void 3162 wg_init(void *xsc) 3163 { 3164 struct wg_softc *sc; 3165 3166 sc = xsc; 3167 wg_up(sc); 3168 } 3169 3170 static void 3171 vnet_wg_init(const void *unused __unused) 3172 { 3173 struct if_clone_addreq req = { 3174 .create_f = wg_clone_create, 3175 .destroy_f = wg_clone_destroy, 3176 .flags = IFC_F_AUTOUNIT, 3177 }; 3178 V_wg_cloner = ifc_attach_cloner(wgname, &req); 3179 } 3180 VNET_SYSINIT(vnet_wg_init, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY, 3181 vnet_wg_init, NULL); 3182 3183 static void 3184 vnet_wg_uninit(const void *unused __unused) 3185 { 3186 if (V_wg_cloner) 3187 ifc_detach_cloner(V_wg_cloner); 3188 } 3189 VNET_SYSUNINIT(vnet_wg_uninit, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY, 3190 vnet_wg_uninit, NULL); 3191 3192 static int 3193 wg_prison_remove(void *obj, void *data __unused) 3194 { 3195 const struct prison *pr = obj; 3196 struct wg_softc *sc; 3197 3198 /* 3199 * Do a pass through all if_wg interfaces and release creds on any from 3200 * the jail that are supposed to be going away. This will, in turn, let 3201 * the jail die so that we don't end up with Schrödinger's jail. 3202 */ 3203 sx_slock(&wg_sx); 3204 LIST_FOREACH(sc, &wg_list, sc_entry) { 3205 sx_xlock(&sc->sc_lock); 3206 if (!(sc->sc_flags & WGF_DYING) && sc->sc_ucred && sc->sc_ucred->cr_prison == pr) { 3207 struct ucred *cred = sc->sc_ucred; 3208 DPRINTF(sc, "Creating jail exiting\n"); 3209 if_link_state_change(sc->sc_ifp, LINK_STATE_DOWN); 3210 wg_socket_uninit(sc); 3211 sc->sc_ucred = NULL; 3212 crfree(cred); 3213 sc->sc_flags |= WGF_DYING; 3214 } 3215 sx_xunlock(&sc->sc_lock); 3216 } 3217 sx_sunlock(&wg_sx); 3218 3219 return (0); 3220 } 3221 3222 #ifdef SELFTESTS 3223 #include "selftest/allowedips.c" 3224 static bool wg_run_selftests(void) 3225 { 3226 bool ret = true; 3227 ret &= wg_allowedips_selftest(); 3228 ret &= noise_counter_selftest(); 3229 ret &= cookie_selftest(); 3230 return ret; 3231 } 3232 #else 3233 static inline bool wg_run_selftests(void) { return true; } 3234 #endif 3235 3236 static int 3237 wg_module_init(void) 3238 { 3239 int ret; 3240 osd_method_t methods[PR_MAXMETHOD] = { 3241 [PR_METHOD_REMOVE] = wg_prison_remove, 3242 }; 3243 3244 wg_packet_zone = uma_zcreate("wg packet", sizeof(struct wg_packet), 3245 NULL, NULL, NULL, NULL, 0, 0); 3246 3247 ret = wg_crypto_init(); 3248 if (ret != 0) 3249 return (ret); 3250 ret = cookie_init(); 3251 if (ret != 0) 3252 return (ret); 3253 3254 wg_osd_jail_slot = osd_jail_register(NULL, methods); 3255 3256 if (!wg_run_selftests()) 3257 return (ENOTRECOVERABLE); 3258 3259 return (0); 3260 } 3261 3262 static void 3263 wg_module_deinit(void) 3264 { 3265 VNET_ITERATOR_DECL(vnet_iter); 3266 VNET_LIST_RLOCK(); 3267 VNET_FOREACH(vnet_iter) { 3268 struct if_clone *clone = VNET_VNET(vnet_iter, wg_cloner); 3269 if (clone) { 3270 ifc_detach_cloner(clone); 3271 VNET_VNET(vnet_iter, wg_cloner) = NULL; 3272 } 3273 } 3274 VNET_LIST_RUNLOCK(); 3275 NET_EPOCH_WAIT(); 3276 MPASS(LIST_EMPTY(&wg_list)); 3277 if (wg_osd_jail_slot != 0) 3278 osd_jail_deregister(wg_osd_jail_slot); 3279 cookie_deinit(); 3280 wg_crypto_deinit(); 3281 if (wg_packet_zone != NULL) 3282 uma_zdestroy(wg_packet_zone); 3283 } 3284 3285 static int 3286 wg_module_event_handler(module_t mod, int what, void *arg) 3287 { 3288 switch (what) { 3289 case MOD_LOAD: 3290 return wg_module_init(); 3291 case MOD_UNLOAD: 3292 wg_module_deinit(); 3293 break; 3294 default: 3295 return (EOPNOTSUPP); 3296 } 3297 return (0); 3298 } 3299 3300 static moduledata_t wg_moduledata = { 3301 "if_wg", 3302 wg_module_event_handler, 3303 NULL 3304 }; 3305 3306 DECLARE_MODULE(if_wg, wg_moduledata, SI_SUB_PSEUDO, SI_ORDER_ANY); 3307 MODULE_VERSION(if_wg, WIREGUARD_VERSION); 3308 MODULE_DEPEND(if_wg, crypto, 1, 1, 1); 3309