Lines Matching +full:pkt +full:- +full:size
1 /* SPDX-License-Identifier: ISC
3 * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
4 * Copyright (C) 2019-2021 Matt Dunwoodie <ncon@noconroy.net>
5 * Copyright (c) 2019-2020 Rubicon Communications, LLC (Netgate)
56 #define DEFAULT_MTU (ETHERMTU - 80)
57 #define MAX_MTU (IF_MAXMTU - 80)
61 #define MAX_QUEUED_PKT_MASK (MAX_QUEUED_PKT - 1)
70 #define DPRINTF(sc, ...) if (if_getflags(sc->sc_ifp) & IFF_DEBUG) if_printf(sc->sc_ifp, ##__VA_ARGS…
256 gtaskqueue_drain((gtask)->gt_taskqueue, &(gtask)->gt_task)
390 sx_assert(&sc->sc_lock, SX_XLOCKED); in wg_peer_create()
394 peer->p_remote = noise_remote_alloc(sc->sc_local, peer, pub_key); in wg_peer_create()
395 if ((*errp = noise_remote_enable(peer->p_remote)) != 0) { in wg_peer_create()
396 noise_remote_free(peer->p_remote, NULL); in wg_peer_create()
401 peer->p_id = peer_counter++; in wg_peer_create()
402 peer->p_sc = sc; in wg_peer_create()
403 peer->p_tx_bytes = counter_u64_alloc(M_WAITOK); in wg_peer_create()
404 peer->p_rx_bytes = counter_u64_alloc(M_WAITOK); in wg_peer_create()
406 cookie_maker_init(&peer->p_cookie, pub_key); in wg_peer_create()
408 rw_init(&peer->p_endpoint_lock, "wg_peer_endpoint"); in wg_peer_create()
410 wg_queue_init(&peer->p_stage_queue, "stageq"); in wg_peer_create()
411 wg_queue_init(&peer->p_encrypt_serial, "txq"); in wg_peer_create()
412 wg_queue_init(&peer->p_decrypt_serial, "rxq"); in wg_peer_create()
414 peer->p_enabled = false; in wg_peer_create()
415 peer->p_need_another_keepalive = false; in wg_peer_create()
416 peer->p_persistent_keepalive_interval = 0; in wg_peer_create()
417 callout_init(&peer->p_new_handshake, true); in wg_peer_create()
418 callout_init(&peer->p_send_keepalive, true); in wg_peer_create()
419 callout_init(&peer->p_retry_handshake, true); in wg_peer_create()
420 callout_init(&peer->p_persistent_keepalive, true); in wg_peer_create()
421 callout_init(&peer->p_zero_key_material, true); in wg_peer_create()
423 mtx_init(&peer->p_handshake_mtx, "peer handshake", NULL, MTX_DEF); in wg_peer_create()
424 bzero(&peer->p_handshake_complete, sizeof(peer->p_handshake_complete)); in wg_peer_create()
425 peer->p_handshake_retries = 0; in wg_peer_create()
427 GROUPTASK_INIT(&peer->p_send, 0, (gtask_fn_t *)wg_deliver_out, peer); in wg_peer_create()
428 taskqgroup_attach(qgroup_wg_tqg, &peer->p_send, peer, NULL, NULL, "wg send"); in wg_peer_create()
429 GROUPTASK_INIT(&peer->p_recv, 0, (gtask_fn_t *)wg_deliver_in, peer); in wg_peer_create()
430 taskqgroup_attach(qgroup_wg_tqg, &peer->p_recv, peer, NULL, NULL, "wg recv"); in wg_peer_create()
432 LIST_INIT(&peer->p_aips); in wg_peer_create()
433 peer->p_aips_num = 0; in wg_peer_create()
435 TAILQ_INSERT_TAIL(&sc->sc_peers, peer, p_entry); in wg_peer_create()
436 sc->sc_peers_num++; in wg_peer_create()
438 if (if_getlinkstate(sc->sc_ifp) == LINK_STATE_UP) in wg_peer_create()
441 DPRINTF(sc, "Peer %" PRIu64 " created\n", peer->p_id); in wg_peer_create()
453 GROUPTASK_DRAIN(&peer->p_recv); in wg_peer_free_deferred()
454 GROUPTASK_DRAIN(&peer->p_send); in wg_peer_free_deferred()
455 taskqgroup_detach(qgroup_wg_tqg, &peer->p_recv); in wg_peer_free_deferred()
456 taskqgroup_detach(qgroup_wg_tqg, &peer->p_send); in wg_peer_free_deferred()
458 wg_queue_deinit(&peer->p_decrypt_serial); in wg_peer_free_deferred()
459 wg_queue_deinit(&peer->p_encrypt_serial); in wg_peer_free_deferred()
460 wg_queue_deinit(&peer->p_stage_queue); in wg_peer_free_deferred()
462 counter_u64_free(peer->p_tx_bytes); in wg_peer_free_deferred()
463 counter_u64_free(peer->p_rx_bytes); in wg_peer_free_deferred()
464 rw_destroy(&peer->p_endpoint_lock); in wg_peer_free_deferred()
465 mtx_destroy(&peer->p_handshake_mtx); in wg_peer_free_deferred()
467 cookie_maker_free(&peer->p_cookie); in wg_peer_free_deferred()
475 struct wg_softc *sc = peer->p_sc; in wg_peer_destroy()
476 sx_assert(&sc->sc_lock, SX_XLOCKED); in wg_peer_destroy()
480 noise_remote_disable(peer->p_remote); in wg_peer_destroy()
490 sc->sc_peers_num--; in wg_peer_destroy()
491 TAILQ_REMOVE(&sc->sc_peers, peer, p_entry); in wg_peer_destroy()
492 DPRINTF(sc, "Peer %" PRIu64 " destroyed\n", peer->p_id); in wg_peer_destroy()
493 noise_remote_free(peer->p_remote, wg_peer_free_deferred); in wg_peer_destroy()
500 TAILQ_FOREACH_SAFE(peer, &sc->sc_peers, p_entry, tpeer) in wg_peer_destroy_all()
507 MPASS(e->e_remote.r_sa.sa_family != 0); in wg_peer_set_endpoint()
508 if (memcmp(e, &peer->p_endpoint, sizeof(*e)) == 0) in wg_peer_set_endpoint()
511 rw_wlock(&peer->p_endpoint_lock); in wg_peer_set_endpoint()
512 peer->p_endpoint = *e; in wg_peer_set_endpoint()
513 rw_wunlock(&peer->p_endpoint_lock); in wg_peer_set_endpoint()
519 rw_wlock(&peer->p_endpoint_lock); in wg_peer_clear_src()
520 bzero(&peer->p_endpoint.e_local, sizeof(peer->p_endpoint.e_local)); in wg_peer_clear_src()
521 rw_wunlock(&peer->p_endpoint_lock); in wg_peer_clear_src()
527 rw_rlock(&peer->p_endpoint_lock); in wg_peer_get_endpoint()
528 *e = peer->p_endpoint; in wg_peer_get_endpoint()
529 rw_runlock(&peer->p_endpoint_lock); in wg_peer_get_endpoint()
538 addr = &aip->a_addr; in wg_aip_addrinfo()
539 mask = &aip->a_mask; in wg_aip_addrinfo()
541 switch (aip->a_af) { in wg_aip_addrinfo()
545 addr->in = *(const struct in_addr *)baddr; in wg_aip_addrinfo()
546 mask->ip = htonl(~((1LL << (32 - cidr)) - 1) & 0xffffffff); in wg_aip_addrinfo()
547 addr->ip &= mask->ip; in wg_aip_addrinfo()
548 addr->length = mask->length = offsetof(struct aip_addr, in) + sizeof(struct in_addr); in wg_aip_addrinfo()
554 addr->in6 = *(const struct in6_addr *)baddr; in wg_aip_addrinfo()
555 in6_prefixlen2mask(&mask->in6, cidr); in wg_aip_addrinfo()
557 addr->ip6[i] &= mask->ip6[i]; in wg_aip_addrinfo()
558 addr->length = mask->length = offsetof(struct aip_addr, in6) + sizeof(struct in6_addr); in wg_aip_addrinfo()
579 aip->a_peer = peer; in wg_aip_add()
580 aip->a_af = af; in wg_aip_add()
588 root = af == AF_INET ? sc->sc_aip4 : sc->sc_aip6; in wg_aip_add()
591 node = root->rnh_addaddr(&aip->a_addr, &aip->a_mask, &root->rh, aip->a_nodes); in wg_aip_add()
592 if (node == aip->a_nodes) { in wg_aip_add()
593 LIST_INSERT_HEAD(&peer->p_aips, aip, a_entry); in wg_aip_add()
594 peer->p_aips_num++; in wg_aip_add()
596 node = root->rnh_lookup(&aip->a_addr, &aip->a_mask, &root->rh); in wg_aip_add()
600 } else if (node != aip->a_nodes) { in wg_aip_add()
603 if (aip->a_peer != peer) { in wg_aip_add()
605 aip->a_peer->p_aips_num--; in wg_aip_add()
606 aip->a_peer = peer; in wg_aip_add()
607 LIST_INSERT_HEAD(&peer->p_aips, aip, a_entry); in wg_aip_add()
608 aip->a_peer->p_aips_num++; in wg_aip_add()
635 root = af == AF_INET ? sc->sc_aip4 : sc->sc_aip6; in wg_aip_del()
640 node = root->rnh_lookup(&addr.a_addr, &addr.a_mask, &root->rh); in wg_aip_del()
647 if (aip->a_peer != peer) { in wg_aip_del()
649 * They could have specified an allowed-ip that belonged to a in wg_aip_del()
657 dnode = root->rnh_deladdr(&aip->a_addr, &aip->a_mask, &root->rh); in wg_aip_del()
662 peer->p_aips_num--; in wg_aip_del()
678 root = sc->sc_aip4; in wg_aip_lookup()
683 root = sc->sc_aip6; in wg_aip_lookup()
692 node = root->rnh_matchaddr(&addr, &root->rh); in wg_aip_lookup()
694 peer = ((struct wg_aip *)node)->a_peer; in wg_aip_lookup()
695 noise_remote_ref(peer->p_remote); in wg_aip_lookup()
709 RADIX_NODE_HEAD_LOCK(sc->sc_aip4); in wg_aip_remove_all()
710 LIST_FOREACH_SAFE(aip, &peer->p_aips, a_entry, taip) { in wg_aip_remove_all()
711 if (aip->a_af == AF_INET) { in wg_aip_remove_all()
712 if (sc->sc_aip4->rnh_deladdr(&aip->a_addr, &aip->a_mask, &sc->sc_aip4->rh) == NULL) in wg_aip_remove_all()
715 peer->p_aips_num--; in wg_aip_remove_all()
719 RADIX_NODE_HEAD_UNLOCK(sc->sc_aip4); in wg_aip_remove_all()
721 RADIX_NODE_HEAD_LOCK(sc->sc_aip6); in wg_aip_remove_all()
722 LIST_FOREACH_SAFE(aip, &peer->p_aips, a_entry, taip) { in wg_aip_remove_all()
723 if (aip->a_af == AF_INET6) { in wg_aip_remove_all()
724 if (sc->sc_aip6->rnh_deladdr(&aip->a_addr, &aip->a_mask, &sc->sc_aip6->rh) == NULL) in wg_aip_remove_all()
727 peer->p_aips_num--; in wg_aip_remove_all()
731 RADIX_NODE_HEAD_UNLOCK(sc->sc_aip6); in wg_aip_remove_all()
733 if (!LIST_EMPTY(&peer->p_aips) || peer->p_aips_num != 0) in wg_aip_remove_all()
740 struct ucred *cred = sc->sc_ucred; in wg_socket_init()
744 sx_assert(&sc->sc_lock, SX_XLOCKED); in wg_socket_init()
752 * WireGuard has on Linux with network namespaces -- that the sockets in wg_socket_init()
778 if (sc->sc_socket.so_user_cookie) { in wg_socket_init()
779 …rc = wg_socket_set_sockopt(so4, so6, SO_USER_COOKIE, &sc->sc_socket.so_user_cookie, sizeof(sc->sc_… in wg_socket_init()
783 …rc = wg_socket_set_sockopt(so4, so6, SO_SETFIB, &sc->sc_socket.so_fibnum, sizeof(sc->sc_socket.so_… in wg_socket_init()
789 sc->sc_socket.so_port = port; in wg_socket_init()
822 struct wg_socket *so = &sc->sc_socket; in wg_socket_set_cookie()
825 sx_assert(&sc->sc_lock, SX_XLOCKED); in wg_socket_set_cookie()
826 …ret = wg_socket_set_sockopt(so->so_so4, so->so_so6, SO_USER_COOKIE, &user_cookie, sizeof(user_cook… in wg_socket_set_cookie()
828 so->so_user_cookie = user_cookie; in wg_socket_set_cookie()
834 struct wg_socket *so = &sc->sc_socket; in wg_socket_set_fibnum()
837 sx_assert(&sc->sc_lock, SX_XLOCKED); in wg_socket_set_fibnum()
839 ret = wg_socket_set_sockopt(so->so_so4, so->so_so6, SO_SETFIB, &fibnum, sizeof(fibnum)); in wg_socket_set_fibnum()
841 so->so_fibnum = fibnum; in wg_socket_set_fibnum()
854 struct wg_socket *so = &sc->sc_socket; in wg_socket_set()
857 sx_assert(&sc->sc_lock, SX_XLOCKED); in wg_socket_set()
859 so4 = atomic_load_ptr(&so->so_so4); in wg_socket_set()
860 so6 = atomic_load_ptr(&so->so_so6); in wg_socket_set()
861 atomic_store_ptr(&so->so_so4, new_so4); in wg_socket_set()
862 atomic_store_ptr(&so->so_so6, new_so6); in wg_socket_set()
941 struct wg_socket *so = &sc->sc_socket; in wg_send()
945 size_t len = m->m_pkthdr.len; in wg_send()
948 if (e->e_remote.r_sa.sa_family == AF_INET) { in wg_send()
949 if (e->e_local.l_in.s_addr != INADDR_ANY) in wg_send()
950 control = sbcreatecontrol((caddr_t)&e->e_local.l_in, in wg_send()
954 } else if (e->e_remote.r_sa.sa_family == AF_INET6) { in wg_send()
955 if (!IN6_IS_ADDR_UNSPECIFIED(&e->e_local.l_in6)) in wg_send()
956 control = sbcreatecontrol((caddr_t)&e->e_local.l_pktinfo6, in wg_send()
966 sa = &e->e_remote.r_sa; in wg_send()
969 so4 = atomic_load_ptr(&so->so_so4); in wg_send()
970 so6 = atomic_load_ptr(&so->so_so6); in wg_send()
971 if (e->e_remote.r_sa.sa_family == AF_INET && so4 != NULL) in wg_send()
973 else if (e->e_remote.r_sa.sa_family == AF_INET6 && so6 != NULL) in wg_send()
982 if_inc_counter(sc->sc_ifp, IFCOUNTER_OPACKETS, 1); in wg_send()
983 if_inc_counter(sc->sc_ifp, IFCOUNTER_OBYTES, len); in wg_send()
1005 /* Retry if we couldn't bind to e->e_local */ in wg_send_buf()
1007 bzero(&e->e_local, sizeof(e->e_local)); in wg_send_buf()
1023 atomic_store_bool(&peer->p_enabled, true); in wg_timers_enable()
1042 atomic_store_bool(&peer->p_enabled, false); in wg_timers_disable()
1044 atomic_store_bool(&peer->p_need_another_keepalive, false); in wg_timers_disable()
1046 callout_stop(&peer->p_new_handshake); in wg_timers_disable()
1047 callout_stop(&peer->p_send_keepalive); in wg_timers_disable()
1048 callout_stop(&peer->p_retry_handshake); in wg_timers_disable()
1049 callout_stop(&peer->p_persistent_keepalive); in wg_timers_disable()
1050 callout_stop(&peer->p_zero_key_material); in wg_timers_disable()
1057 if (interval != peer->p_persistent_keepalive_interval) { in wg_timers_set_persistent_keepalive()
1058 atomic_store_16(&peer->p_persistent_keepalive_interval, interval); in wg_timers_set_persistent_keepalive()
1060 if (atomic_load_bool(&peer->p_enabled)) in wg_timers_set_persistent_keepalive()
1069 mtx_lock(&peer->p_handshake_mtx); in wg_timers_get_last_handshake()
1070 time->tv_sec = peer->p_handshake_complete.tv_sec; in wg_timers_get_last_handshake()
1071 time->tv_nsec = peer->p_handshake_complete.tv_nsec; in wg_timers_get_last_handshake()
1072 mtx_unlock(&peer->p_handshake_mtx); in wg_timers_get_last_handshake()
1080 if (atomic_load_bool(&peer->p_enabled) && in wg_timers_event_data_sent()
1081 !callout_pending(&peer->p_new_handshake)) in wg_timers_event_data_sent()
1082 callout_reset(&peer->p_new_handshake, MSEC_2_TICKS( in wg_timers_event_data_sent()
1094 if (atomic_load_bool(&peer->p_enabled)) { in wg_timers_event_data_received()
1095 if (!callout_pending(&peer->p_send_keepalive)) in wg_timers_event_data_received()
1096 callout_reset(&peer->p_send_keepalive, in wg_timers_event_data_received()
1100 atomic_store_bool(&peer->p_need_another_keepalive, in wg_timers_event_data_received()
1109 callout_stop(&peer->p_send_keepalive); in wg_timers_event_any_authenticated_packet_sent()
1115 callout_stop(&peer->p_new_handshake); in wg_timers_event_any_authenticated_packet_received()
1124 interval = atomic_load_16(&peer->p_persistent_keepalive_interval); in wg_timers_event_any_authenticated_packet_traversal()
1125 if (atomic_load_bool(&peer->p_enabled) && interval > 0) in wg_timers_event_any_authenticated_packet_traversal()
1126 callout_reset(&peer->p_persistent_keepalive, in wg_timers_event_any_authenticated_packet_traversal()
1137 if (atomic_load_bool(&peer->p_enabled)) in wg_timers_event_handshake_initiated()
1138 callout_reset(&peer->p_retry_handshake, MSEC_2_TICKS( in wg_timers_event_handshake_initiated()
1150 if (atomic_load_bool(&peer->p_enabled)) { in wg_timers_event_handshake_complete()
1151 mtx_lock(&peer->p_handshake_mtx); in wg_timers_event_handshake_complete()
1152 callout_stop(&peer->p_retry_handshake); in wg_timers_event_handshake_complete()
1153 peer->p_handshake_retries = 0; in wg_timers_event_handshake_complete()
1154 getnanotime(&peer->p_handshake_complete); in wg_timers_event_handshake_complete()
1155 mtx_unlock(&peer->p_handshake_mtx); in wg_timers_event_handshake_complete()
1166 if (atomic_load_bool(&peer->p_enabled)) in wg_timers_event_session_derived()
1167 callout_reset(&peer->p_zero_key_material, in wg_timers_event_session_derived()
1178 if (atomic_load_bool(&peer->p_enabled)) in wg_timers_event_want_initiation()
1187 peer->p_handshake_retries = 0; in wg_timers_run_send_initiation()
1188 if (noise_remote_initiation_expired(peer->p_remote) == ETIMEDOUT) in wg_timers_run_send_initiation()
1198 mtx_lock(&peer->p_handshake_mtx); in wg_timers_run_retry_handshake()
1199 if (peer->p_handshake_retries <= MAX_TIMER_HANDSHAKES) { in wg_timers_run_retry_handshake()
1200 peer->p_handshake_retries++; in wg_timers_run_retry_handshake()
1201 mtx_unlock(&peer->p_handshake_mtx); in wg_timers_run_retry_handshake()
1203 DPRINTF(peer->p_sc, "Handshake for peer %" PRIu64 " did not complete " in wg_timers_run_retry_handshake()
1204 "after %d seconds, retrying (try %d)\n", peer->p_id, in wg_timers_run_retry_handshake()
1205 REKEY_TIMEOUT, peer->p_handshake_retries + 1); in wg_timers_run_retry_handshake()
1209 mtx_unlock(&peer->p_handshake_mtx); in wg_timers_run_retry_handshake()
1211 DPRINTF(peer->p_sc, "Handshake for peer %" PRIu64 " did not complete " in wg_timers_run_retry_handshake()
1212 "after %d retries, giving up\n", peer->p_id, in wg_timers_run_retry_handshake()
1215 callout_stop(&peer->p_send_keepalive); in wg_timers_run_retry_handshake()
1216 wg_queue_purge(&peer->p_stage_queue); in wg_timers_run_retry_handshake()
1218 if (atomic_load_bool(&peer->p_enabled) && in wg_timers_run_retry_handshake()
1219 !callout_pending(&peer->p_zero_key_material)) in wg_timers_run_retry_handshake()
1220 callout_reset(&peer->p_zero_key_material, in wg_timers_run_retry_handshake()
1235 if (atomic_load_bool(&peer->p_enabled) && in wg_timers_run_send_keepalive()
1236 atomic_load_bool(&peer->p_need_another_keepalive)) { in wg_timers_run_send_keepalive()
1237 atomic_store_bool(&peer->p_need_another_keepalive, false); in wg_timers_run_send_keepalive()
1238 callout_reset(&peer->p_send_keepalive, in wg_timers_run_send_keepalive()
1250 DPRINTF(peer->p_sc, "Retrying handshake with peer %" PRIu64 " because we " in wg_timers_run_new_handshake()
1252 peer->p_id, NEW_HANDSHAKE_TIMEOUT); in wg_timers_run_new_handshake()
1263 DPRINTF(peer->p_sc, "Zeroing out keys for peer %" PRIu64 ", since we " in wg_timers_run_zero_key_material()
1265 peer->p_id, REJECT_AFTER_TIME * 3); in wg_timers_run_zero_key_material()
1266 noise_remote_keypairs_clear(peer->p_remote); in wg_timers_run_zero_key_material()
1274 if (atomic_load_16(&peer->p_persistent_keepalive_interval) > 0) in wg_timers_run_persistent_keepalive()
1284 counter_u64_add(peer->p_tx_bytes, len); in wg_peer_send_buf()
1288 wg_send_buf(peer->p_sc, &endpoint, buf, len); in wg_peer_send_buf()
1294 struct wg_pkt_initiation pkt; in wg_send_initiation() local
1296 if (noise_create_initiation(peer->p_remote, &pkt.s_idx, pkt.ue, in wg_send_initiation()
1297 pkt.es, pkt.ets) != 0) in wg_send_initiation()
1300 DPRINTF(peer->p_sc, "Sending handshake initiation to peer %" PRIu64 "\n", peer->p_id); in wg_send_initiation()
1302 pkt.t = WG_PKT_INITIATION; in wg_send_initiation()
1303 cookie_maker_mac(&peer->p_cookie, &pkt.m, &pkt, in wg_send_initiation()
1304 sizeof(pkt) - sizeof(pkt.m)); in wg_send_initiation()
1305 wg_peer_send_buf(peer, (uint8_t *)&pkt, sizeof(pkt)); in wg_send_initiation()
1312 struct wg_pkt_response pkt; in wg_send_response() local
1314 if (noise_create_response(peer->p_remote, &pkt.s_idx, &pkt.r_idx, in wg_send_response()
1315 pkt.ue, pkt.en) != 0) in wg_send_response()
1318 DPRINTF(peer->p_sc, "Sending handshake response to peer %" PRIu64 "\n", peer->p_id); in wg_send_response()
1321 pkt.t = WG_PKT_RESPONSE; in wg_send_response()
1322 cookie_maker_mac(&peer->p_cookie, &pkt.m, &pkt, in wg_send_response()
1323 sizeof(pkt)-sizeof(pkt.m)); in wg_send_response()
1324 wg_peer_send_buf(peer, (uint8_t*)&pkt, sizeof(pkt)); in wg_send_response()
1331 struct wg_pkt_cookie pkt; in wg_send_cookie() local
1335 pkt.t = WG_PKT_COOKIE; in wg_send_cookie()
1336 pkt.r_idx = idx; in wg_send_cookie()
1338 cookie_checker_create_payload(&sc->sc_cookie, cm, pkt.nonce, in wg_send_cookie()
1339 pkt.ec, &e->e_remote.r_sa); in wg_send_cookie()
1340 wg_send_buf(sc, e, (uint8_t *)&pkt, sizeof(pkt)); in wg_send_cookie()
1346 struct wg_packet *pkt; in wg_send_keepalive() local
1349 if (wg_queue_len(&peer->p_stage_queue) > 0) in wg_send_keepalive()
1353 if ((pkt = wg_packet_alloc(m)) == NULL) { in wg_send_keepalive()
1357 wg_queue_push_staged(&peer->p_stage_queue, pkt); in wg_send_keepalive()
1358 DPRINTF(peer->p_sc, "Sending keepalive packet to peer %" PRIu64 "\n", peer->p_id); in wg_send_keepalive()
1364 wg_handshake(struct wg_softc *sc, struct wg_packet *pkt) in wg_handshake() argument
1377 underload = wg_queue_len(&sc->sc_handshake_queue) >= MAX_QUEUED_HANDSHAKES / 8; in wg_handshake()
1386 m = pkt->p_mbuf; in wg_handshake()
1387 e = &pkt->p_endpoint; in wg_handshake()
1389 if ((pkt->p_mbuf = m = m_pullup(m, m->m_pkthdr.len)) == NULL) in wg_handshake()
1396 res = cookie_checker_validate_macs(&sc->sc_cookie, &init->m, in wg_handshake()
1397 init, sizeof(*init) - sizeof(init->m), in wg_handshake()
1398 underload, &e->e_remote.r_sa, in wg_handshake()
1399 if_getvnet(sc->sc_ifp)); in wg_handshake()
1408 wg_send_cookie(sc, &init->m, init->s_idx, e); in wg_handshake()
1414 if (noise_consume_initiation(sc->sc_local, &remote, in wg_handshake()
1415 init->s_idx, init->ue, init->es, init->ets) != 0) { in wg_handshake()
1422 DPRINTF(sc, "Receiving handshake initiation from peer %" PRIu64 "\n", peer->p_id); in wg_handshake()
1430 res = cookie_checker_validate_macs(&sc->sc_cookie, &resp->m, in wg_handshake()
1431 resp, sizeof(*resp) - sizeof(resp->m), in wg_handshake()
1432 underload, &e->e_remote.r_sa, in wg_handshake()
1433 if_getvnet(sc->sc_ifp)); in wg_handshake()
1442 wg_send_cookie(sc, &resp->m, resp->s_idx, e); in wg_handshake()
1448 if (noise_consume_response(sc->sc_local, &remote, in wg_handshake()
1449 resp->s_idx, resp->r_idx, resp->ue, resp->en) != 0) { in wg_handshake()
1455 DPRINTF(sc, "Receiving handshake response from peer %" PRIu64 "\n", peer->p_id); in wg_handshake()
1464 if ((remote = noise_remote_index(sc->sc_local, cook->r_idx)) == NULL) { in wg_handshake()
1471 if (cookie_maker_consume_payload(&peer->p_cookie, in wg_handshake()
1472 cook->nonce, cook->ec) == 0) { in wg_handshake()
1488 counter_u64_add(peer->p_rx_bytes, m->m_pkthdr.len); in wg_handshake()
1489 if_inc_counter(sc->sc_ifp, IFCOUNTER_IPACKETS, 1); in wg_handshake()
1490 if_inc_counter(sc->sc_ifp, IFCOUNTER_IBYTES, m->m_pkthdr.len); in wg_handshake()
1494 wg_packet_free(pkt); in wg_handshake()
1500 struct wg_packet *pkt; in wg_softc_handshake_receive() local
1501 while ((pkt = wg_queue_dequeue_handshake(&sc->sc_handshake_queue)) != NULL) in wg_softc_handshake_receive()
1502 wg_handshake(sc, pkt); in wg_softc_handshake_receive()
1527 m->m_flags &= ~(M_BCAST|M_MCAST|M_VLANTAG|M_PROMISC|M_PROTOFLAGS); in wg_mbuf_reset()
1531 m->m_pkthdr.numa_domain = M_NODOM; in wg_mbuf_reset()
1533 SLIST_FOREACH_SAFE(t, &m->m_pkthdr.tags, m_tag_link, tmp) { in wg_mbuf_reset()
1534 if ((t->m_tag_id != 0 || t->m_tag_cookie != MTAG_WGLOOP) && in wg_mbuf_reset()
1535 t->m_tag_id != PACKET_TAG_MACLABEL) in wg_mbuf_reset()
1539 KASSERT((m->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0, in wg_mbuf_reset()
1542 m->m_pkthdr.csum_flags = 0; in wg_mbuf_reset()
1543 m->m_pkthdr.PH_per.sixtyfour[0] = 0; in wg_mbuf_reset()
1544 m->m_pkthdr.PH_loc.sixtyfour[0] = 0; in wg_mbuf_reset()
1548 calculate_padding(struct wg_packet *pkt) in calculate_padding() argument
1550 unsigned int padded_size, last_unit = pkt->p_mbuf->m_pkthdr.len; in calculate_padding()
1553 if (__predict_false(pkt->p_mtu == 0)) { in calculate_padding()
1554 padded_size = (last_unit + (WG_PKT_PADDING - 1)) & in calculate_padding()
1555 ~(WG_PKT_PADDING - 1); in calculate_padding()
1556 return (padded_size - last_unit); in calculate_padding()
1559 if (__predict_false(last_unit > pkt->p_mtu)) in calculate_padding()
1560 last_unit %= pkt->p_mtu; in calculate_padding()
1562 padded_size = (last_unit + (WG_PKT_PADDING - 1)) & ~(WG_PKT_PADDING - 1); in calculate_padding()
1563 if (pkt->p_mtu < padded_size) in calculate_padding()
1564 padded_size = pkt->p_mtu; in calculate_padding()
1565 return (padded_size - last_unit); in calculate_padding()
1569 wg_encrypt(struct wg_softc *sc, struct wg_packet *pkt) in wg_encrypt() argument
1580 remote = noise_keypair_remote(pkt->p_keypair); in wg_encrypt()
1582 m = pkt->p_mbuf; in wg_encrypt()
1585 padlen = calculate_padding(pkt); in wg_encrypt()
1590 if (noise_keypair_encrypt(pkt->p_keypair, &idx, pkt->p_nonce, m) != 0) in wg_encrypt()
1598 data->t = WG_PKT_DATA; in wg_encrypt()
1599 data->r_idx = idx; in wg_encrypt()
1600 data->nonce = htole64(pkt->p_nonce); in wg_encrypt()
1605 pkt->p_mbuf = m; in wg_encrypt()
1606 atomic_store_rel_int(&pkt->p_state, state); in wg_encrypt()
1607 GROUPTASK_ENQUEUE(&peer->p_send); in wg_encrypt()
1612 wg_decrypt(struct wg_softc *sc, struct wg_packet *pkt) in wg_decrypt() argument
1620 remote = noise_keypair_remote(pkt->p_keypair); in wg_decrypt()
1622 m = pkt->p_mbuf; in wg_decrypt()
1625 pkt->p_nonce = le64toh(mtod(m, struct wg_pkt_data *)->nonce); in wg_decrypt()
1628 if (noise_keypair_decrypt(pkt->p_keypair, pkt->p_nonce, m) != 0) in wg_decrypt()
1632 if (__predict_false(m->m_pkthdr.len == 0)) { in wg_decrypt()
1634 "%" PRIu64 "\n", peer->p_id); in wg_decrypt()
1645 if (determine_af_and_pullup(&m, &pkt->p_af) == 0) { in wg_decrypt()
1646 if (pkt->p_af == AF_INET) { in wg_decrypt()
1648 allowed_peer = wg_aip_lookup(sc, AF_INET, &ip->ip_src); in wg_decrypt()
1649 len = ntohs(ip->ip_len); in wg_decrypt()
1650 if (len >= sizeof(struct ip) && len < m->m_pkthdr.len) in wg_decrypt()
1651 m_adj(m, len - m->m_pkthdr.len); in wg_decrypt()
1652 } else if (pkt->p_af == AF_INET6) { in wg_decrypt()
1654 allowed_peer = wg_aip_lookup(sc, AF_INET6, &ip6->ip6_src); in wg_decrypt()
1655 len = ntohs(ip6->ip6_plen) + sizeof(struct ip6_hdr); in wg_decrypt()
1656 if (len < m->m_pkthdr.len) in wg_decrypt()
1657 m_adj(m, len - m->m_pkthdr.len); in wg_decrypt()
1661 DPRINTF(sc, "Packet is neither ipv4 nor ipv6 from peer %" PRIu64 "\n", peer->p_id); in wg_decrypt()
1667 noise_remote_put(allowed_peer->p_remote); in wg_decrypt()
1670 DPRINTF(sc, "Packet has unallowed src IP from peer %" PRIu64 "\n", peer->p_id); in wg_decrypt()
1677 pkt->p_mbuf = m; in wg_decrypt()
1678 atomic_store_rel_int(&pkt->p_state, state); in wg_decrypt()
1679 GROUPTASK_ENQUEUE(&peer->p_recv); in wg_decrypt()
1686 struct wg_packet *pkt; in wg_softc_decrypt() local
1688 while ((pkt = wg_queue_dequeue_parallel(&sc->sc_decrypt_parallel)) != NULL) in wg_softc_decrypt()
1689 wg_decrypt(sc, pkt); in wg_softc_decrypt()
1695 struct wg_packet *pkt; in wg_softc_encrypt() local
1697 while ((pkt = wg_queue_dequeue_parallel(&sc->sc_encrypt_parallel)) != NULL) in wg_softc_encrypt()
1698 wg_encrypt(sc, pkt); in wg_softc_encrypt()
1709 u_int cpu = (sc->sc_encrypt_last_cpu + 1) % mp_ncpus; in wg_encrypt_dispatch()
1710 sc->sc_encrypt_last_cpu = cpu; in wg_encrypt_dispatch()
1711 GROUPTASK_ENQUEUE(&sc->sc_encrypt[cpu]); in wg_encrypt_dispatch()
1717 u_int cpu = (sc->sc_decrypt_last_cpu + 1) % mp_ncpus; in wg_decrypt_dispatch()
1718 sc->sc_decrypt_last_cpu = cpu; in wg_decrypt_dispatch()
1719 GROUPTASK_ENQUEUE(&sc->sc_decrypt[cpu]); in wg_decrypt_dispatch()
1726 struct wg_softc *sc = peer->p_sc; in wg_deliver_out()
1727 struct wg_packet *pkt; in wg_deliver_out() local
1733 while ((pkt = wg_queue_dequeue_serial(&peer->p_encrypt_serial)) != NULL) { in wg_deliver_out()
1734 if (atomic_load_acq_int(&pkt->p_state) != WG_PACKET_CRYPTED) in wg_deliver_out()
1737 m = pkt->p_mbuf; in wg_deliver_out()
1738 pkt->p_mbuf = NULL; in wg_deliver_out()
1740 len = m->m_pkthdr.len; in wg_deliver_out()
1748 counter_u64_add(peer->p_tx_bytes, len); in wg_deliver_out()
1756 wg_packet_free(pkt); in wg_deliver_out()
1757 if (noise_keep_key_fresh_send(peer->p_remote)) in wg_deliver_out()
1761 if_inc_counter(sc->sc_ifp, IFCOUNTER_OERRORS, 1); in wg_deliver_out()
1762 wg_packet_free(pkt); in wg_deliver_out()
1783 eh->ether_type = af == AF_INET ? in wg_deliver_netmap()
1785 memcpy(eh->ether_shost, "\x02\x02\x02\x02\x02\x02", ETHER_ADDR_LEN); in wg_deliver_netmap()
1786 memcpy(eh->ether_dhost, "\xff\xff\xff\xff\xff\xff", ETHER_ADDR_LEN); in wg_deliver_netmap()
1794 struct wg_softc *sc = peer->p_sc; in wg_deliver_in()
1795 if_t ifp = sc->sc_ifp; in wg_deliver_in()
1796 struct wg_packet *pkt; in wg_deliver_in() local
1801 while ((pkt = wg_queue_dequeue_serial(&peer->p_decrypt_serial)) != NULL) { in wg_deliver_in()
1802 if (atomic_load_acq_int(&pkt->p_state) != WG_PACKET_CRYPTED) in wg_deliver_in()
1805 m = pkt->p_mbuf; in wg_deliver_in()
1806 if (noise_keypair_nonce_check(pkt->p_keypair, pkt->p_nonce) != 0) in wg_deliver_in()
1809 if (noise_keypair_received_with(pkt->p_keypair) == ECONNRESET) in wg_deliver_in()
1814 wg_peer_set_endpoint(peer, &pkt->p_endpoint); in wg_deliver_in()
1816 counter_u64_add(peer->p_rx_bytes, m->m_pkthdr.len + in wg_deliver_in()
1818 if_inc_counter(sc->sc_ifp, IFCOUNTER_IPACKETS, 1); in wg_deliver_in()
1819 if_inc_counter(sc->sc_ifp, IFCOUNTER_IBYTES, m->m_pkthdr.len + in wg_deliver_in()
1822 if (m->m_pkthdr.len == 0) in wg_deliver_in()
1825 af = pkt->p_af; in wg_deliver_in()
1827 pkt->p_mbuf = NULL; in wg_deliver_in()
1829 m->m_pkthdr.rcvif = ifp; in wg_deliver_in()
1851 if (noise_keep_key_fresh_recv(peer->p_remote)) in wg_deliver_in()
1853 wg_packet_free(pkt); in wg_deliver_in()
1857 wg_packet_free(pkt); in wg_deliver_in()
1864 struct wg_packet *pkt; in wg_packet_alloc() local
1866 if ((pkt = uma_zalloc(wg_packet_zone, M_NOWAIT | M_ZERO)) == NULL) in wg_packet_alloc()
1868 pkt->p_mbuf = m; in wg_packet_alloc()
1869 return (pkt); in wg_packet_alloc()
1873 wg_packet_free(struct wg_packet *pkt) in wg_packet_free() argument
1875 if (pkt->p_keypair != NULL) in wg_packet_free()
1876 noise_keypair_put(pkt->p_keypair); in wg_packet_free()
1877 if (pkt->p_mbuf != NULL) in wg_packet_free()
1878 m_freem(pkt->p_mbuf); in wg_packet_free()
1879 uma_zfree(wg_packet_zone, pkt); in wg_packet_free()
1885 mtx_init(&queue->q_mtx, name, NULL, MTX_DEF); in wg_queue_init()
1886 STAILQ_INIT(&queue->q_queue); in wg_queue_init()
1887 queue->q_len = 0; in wg_queue_init()
1894 mtx_destroy(&queue->q_mtx); in wg_queue_deinit()
1900 return (queue->q_len); in wg_queue_len()
1904 wg_queue_enqueue_handshake(struct wg_queue *hs, struct wg_packet *pkt) in wg_queue_enqueue_handshake() argument
1907 mtx_lock(&hs->q_mtx); in wg_queue_enqueue_handshake()
1908 if (hs->q_len < MAX_QUEUED_HANDSHAKES) { in wg_queue_enqueue_handshake()
1909 STAILQ_INSERT_TAIL(&hs->q_queue, pkt, p_parallel); in wg_queue_enqueue_handshake()
1910 hs->q_len++; in wg_queue_enqueue_handshake()
1914 mtx_unlock(&hs->q_mtx); in wg_queue_enqueue_handshake()
1916 wg_packet_free(pkt); in wg_queue_enqueue_handshake()
1923 struct wg_packet *pkt; in wg_queue_dequeue_handshake() local
1924 mtx_lock(&hs->q_mtx); in wg_queue_dequeue_handshake()
1925 if ((pkt = STAILQ_FIRST(&hs->q_queue)) != NULL) { in wg_queue_dequeue_handshake()
1926 STAILQ_REMOVE_HEAD(&hs->q_queue, p_parallel); in wg_queue_dequeue_handshake()
1927 hs->q_len--; in wg_queue_dequeue_handshake()
1929 mtx_unlock(&hs->q_mtx); in wg_queue_dequeue_handshake()
1930 return (pkt); in wg_queue_dequeue_handshake()
1934 wg_queue_push_staged(struct wg_queue *staged, struct wg_packet *pkt) in wg_queue_push_staged() argument
1938 mtx_lock(&staged->q_mtx); in wg_queue_push_staged()
1939 if (staged->q_len >= MAX_STAGED_PKT) { in wg_queue_push_staged()
1940 old = STAILQ_FIRST(&staged->q_queue); in wg_queue_push_staged()
1941 STAILQ_REMOVE_HEAD(&staged->q_queue, p_parallel); in wg_queue_push_staged()
1942 staged->q_len--; in wg_queue_push_staged()
1944 STAILQ_INSERT_TAIL(&staged->q_queue, pkt, p_parallel); in wg_queue_push_staged()
1945 staged->q_len++; in wg_queue_push_staged()
1946 mtx_unlock(&staged->q_mtx); in wg_queue_push_staged()
1955 struct wg_packet *pkt, *tpkt; in wg_queue_enlist_staged() local
1956 STAILQ_FOREACH_SAFE(pkt, list, p_parallel, tpkt) in wg_queue_enlist_staged()
1957 wg_queue_push_staged(staged, pkt); in wg_queue_enlist_staged()
1964 mtx_lock(&staged->q_mtx); in wg_queue_delist_staged()
1965 STAILQ_CONCAT(list, &staged->q_queue); in wg_queue_delist_staged()
1966 staged->q_len = 0; in wg_queue_delist_staged()
1967 mtx_unlock(&staged->q_mtx); in wg_queue_delist_staged()
1974 struct wg_packet *pkt, *tpkt; in wg_queue_purge() local
1976 STAILQ_FOREACH_SAFE(pkt, &list, p_parallel, tpkt) in wg_queue_purge()
1977 wg_packet_free(pkt); in wg_queue_purge()
1981 wg_queue_both(struct wg_queue *parallel, struct wg_queue *serial, struct wg_packet *pkt) in wg_queue_both() argument
1983 pkt->p_state = WG_PACKET_UNCRYPTED; in wg_queue_both()
1985 mtx_lock(&serial->q_mtx); in wg_queue_both()
1986 if (serial->q_len < MAX_QUEUED_PKT) { in wg_queue_both()
1987 serial->q_len++; in wg_queue_both()
1988 STAILQ_INSERT_TAIL(&serial->q_queue, pkt, p_serial); in wg_queue_both()
1990 mtx_unlock(&serial->q_mtx); in wg_queue_both()
1991 wg_packet_free(pkt); in wg_queue_both()
1994 mtx_unlock(&serial->q_mtx); in wg_queue_both()
1996 mtx_lock(¶llel->q_mtx); in wg_queue_both()
1997 if (parallel->q_len < MAX_QUEUED_PKT) { in wg_queue_both()
1998 parallel->q_len++; in wg_queue_both()
1999 STAILQ_INSERT_TAIL(¶llel->q_queue, pkt, p_parallel); in wg_queue_both()
2001 mtx_unlock(¶llel->q_mtx); in wg_queue_both()
2002 pkt->p_state = WG_PACKET_DEAD; in wg_queue_both()
2005 mtx_unlock(¶llel->q_mtx); in wg_queue_both()
2013 struct wg_packet *pkt = NULL; in wg_queue_dequeue_serial() local
2014 mtx_lock(&serial->q_mtx); in wg_queue_dequeue_serial()
2015 if (serial->q_len > 0 && STAILQ_FIRST(&serial->q_queue)->p_state != WG_PACKET_UNCRYPTED) { in wg_queue_dequeue_serial()
2016 serial->q_len--; in wg_queue_dequeue_serial()
2017 pkt = STAILQ_FIRST(&serial->q_queue); in wg_queue_dequeue_serial()
2018 STAILQ_REMOVE_HEAD(&serial->q_queue, p_serial); in wg_queue_dequeue_serial()
2020 mtx_unlock(&serial->q_mtx); in wg_queue_dequeue_serial()
2021 return (pkt); in wg_queue_dequeue_serial()
2027 struct wg_packet *pkt = NULL; in wg_queue_dequeue_parallel() local
2028 mtx_lock(¶llel->q_mtx); in wg_queue_dequeue_parallel()
2029 if (parallel->q_len > 0) { in wg_queue_dequeue_parallel()
2030 parallel->q_len--; in wg_queue_dequeue_parallel()
2031 pkt = STAILQ_FIRST(¶llel->q_queue); in wg_queue_dequeue_parallel()
2032 STAILQ_REMOVE_HEAD(¶llel->q_queue, p_parallel); in wg_queue_dequeue_parallel()
2034 mtx_unlock(¶llel->q_mtx); in wg_queue_dequeue_parallel()
2035 return (pkt); in wg_queue_dequeue_parallel()
2050 struct wg_packet *pkt; in wg_input() local
2060 if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1); in wg_input()
2069 if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1); in wg_input()
2073 if ((pkt = wg_packet_alloc(m)) == NULL) { in wg_input()
2074 if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1); in wg_input()
2080 switch (sa->sa_family) { in wg_input()
2084 pkt->p_endpoint.e_remote.r_sin = sin[0]; in wg_input()
2085 pkt->p_endpoint.e_local.l_in = sin[1].sin_addr; in wg_input()
2091 pkt->p_endpoint.e_remote.r_sin6 = sin6[0]; in wg_input()
2092 pkt->p_endpoint.e_local.l_in6 = sin6[1].sin6_addr; in wg_input()
2099 if ((m->m_pkthdr.len == sizeof(struct wg_pkt_initiation) && in wg_input()
2101 (m->m_pkthdr.len == sizeof(struct wg_pkt_response) && in wg_input()
2103 (m->m_pkthdr.len == sizeof(struct wg_pkt_cookie) && in wg_input()
2106 if (wg_queue_enqueue_handshake(&sc->sc_handshake_queue, pkt) != 0) { in wg_input()
2107 if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1); in wg_input()
2110 GROUPTASK_ENQUEUE(&sc->sc_handshake); in wg_input()
2111 } else if (m->m_pkthdr.len >= sizeof(struct wg_pkt_data) + in wg_input()
2115 if ((pkt->p_mbuf = m_pullup(m, sizeof(struct wg_pkt_data))) == NULL) in wg_input()
2118 data = mtod(pkt->p_mbuf, struct wg_pkt_data *); in wg_input()
2119 if ((pkt->p_keypair = noise_keypair_lookup(sc->sc_local, data->r_idx)) == NULL) in wg_input()
2122 remote = noise_keypair_remote(pkt->p_keypair); in wg_input()
2124 if (wg_queue_both(&sc->sc_decrypt_parallel, &peer->p_decrypt_serial, pkt) != 0) in wg_input()
2125 if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1); in wg_input()
2133 if_inc_counter(sc->sc_ifp, IFCOUNTER_IERRORS, 1); in wg_input()
2134 wg_packet_free(pkt); in wg_input()
2143 struct wg_packet *pkt, *tpkt; in wg_peer_send_staged() local
2144 struct wg_softc *sc = peer->p_sc; in wg_peer_send_staged()
2146 wg_queue_delist_staged(&peer->p_stage_queue, &list); in wg_peer_send_staged()
2151 if ((keypair = noise_keypair_current(peer->p_remote)) == NULL) in wg_peer_send_staged()
2154 STAILQ_FOREACH(pkt, &list, p_parallel) { in wg_peer_send_staged()
2155 if (noise_keypair_nonce_next(keypair, &pkt->p_nonce) != 0) in wg_peer_send_staged()
2158 STAILQ_FOREACH_SAFE(pkt, &list, p_parallel, tpkt) { in wg_peer_send_staged()
2159 pkt->p_keypair = noise_keypair_ref(keypair); in wg_peer_send_staged()
2160 if (wg_queue_both(&sc->sc_encrypt_parallel, &peer->p_encrypt_serial, pkt) != 0) in wg_peer_send_staged()
2161 if_inc_counter(sc->sc_ifp, IFCOUNTER_OQDROPS, 1); in wg_peer_send_staged()
2170 wg_queue_enlist_staged(&peer->p_stage_queue, &list); in wg_peer_send_staged()
2175 xmit_err(if_t ifp, struct mbuf *m, struct wg_packet *pkt, sa_family_t af) in xmit_err() argument
2182 if (pkt) in xmit_err()
2183 pkt->p_mbuf = NULL; in xmit_err()
2190 if (pkt) in xmit_err()
2191 pkt->p_mbuf = NULL; in xmit_err()
2196 if (pkt) in xmit_err()
2197 wg_packet_free(pkt); in xmit_err()
2205 struct wg_packet *pkt = NULL; in wg_xmit() local
2217 if ((pkt = wg_packet_alloc(m)) == NULL) { in wg_xmit()
2221 pkt->p_mtu = mtu; in wg_xmit()
2222 pkt->p_af = af; in wg_xmit()
2225 peer = wg_aip_lookup(sc, AF_INET, &mtod(m, struct ip *)->ip_dst); in wg_xmit()
2227 peer = wg_aip_lookup(sc, AF_INET6, &mtod(m, struct ip6_hdr *)->ip6_dst); in wg_xmit()
2233 BPF_MTAP2_AF(ifp, m, pkt->p_af); in wg_xmit()
2246 peer_af = peer->p_endpoint.e_remote.r_sa.sa_family; in wg_xmit()
2249 "discovered for peer %" PRIu64 "\n", peer->p_id); in wg_xmit()
2254 wg_queue_push_staged(&peer->p_stage_queue, pkt); in wg_xmit()
2256 noise_remote_put(peer->p_remote); in wg_xmit()
2260 noise_remote_put(peer->p_remote); in wg_xmit()
2262 xmit_err(ifp, m, pkt, af); in wg_xmit()
2270 if ((*m)->m_pkthdr.len >= sizeof(struct ip6_hdr)) in determine_af_and_pullup()
2272 else if ((*m)->m_pkthdr.len >= sizeof(struct ip)) in determine_af_and_pullup()
2278 ipv = mtod(*m, struct ip *)->ip_v; in determine_af_and_pullup()
2281 else if (ipv == 6 && (*m)->m_pkthdr.len >= sizeof(struct ip6_hdr)) in determine_af_and_pullup()
2297 *etp = ntohs(eh->ether_type); in determine_ethertype_and_pullup()
2407 eh->ether_type = af == AF_INET ? in wg_xmit_netmap()
2409 memcpy(eh->ether_shost, "\x06\x06\x06\x06\x06\x06", ETHER_ADDR_LEN); in wg_xmit_netmap()
2410 memcpy(eh->ether_dhost, "\xff\xff\xff\xff\xff\xff", ETHER_ADDR_LEN); in wg_xmit_netmap()
2424 if (dst->sa_family == AF_UNSPEC || dst->sa_family == pseudo_AF_HDRCMPLT) in wg_output()
2425 memcpy(&af, dst->sa_data, sizeof(af)); in wg_output()
2454 mtu = (ro != NULL && ro->ro_mtu > 0) ? ro->ro_mtu : if_getmtu(ifp); in wg_output()
2465 size_t size; in wg_peer_add() local
2470 sx_assert(&sc->sc_lock, SX_XLOCKED); in wg_peer_add()
2472 if (!nvlist_exists_binary(nvl, "public-key")) { in wg_peer_add()
2475 pub_key = nvlist_get_binary(nvl, "public-key", &size); in wg_peer_add()
2476 if (size != WG_KEY_SIZE) { in wg_peer_add()
2479 if (noise_local_keys(sc->sc_local, public, NULL) == 0 && in wg_peer_add()
2483 if ((remote = noise_remote_lookup(sc->sc_local, pub_key)) != NULL) in wg_peer_add()
2493 if (nvlist_exists_bool(nvl, "replace-allowedips") && in wg_peer_add()
2494 nvlist_get_bool(nvl, "replace-allowedips") && in wg_peer_add()
2506 endpoint = nvlist_get_binary(nvl, "endpoint", &size); in wg_peer_add()
2507 if (size > sizeof(peer->p_endpoint.e_remote)) { in wg_peer_add()
2511 memcpy(&peer->p_endpoint.e_remote, endpoint, size); in wg_peer_add()
2513 if (nvlist_exists_binary(nvl, "preshared-key")) { in wg_peer_add()
2514 preshared_key = nvlist_get_binary(nvl, "preshared-key", &size); in wg_peer_add()
2515 if (size != WG_KEY_SIZE) { in wg_peer_add()
2519 noise_remote_set_psk(peer->p_remote, preshared_key); in wg_peer_add()
2521 if (nvlist_exists_number(nvl, "persistent-keepalive-interval")) { in wg_peer_add()
2522 uint64_t pki = nvlist_get_number(nvl, "persistent-keepalive-interval"); in wg_peer_add()
2529 if (nvlist_exists_nvlist_array(nvl, "allowed-ips")) { in wg_peer_add()
2535 aipl = nvlist_get_nvlist_array(nvl, "allowed-ips", &allowedip_count); in wg_peer_add()
2553 addr = nvlist_get_binary(aipl[idx], "ipv4", &size); in wg_peer_add()
2554 if (addr == NULL || cidr > 32 || size != sizeof(struct in_addr)) { in wg_peer_add()
2561 addr = nvlist_get_binary(aipl[idx], "ipv6", &size); in wg_peer_add()
2562 if (addr == NULL || cidr > 128 || size != sizeof(struct in6_addr)) { in wg_peer_add()
2601 ssize_t size; in wgc_set() local
2604 ifp = sc->sc_ifp; in wgc_set()
2605 if (wgd->wgd_size == 0 || wgd->wgd_data == NULL) in wgc_set()
2610 if (wgd->wgd_size >= UINT32_MAX / 2) in wgc_set()
2613 nvlpacked = malloc(wgd->wgd_size, M_TEMP, M_WAITOK | M_ZERO); in wgc_set()
2615 err = copyin(wgd->wgd_data, nvlpacked, wgd->wgd_size); in wgc_set()
2618 nvl = nvlist_unpack(nvlpacked, wgd->wgd_size, 0); in wgc_set()
2623 sx_xlock(&sc->sc_lock); in wgc_set()
2624 if (nvlist_exists_bool(nvl, "replace-peers") && in wgc_set()
2625 nvlist_get_bool(nvl, "replace-peers")) in wgc_set()
2627 if (nvlist_exists_number(nvl, "listen-port")) { in wgc_set()
2628 uint64_t new_port = nvlist_get_number(nvl, "listen-port"); in wgc_set()
2633 if (new_port != sc->sc_socket.so_port) { in wgc_set()
2638 sc->sc_socket.so_port = new_port; in wgc_set()
2641 if (nvlist_exists_binary(nvl, "private-key")) { in wgc_set()
2642 const void *key = nvlist_get_binary(nvl, "private-key", &size); in wgc_set()
2643 if (size != WG_KEY_SIZE) { in wgc_set()
2648 if (noise_local_keys(sc->sc_local, NULL, private) != 0 || in wgc_set()
2655 if ((remote = noise_remote_lookup(sc->sc_local, in wgc_set()
2668 noise_local_private(sc->sc_local, key); in wgc_set()
2669 if (noise_local_keys(sc->sc_local, NULL, NULL) == 0) in wgc_set()
2670 cookie_checker_update(&sc->sc_cookie, public); in wgc_set()
2672 cookie_checker_update(&sc->sc_cookie, NULL); in wgc_set()
2675 if (nvlist_exists_number(nvl, "user-cookie")) { in wgc_set()
2676 uint64_t user_cookie = nvlist_get_number(nvl, "user-cookie"); in wgc_set()
2698 sx_xunlock(&sc->sc_lock); in wgc_set()
2712 size_t size, peer_count, aip_count, i, j; in wgc_get() local
2723 sx_slock(&sc->sc_lock); in wgc_get()
2725 if (sc->sc_socket.so_port != 0) in wgc_get()
2726 nvlist_add_number(nvl, "listen-port", sc->sc_socket.so_port); in wgc_get()
2727 if (sc->sc_socket.so_user_cookie != 0) in wgc_get()
2728 nvlist_add_number(nvl, "user-cookie", sc->sc_socket.so_user_cookie); in wgc_get()
2729 if (noise_local_keys(sc->sc_local, public_key, private_key) == 0) { in wgc_get()
2730 nvlist_add_binary(nvl, "public-key", public_key, WG_KEY_SIZE); in wgc_get()
2732 nvlist_add_binary(nvl, "private-key", private_key, WG_KEY_SIZE); in wgc_get()
2735 peer_count = sc->sc_peers_num; in wgc_get()
2739 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) { in wgc_get()
2749 (void)noise_remote_keys(peer->p_remote, public_key, preshared_key); in wgc_get()
2750 nvlist_add_binary(nvl_peer, "public-key", public_key, sizeof(public_key)); in wgc_get()
2752 nvlist_add_binary(nvl_peer, "preshared-key", preshared_key, sizeof(preshared_key)); in wgc_get()
2754 if (peer->p_endpoint.e_remote.r_sa.sa_family == AF_INET) in wgc_get()
2755 nvlist_add_binary(nvl_peer, "endpoint", &peer->p_endpoint.e_remote, sizeof(struct sockaddr_in)); in wgc_get()
2756 else if (peer->p_endpoint.e_remote.r_sa.sa_family == AF_INET6) in wgc_get()
2757 … nvlist_add_binary(nvl_peer, "endpoint", &peer->p_endpoint.e_remote, sizeof(struct sockaddr_in6)); in wgc_get()
2759 nvlist_add_binary(nvl_peer, "last-handshake-time", &ts64, sizeof(ts64)); in wgc_get()
2760 …nvlist_add_number(nvl_peer, "persistent-keepalive-interval", peer->p_persistent_keepalive_interval… in wgc_get()
2761 nvlist_add_number(nvl_peer, "rx-bytes", counter_u64_fetch(peer->p_rx_bytes)); in wgc_get()
2762 nvlist_add_number(nvl_peer, "tx-bytes", counter_u64_fetch(peer->p_tx_bytes)); in wgc_get()
2764 aip_count = peer->p_aips_num; in wgc_get()
2768 LIST_FOREACH(aip, &peer->p_aips, a_entry) { in wgc_get()
2777 if (aip->a_af == AF_INET) { in wgc_get()
2778 nvlist_add_binary(nvl_aip, "ipv4", &aip->a_addr.in, sizeof(aip->a_addr.in)); in wgc_get()
2779 nvlist_add_number(nvl_aip, "cidr", bitcount32(aip->a_mask.ip)); in wgc_get()
2782 else if (aip->a_af == AF_INET6) { in wgc_get()
2783 nvlist_add_binary(nvl_aip, "ipv6", &aip->a_addr.in6, sizeof(aip->a_addr.in6)); in wgc_get()
2784 nvlist_add_number(nvl_aip, "cidr", in6_mask2len(&aip->a_mask.in6, NULL)); in wgc_get()
2788 nvlist_add_nvlist_array(nvl_peer, "allowed-ips", (const nvlist_t *const *)nvl_aips, aip_count); in wgc_get()
2803 sx_sunlock(&sc->sc_lock); in wgc_get()
2807 sx_sunlock(&sc->sc_lock); in wgc_get()
2808 packed = nvlist_pack(nvl, &size); in wgc_get()
2813 if (!wgd->wgd_size) { in wgc_get()
2814 wgd->wgd_size = size; in wgc_get()
2817 if (wgd->wgd_size < size) { in wgc_get()
2821 err = copyout(packed, wgd->wgd_data, size); in wgc_get()
2822 wgd->wgd_size = size; in wgc_get()
2869 if (ifr->ifr_mtu <= 0 || ifr->ifr_mtu > MAX_MTU) in wg_ioctl()
2872 if_setmtu(ifp, ifr->ifr_mtu); in wg_ioctl()
2878 ifr->ifr_fib = sc->sc_socket.so_fibnum; in wg_ioctl()
2887 sx_xlock(&sc->sc_lock); in wg_ioctl()
2888 ret = wg_socket_set_fibnum(sc, ifr->ifr_fib); in wg_ioctl()
2889 sx_xunlock(&sc->sc_lock); in wg_ioctl()
2903 if_t ifp = sc->sc_ifp; in wg_up()
2907 sx_xlock(&sc->sc_lock); in wg_up()
2909 if ((sc->sc_flags & WGF_DYING) != 0) in wg_up()
2918 rc = wg_socket_init(sc, sc->sc_socket.so_port); in wg_up()
2920 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) in wg_up()
2922 if_link_state_change(sc->sc_ifp, LINK_STATE_UP); in wg_up()
2928 sx_xunlock(&sc->sc_lock); in wg_up()
2935 if_t ifp = sc->sc_ifp; in wg_down()
2938 sx_xlock(&sc->sc_lock); in wg_down()
2940 sx_xunlock(&sc->sc_lock); in wg_down()
2945 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) { in wg_down()
2946 wg_queue_purge(&peer->p_stage_queue); in wg_down()
2950 wg_queue_purge(&sc->sc_handshake_queue); in wg_down()
2952 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) { in wg_down()
2953 noise_remote_handshake_clear(peer->p_remote); in wg_down()
2954 noise_remote_keypairs_clear(peer->p_remote); in wg_down()
2957 if_link_state_change(sc->sc_ifp, LINK_STATE_DOWN); in wg_down()
2960 sx_xunlock(&sc->sc_lock); in wg_down()
2972 sc->sc_local = noise_local_alloc(sc); in wg_clone_create()
2974 sc->sc_encrypt = mallocarray(sizeof(struct grouptask), mp_ncpus, M_WG, M_WAITOK | M_ZERO); in wg_clone_create()
2976 sc->sc_decrypt = mallocarray(sizeof(struct grouptask), mp_ncpus, M_WG, M_WAITOK | M_ZERO); in wg_clone_create()
2978 if (!rn_inithead((void **)&sc->sc_aip4, offsetof(struct aip_addr, in) * NBBY)) in wg_clone_create()
2981 if (!rn_inithead((void **)&sc->sc_aip6, offsetof(struct aip_addr, in6) * NBBY)) in wg_clone_create()
2985 ifp = sc->sc_ifp = if_alloc(IFT_WIREGUARD); in wg_clone_create()
2987 sc->sc_ucred = crhold(curthread->td_ucred); in wg_clone_create()
2988 sc->sc_socket.so_fibnum = curthread->td_proc->p_fibnum; in wg_clone_create()
2989 sc->sc_socket.so_port = 0; in wg_clone_create()
2991 TAILQ_INIT(&sc->sc_peers); in wg_clone_create()
2992 sc->sc_peers_num = 0; in wg_clone_create()
2994 cookie_checker_init(&sc->sc_cookie); in wg_clone_create()
2996 RADIX_NODE_HEAD_LOCK_INIT(sc->sc_aip4); in wg_clone_create()
2997 RADIX_NODE_HEAD_LOCK_INIT(sc->sc_aip6); in wg_clone_create()
2999 GROUPTASK_INIT(&sc->sc_handshake, 0, (gtask_fn_t *)wg_softc_handshake_receive, sc); in wg_clone_create()
3000 taskqgroup_attach(qgroup_wg_tqg, &sc->sc_handshake, sc, NULL, NULL, "wg tx initiation"); in wg_clone_create()
3001 wg_queue_init(&sc->sc_handshake_queue, "hsq"); in wg_clone_create()
3004 GROUPTASK_INIT(&sc->sc_encrypt[i], 0, in wg_clone_create()
3006 taskqgroup_attach_cpu(qgroup_wg_tqg, &sc->sc_encrypt[i], sc, i, NULL, NULL, "wg encrypt"); in wg_clone_create()
3007 GROUPTASK_INIT(&sc->sc_decrypt[i], 0, in wg_clone_create()
3009 taskqgroup_attach_cpu(qgroup_wg_tqg, &sc->sc_decrypt[i], sc, i, NULL, NULL, "wg decrypt"); in wg_clone_create()
3012 wg_queue_init(&sc->sc_encrypt_parallel, "encp"); in wg_clone_create()
3013 wg_queue_init(&sc->sc_decrypt_parallel, "decp"); in wg_clone_create()
3015 sx_init(&sc->sc_lock, "wg softc lock"); in wg_clone_create()
3020 if_initname(ifp, wgname, ifd->unit); in wg_clone_create()
3036 ND_IFINFO(ifp)->flags &= ~ND6_IFF_AUTO_LINKLOCAL; in wg_clone_create()
3037 ND_IFINFO(ifp)->flags |= ND6_IFF_NO_DAD; in wg_clone_create()
3045 RADIX_NODE_HEAD_DESTROY(sc->sc_aip4); in wg_clone_create()
3046 free(sc->sc_aip4, M_RTABLE); in wg_clone_create()
3048 free(sc->sc_decrypt, M_WG); in wg_clone_create()
3049 free(sc->sc_encrypt, M_WG); in wg_clone_create()
3050 noise_local_free(sc->sc_local, NULL); in wg_clone_create()
3061 atomic_add_int(&clone_count, -1); in wg_clone_deferred_free()
3072 sx_xlock(&sc->sc_lock); in wg_clone_destroy()
3073 sc->sc_flags |= WGF_DYING; in wg_clone_destroy()
3074 cred = sc->sc_ucred; in wg_clone_destroy()
3075 sc->sc_ucred = NULL; in wg_clone_destroy()
3076 sx_xunlock(&sc->sc_lock); in wg_clone_destroy()
3080 if_link_state_change(sc->sc_ifp, LINK_STATE_DOWN); in wg_clone_destroy()
3081 CURVNET_SET(if_getvnet(sc->sc_ifp)); in wg_clone_destroy()
3082 if_purgeaddrs(sc->sc_ifp); in wg_clone_destroy()
3085 sx_xlock(&sc->sc_lock); in wg_clone_destroy()
3087 sx_xunlock(&sc->sc_lock); in wg_clone_destroy()
3096 sx_xlock(&sc->sc_lock); in wg_clone_destroy()
3099 sx_xunlock(&sc->sc_lock); in wg_clone_destroy()
3100 sx_destroy(&sc->sc_lock); in wg_clone_destroy()
3101 taskqgroup_detach(qgroup_wg_tqg, &sc->sc_handshake); in wg_clone_destroy()
3103 taskqgroup_detach(qgroup_wg_tqg, &sc->sc_encrypt[i]); in wg_clone_destroy()
3104 taskqgroup_detach(qgroup_wg_tqg, &sc->sc_decrypt[i]); in wg_clone_destroy()
3106 free(sc->sc_encrypt, M_WG); in wg_clone_destroy()
3107 free(sc->sc_decrypt, M_WG); in wg_clone_destroy()
3108 wg_queue_deinit(&sc->sc_handshake_queue); in wg_clone_destroy()
3109 wg_queue_deinit(&sc->sc_encrypt_parallel); in wg_clone_destroy()
3110 wg_queue_deinit(&sc->sc_decrypt_parallel); in wg_clone_destroy()
3112 RADIX_NODE_HEAD_DESTROY(sc->sc_aip4); in wg_clone_destroy()
3113 RADIX_NODE_HEAD_DESTROY(sc->sc_aip6); in wg_clone_destroy()
3114 rn_detachhead((void **)&sc->sc_aip4); in wg_clone_destroy()
3115 rn_detachhead((void **)&sc->sc_aip6); in wg_clone_destroy()
3117 cookie_checker_free(&sc->sc_cookie); in wg_clone_destroy()
3121 bpfdetach(sc->sc_ifp); in wg_clone_destroy()
3122 if_detach(sc->sc_ifp); in wg_clone_destroy()
3123 if_free(sc->sc_ifp); in wg_clone_destroy()
3125 noise_local_free(sc->sc_local, wg_clone_deferred_free); in wg_clone_destroy()
3136 * Privileged information (private-key, preshared-key) are only exported for
3202 sx_xlock(&sc->sc_lock); in wg_prison_remove()
3203 if (!(sc->sc_flags & WGF_DYING) && sc->sc_ucred && sc->sc_ucred->cr_prison == pr) { in wg_prison_remove()
3204 struct ucred *cred = sc->sc_ucred; in wg_prison_remove()
3206 if_link_state_change(sc->sc_ifp, LINK_STATE_DOWN); in wg_prison_remove()
3208 sc->sc_ucred = NULL; in wg_prison_remove()
3210 sc->sc_flags |= WGF_DYING; in wg_prison_remove()
3212 sx_xunlock(&sc->sc_lock); in wg_prison_remove()