Lines Matching +full:ring +full:- +full:disable +full:- +full:pullup
1 /* SPDX-License-Identifier: ISC
3 * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
4 * Copyright (C) 2019-2021 Matt Dunwoodie <ncon@noconroy.net>
5 * Copyright (c) 2019-2020 Rubicon Communications, LLC (Netgate)
56 #define DEFAULT_MTU (ETHERMTU - 80)
57 #define MAX_MTU (IF_MAXMTU - 80)
61 #define MAX_QUEUED_PKT_MASK (MAX_QUEUED_PKT - 1)
70 #define DPRINTF(sc, ...) if (if_getflags(sc->sc_ifp) & IFF_DEBUG) if_printf(sc->sc_ifp, ##__VA_ARGS…
256 gtaskqueue_drain((gtask)->gt_taskqueue, &(gtask)->gt_task)
387 sx_assert(&sc->sc_lock, SX_XLOCKED); in wg_peer_create()
391 peer->p_remote = noise_remote_alloc(sc->sc_local, peer, pub_key); in wg_peer_create()
392 if ((*errp = noise_remote_enable(peer->p_remote)) != 0) { in wg_peer_create()
393 noise_remote_free(peer->p_remote, NULL); in wg_peer_create()
398 peer->p_id = peer_counter++; in wg_peer_create()
399 peer->p_sc = sc; in wg_peer_create()
400 peer->p_tx_bytes = counter_u64_alloc(M_WAITOK); in wg_peer_create()
401 peer->p_rx_bytes = counter_u64_alloc(M_WAITOK); in wg_peer_create()
403 cookie_maker_init(&peer->p_cookie, pub_key); in wg_peer_create()
405 rw_init(&peer->p_endpoint_lock, "wg_peer_endpoint"); in wg_peer_create()
407 wg_queue_init(&peer->p_stage_queue, "stageq"); in wg_peer_create()
408 wg_queue_init(&peer->p_encrypt_serial, "txq"); in wg_peer_create()
409 wg_queue_init(&peer->p_decrypt_serial, "rxq"); in wg_peer_create()
411 peer->p_enabled = false; in wg_peer_create()
412 peer->p_need_another_keepalive = false; in wg_peer_create()
413 peer->p_persistent_keepalive_interval = 0; in wg_peer_create()
414 callout_init(&peer->p_new_handshake, true); in wg_peer_create()
415 callout_init(&peer->p_send_keepalive, true); in wg_peer_create()
416 callout_init(&peer->p_retry_handshake, true); in wg_peer_create()
417 callout_init(&peer->p_persistent_keepalive, true); in wg_peer_create()
418 callout_init(&peer->p_zero_key_material, true); in wg_peer_create()
420 mtx_init(&peer->p_handshake_mtx, "peer handshake", NULL, MTX_DEF); in wg_peer_create()
421 bzero(&peer->p_handshake_complete, sizeof(peer->p_handshake_complete)); in wg_peer_create()
422 peer->p_handshake_retries = 0; in wg_peer_create()
424 GROUPTASK_INIT(&peer->p_send, 0, (gtask_fn_t *)wg_deliver_out, peer); in wg_peer_create()
425 taskqgroup_attach(qgroup_wg_tqg, &peer->p_send, peer, NULL, NULL, "wg send"); in wg_peer_create()
426 GROUPTASK_INIT(&peer->p_recv, 0, (gtask_fn_t *)wg_deliver_in, peer); in wg_peer_create()
427 taskqgroup_attach(qgroup_wg_tqg, &peer->p_recv, peer, NULL, NULL, "wg recv"); in wg_peer_create()
429 LIST_INIT(&peer->p_aips); in wg_peer_create()
430 peer->p_aips_num = 0; in wg_peer_create()
432 TAILQ_INSERT_TAIL(&sc->sc_peers, peer, p_entry); in wg_peer_create()
433 sc->sc_peers_num++; in wg_peer_create()
435 if (if_getlinkstate(sc->sc_ifp) == LINK_STATE_UP) in wg_peer_create()
438 DPRINTF(sc, "Peer %" PRIu64 " created\n", peer->p_id); in wg_peer_create()
450 GROUPTASK_DRAIN(&peer->p_recv); in wg_peer_free_deferred()
451 GROUPTASK_DRAIN(&peer->p_send); in wg_peer_free_deferred()
452 taskqgroup_detach(qgroup_wg_tqg, &peer->p_recv); in wg_peer_free_deferred()
453 taskqgroup_detach(qgroup_wg_tqg, &peer->p_send); in wg_peer_free_deferred()
455 wg_queue_deinit(&peer->p_decrypt_serial); in wg_peer_free_deferred()
456 wg_queue_deinit(&peer->p_encrypt_serial); in wg_peer_free_deferred()
457 wg_queue_deinit(&peer->p_stage_queue); in wg_peer_free_deferred()
459 counter_u64_free(peer->p_tx_bytes); in wg_peer_free_deferred()
460 counter_u64_free(peer->p_rx_bytes); in wg_peer_free_deferred()
461 rw_destroy(&peer->p_endpoint_lock); in wg_peer_free_deferred()
462 mtx_destroy(&peer->p_handshake_mtx); in wg_peer_free_deferred()
464 cookie_maker_free(&peer->p_cookie); in wg_peer_free_deferred()
472 struct wg_softc *sc = peer->p_sc; in wg_peer_destroy()
473 sx_assert(&sc->sc_lock, SX_XLOCKED); in wg_peer_destroy()
475 /* Disable remote and timers. This will prevent any new handshakes in wg_peer_destroy()
477 noise_remote_disable(peer->p_remote); in wg_peer_destroy()
487 sc->sc_peers_num--; in wg_peer_destroy()
488 TAILQ_REMOVE(&sc->sc_peers, peer, p_entry); in wg_peer_destroy()
489 DPRINTF(sc, "Peer %" PRIu64 " destroyed\n", peer->p_id); in wg_peer_destroy()
490 noise_remote_free(peer->p_remote, wg_peer_free_deferred); in wg_peer_destroy()
497 TAILQ_FOREACH_SAFE(peer, &sc->sc_peers, p_entry, tpeer) in wg_peer_destroy_all()
504 MPASS(e->e_remote.r_sa.sa_family != 0); in wg_peer_set_endpoint()
505 if (memcmp(e, &peer->p_endpoint, sizeof(*e)) == 0) in wg_peer_set_endpoint()
508 rw_wlock(&peer->p_endpoint_lock); in wg_peer_set_endpoint()
509 peer->p_endpoint = *e; in wg_peer_set_endpoint()
510 rw_wunlock(&peer->p_endpoint_lock); in wg_peer_set_endpoint()
516 rw_wlock(&peer->p_endpoint_lock); in wg_peer_clear_src()
517 bzero(&peer->p_endpoint.e_local, sizeof(peer->p_endpoint.e_local)); in wg_peer_clear_src()
518 rw_wunlock(&peer->p_endpoint_lock); in wg_peer_clear_src()
524 rw_rlock(&peer->p_endpoint_lock); in wg_peer_get_endpoint()
525 *e = peer->p_endpoint; in wg_peer_get_endpoint()
526 rw_runlock(&peer->p_endpoint_lock); in wg_peer_get_endpoint()
539 aip->a_peer = peer; in wg_aip_add()
540 aip->a_af = af; in wg_aip_add()
546 root = sc->sc_aip4; in wg_aip_add()
547 aip->a_addr.in = *(const struct in_addr *)addr; in wg_aip_add()
548 aip->a_mask.ip = htonl(~((1LL << (32 - cidr)) - 1) & 0xffffffff); in wg_aip_add()
549 aip->a_addr.ip &= aip->a_mask.ip; in wg_aip_add()
550 aip->a_addr.length = aip->a_mask.length = offsetof(struct aip_addr, in) + sizeof(struct in_addr); in wg_aip_add()
556 root = sc->sc_aip6; in wg_aip_add()
557 aip->a_addr.in6 = *(const struct in6_addr *)addr; in wg_aip_add()
558 in6_prefixlen2mask(&aip->a_mask.in6, cidr); in wg_aip_add()
560 aip->a_addr.ip6[i] &= aip->a_mask.ip6[i]; in wg_aip_add()
561 …aip->a_addr.length = aip->a_mask.length = offsetof(struct aip_addr, in6) + sizeof(struct in6_addr); in wg_aip_add()
570 node = root->rnh_addaddr(&aip->a_addr, &aip->a_mask, &root->rh, aip->a_nodes); in wg_aip_add()
571 if (node == aip->a_nodes) { in wg_aip_add()
572 LIST_INSERT_HEAD(&peer->p_aips, aip, a_entry); in wg_aip_add()
573 peer->p_aips_num++; in wg_aip_add()
575 node = root->rnh_lookup(&aip->a_addr, &aip->a_mask, &root->rh); in wg_aip_add()
579 } else if (node != aip->a_nodes) { in wg_aip_add()
582 if (aip->a_peer != peer) { in wg_aip_add()
584 aip->a_peer->p_aips_num--; in wg_aip_add()
585 aip->a_peer = peer; in wg_aip_add()
586 LIST_INSERT_HEAD(&peer->p_aips, aip, a_entry); in wg_aip_add()
587 aip->a_peer->p_aips_num++; in wg_aip_add()
605 root = sc->sc_aip4; in wg_aip_lookup()
610 root = sc->sc_aip6; in wg_aip_lookup()
619 node = root->rnh_matchaddr(&addr, &root->rh); in wg_aip_lookup()
621 peer = ((struct wg_aip *)node)->a_peer; in wg_aip_lookup()
622 noise_remote_ref(peer->p_remote); in wg_aip_lookup()
636 RADIX_NODE_HEAD_LOCK(sc->sc_aip4); in wg_aip_remove_all()
637 LIST_FOREACH_SAFE(aip, &peer->p_aips, a_entry, taip) { in wg_aip_remove_all()
638 if (aip->a_af == AF_INET) { in wg_aip_remove_all()
639 if (sc->sc_aip4->rnh_deladdr(&aip->a_addr, &aip->a_mask, &sc->sc_aip4->rh) == NULL) in wg_aip_remove_all()
642 peer->p_aips_num--; in wg_aip_remove_all()
646 RADIX_NODE_HEAD_UNLOCK(sc->sc_aip4); in wg_aip_remove_all()
648 RADIX_NODE_HEAD_LOCK(sc->sc_aip6); in wg_aip_remove_all()
649 LIST_FOREACH_SAFE(aip, &peer->p_aips, a_entry, taip) { in wg_aip_remove_all()
650 if (aip->a_af == AF_INET6) { in wg_aip_remove_all()
651 if (sc->sc_aip6->rnh_deladdr(&aip->a_addr, &aip->a_mask, &sc->sc_aip6->rh) == NULL) in wg_aip_remove_all()
654 peer->p_aips_num--; in wg_aip_remove_all()
658 RADIX_NODE_HEAD_UNLOCK(sc->sc_aip6); in wg_aip_remove_all()
660 if (!LIST_EMPTY(&peer->p_aips) || peer->p_aips_num != 0) in wg_aip_remove_all()
667 struct ucred *cred = sc->sc_ucred; in wg_socket_init()
671 sx_assert(&sc->sc_lock, SX_XLOCKED); in wg_socket_init()
679 * WireGuard has on Linux with network namespaces -- that the sockets in wg_socket_init()
705 if (sc->sc_socket.so_user_cookie) { in wg_socket_init()
706 …rc = wg_socket_set_sockopt(so4, so6, SO_USER_COOKIE, &sc->sc_socket.so_user_cookie, sizeof(sc->sc_… in wg_socket_init()
710 …rc = wg_socket_set_sockopt(so4, so6, SO_SETFIB, &sc->sc_socket.so_fibnum, sizeof(sc->sc_socket.so_… in wg_socket_init()
716 sc->sc_socket.so_port = port; in wg_socket_init()
749 struct wg_socket *so = &sc->sc_socket; in wg_socket_set_cookie()
752 sx_assert(&sc->sc_lock, SX_XLOCKED); in wg_socket_set_cookie()
753 …ret = wg_socket_set_sockopt(so->so_so4, so->so_so6, SO_USER_COOKIE, &user_cookie, sizeof(user_cook… in wg_socket_set_cookie()
755 so->so_user_cookie = user_cookie; in wg_socket_set_cookie()
761 struct wg_socket *so = &sc->sc_socket; in wg_socket_set_fibnum()
764 sx_assert(&sc->sc_lock, SX_XLOCKED); in wg_socket_set_fibnum()
766 ret = wg_socket_set_sockopt(so->so_so4, so->so_so6, SO_SETFIB, &fibnum, sizeof(fibnum)); in wg_socket_set_fibnum()
768 so->so_fibnum = fibnum; in wg_socket_set_fibnum()
781 struct wg_socket *so = &sc->sc_socket; in wg_socket_set()
784 sx_assert(&sc->sc_lock, SX_XLOCKED); in wg_socket_set()
786 so4 = atomic_load_ptr(&so->so_so4); in wg_socket_set()
787 so6 = atomic_load_ptr(&so->so_so6); in wg_socket_set()
788 atomic_store_ptr(&so->so_so4, new_so4); in wg_socket_set()
789 atomic_store_ptr(&so->so_so6, new_so6); in wg_socket_set()
868 struct wg_socket *so = &sc->sc_socket; in wg_send()
872 size_t len = m->m_pkthdr.len; in wg_send()
875 if (e->e_remote.r_sa.sa_family == AF_INET) { in wg_send()
876 if (e->e_local.l_in.s_addr != INADDR_ANY) in wg_send()
877 control = sbcreatecontrol((caddr_t)&e->e_local.l_in, in wg_send()
881 } else if (e->e_remote.r_sa.sa_family == AF_INET6) { in wg_send()
882 if (!IN6_IS_ADDR_UNSPECIFIED(&e->e_local.l_in6)) in wg_send()
883 control = sbcreatecontrol((caddr_t)&e->e_local.l_pktinfo6, in wg_send()
893 sa = &e->e_remote.r_sa; in wg_send()
896 so4 = atomic_load_ptr(&so->so_so4); in wg_send()
897 so6 = atomic_load_ptr(&so->so_so6); in wg_send()
898 if (e->e_remote.r_sa.sa_family == AF_INET && so4 != NULL) in wg_send()
900 else if (e->e_remote.r_sa.sa_family == AF_INET6 && so6 != NULL) in wg_send()
909 if_inc_counter(sc->sc_ifp, IFCOUNTER_OPACKETS, 1); in wg_send()
910 if_inc_counter(sc->sc_ifp, IFCOUNTER_OBYTES, len); in wg_send()
932 /* Retry if we couldn't bind to e->e_local */ in wg_send_buf()
934 bzero(&e->e_local, sizeof(e->e_local)); in wg_send_buf()
950 atomic_store_bool(&peer->p_enabled, true); in wg_timers_enable()
969 atomic_store_bool(&peer->p_enabled, false); in wg_timers_disable()
971 atomic_store_bool(&peer->p_need_another_keepalive, false); in wg_timers_disable()
973 callout_stop(&peer->p_new_handshake); in wg_timers_disable()
974 callout_stop(&peer->p_send_keepalive); in wg_timers_disable()
975 callout_stop(&peer->p_retry_handshake); in wg_timers_disable()
976 callout_stop(&peer->p_persistent_keepalive); in wg_timers_disable()
977 callout_stop(&peer->p_zero_key_material); in wg_timers_disable()
984 if (interval != peer->p_persistent_keepalive_interval) { in wg_timers_set_persistent_keepalive()
985 atomic_store_16(&peer->p_persistent_keepalive_interval, interval); in wg_timers_set_persistent_keepalive()
987 if (atomic_load_bool(&peer->p_enabled)) in wg_timers_set_persistent_keepalive()
996 mtx_lock(&peer->p_handshake_mtx); in wg_timers_get_last_handshake()
997 time->tv_sec = peer->p_handshake_complete.tv_sec; in wg_timers_get_last_handshake()
998 time->tv_nsec = peer->p_handshake_complete.tv_nsec; in wg_timers_get_last_handshake()
999 mtx_unlock(&peer->p_handshake_mtx); in wg_timers_get_last_handshake()
1007 if (atomic_load_bool(&peer->p_enabled) && in wg_timers_event_data_sent()
1008 !callout_pending(&peer->p_new_handshake)) in wg_timers_event_data_sent()
1009 callout_reset(&peer->p_new_handshake, MSEC_2_TICKS( in wg_timers_event_data_sent()
1021 if (atomic_load_bool(&peer->p_enabled)) { in wg_timers_event_data_received()
1022 if (!callout_pending(&peer->p_send_keepalive)) in wg_timers_event_data_received()
1023 callout_reset(&peer->p_send_keepalive, in wg_timers_event_data_received()
1027 atomic_store_bool(&peer->p_need_another_keepalive, in wg_timers_event_data_received()
1036 callout_stop(&peer->p_send_keepalive); in wg_timers_event_any_authenticated_packet_sent()
1042 callout_stop(&peer->p_new_handshake); in wg_timers_event_any_authenticated_packet_received()
1051 interval = atomic_load_16(&peer->p_persistent_keepalive_interval); in wg_timers_event_any_authenticated_packet_traversal()
1052 if (atomic_load_bool(&peer->p_enabled) && interval > 0) in wg_timers_event_any_authenticated_packet_traversal()
1053 callout_reset(&peer->p_persistent_keepalive, in wg_timers_event_any_authenticated_packet_traversal()
1064 if (atomic_load_bool(&peer->p_enabled)) in wg_timers_event_handshake_initiated()
1065 callout_reset(&peer->p_retry_handshake, MSEC_2_TICKS( in wg_timers_event_handshake_initiated()
1077 if (atomic_load_bool(&peer->p_enabled)) { in wg_timers_event_handshake_complete()
1078 mtx_lock(&peer->p_handshake_mtx); in wg_timers_event_handshake_complete()
1079 callout_stop(&peer->p_retry_handshake); in wg_timers_event_handshake_complete()
1080 peer->p_handshake_retries = 0; in wg_timers_event_handshake_complete()
1081 getnanotime(&peer->p_handshake_complete); in wg_timers_event_handshake_complete()
1082 mtx_unlock(&peer->p_handshake_mtx); in wg_timers_event_handshake_complete()
1093 if (atomic_load_bool(&peer->p_enabled)) in wg_timers_event_session_derived()
1094 callout_reset(&peer->p_zero_key_material, in wg_timers_event_session_derived()
1105 if (atomic_load_bool(&peer->p_enabled)) in wg_timers_event_want_initiation()
1114 peer->p_handshake_retries = 0; in wg_timers_run_send_initiation()
1115 if (noise_remote_initiation_expired(peer->p_remote) == ETIMEDOUT) in wg_timers_run_send_initiation()
1125 mtx_lock(&peer->p_handshake_mtx); in wg_timers_run_retry_handshake()
1126 if (peer->p_handshake_retries <= MAX_TIMER_HANDSHAKES) { in wg_timers_run_retry_handshake()
1127 peer->p_handshake_retries++; in wg_timers_run_retry_handshake()
1128 mtx_unlock(&peer->p_handshake_mtx); in wg_timers_run_retry_handshake()
1130 DPRINTF(peer->p_sc, "Handshake for peer %" PRIu64 " did not complete " in wg_timers_run_retry_handshake()
1131 "after %d seconds, retrying (try %d)\n", peer->p_id, in wg_timers_run_retry_handshake()
1132 REKEY_TIMEOUT, peer->p_handshake_retries + 1); in wg_timers_run_retry_handshake()
1136 mtx_unlock(&peer->p_handshake_mtx); in wg_timers_run_retry_handshake()
1138 DPRINTF(peer->p_sc, "Handshake for peer %" PRIu64 " did not complete " in wg_timers_run_retry_handshake()
1139 "after %d retries, giving up\n", peer->p_id, in wg_timers_run_retry_handshake()
1142 callout_stop(&peer->p_send_keepalive); in wg_timers_run_retry_handshake()
1143 wg_queue_purge(&peer->p_stage_queue); in wg_timers_run_retry_handshake()
1145 if (atomic_load_bool(&peer->p_enabled) && in wg_timers_run_retry_handshake()
1146 !callout_pending(&peer->p_zero_key_material)) in wg_timers_run_retry_handshake()
1147 callout_reset(&peer->p_zero_key_material, in wg_timers_run_retry_handshake()
1162 if (atomic_load_bool(&peer->p_enabled) && in wg_timers_run_send_keepalive()
1163 atomic_load_bool(&peer->p_need_another_keepalive)) { in wg_timers_run_send_keepalive()
1164 atomic_store_bool(&peer->p_need_another_keepalive, false); in wg_timers_run_send_keepalive()
1165 callout_reset(&peer->p_send_keepalive, in wg_timers_run_send_keepalive()
1177 DPRINTF(peer->p_sc, "Retrying handshake with peer %" PRIu64 " because we " in wg_timers_run_new_handshake()
1179 peer->p_id, NEW_HANDSHAKE_TIMEOUT); in wg_timers_run_new_handshake()
1190 DPRINTF(peer->p_sc, "Zeroing out keys for peer %" PRIu64 ", since we " in wg_timers_run_zero_key_material()
1192 peer->p_id, REJECT_AFTER_TIME * 3); in wg_timers_run_zero_key_material()
1193 noise_remote_keypairs_clear(peer->p_remote); in wg_timers_run_zero_key_material()
1201 if (atomic_load_16(&peer->p_persistent_keepalive_interval) > 0) in wg_timers_run_persistent_keepalive()
1211 counter_u64_add(peer->p_tx_bytes, len); in wg_peer_send_buf()
1215 wg_send_buf(peer->p_sc, &endpoint, buf, len); in wg_peer_send_buf()
1223 if (noise_create_initiation(peer->p_remote, &pkt.s_idx, pkt.ue, in wg_send_initiation()
1227 DPRINTF(peer->p_sc, "Sending handshake initiation to peer %" PRIu64 "\n", peer->p_id); in wg_send_initiation()
1230 cookie_maker_mac(&peer->p_cookie, &pkt.m, &pkt, in wg_send_initiation()
1231 sizeof(pkt) - sizeof(pkt.m)); in wg_send_initiation()
1241 if (noise_create_response(peer->p_remote, &pkt.s_idx, &pkt.r_idx, in wg_send_response()
1245 DPRINTF(peer->p_sc, "Sending handshake response to peer %" PRIu64 "\n", peer->p_id); in wg_send_response()
1249 cookie_maker_mac(&peer->p_cookie, &pkt.m, &pkt, in wg_send_response()
1250 sizeof(pkt)-sizeof(pkt.m)); in wg_send_response()
1265 cookie_checker_create_payload(&sc->sc_cookie, cm, pkt.nonce, in wg_send_cookie()
1266 pkt.ec, &e->e_remote.r_sa); in wg_send_cookie()
1276 if (wg_queue_len(&peer->p_stage_queue) > 0) in wg_send_keepalive()
1284 wg_queue_push_staged(&peer->p_stage_queue, pkt); in wg_send_keepalive()
1285 DPRINTF(peer->p_sc, "Sending keepalive packet to peer %" PRIu64 "\n", peer->p_id); in wg_send_keepalive()
1304 underload = wg_queue_len(&sc->sc_handshake_queue) >= MAX_QUEUED_HANDSHAKES / 8; in wg_handshake()
1313 m = pkt->p_mbuf; in wg_handshake()
1314 e = &pkt->p_endpoint; in wg_handshake()
1316 if ((pkt->p_mbuf = m = m_pullup(m, m->m_pkthdr.len)) == NULL) in wg_handshake()
1323 res = cookie_checker_validate_macs(&sc->sc_cookie, &init->m, in wg_handshake()
1324 init, sizeof(*init) - sizeof(init->m), in wg_handshake()
1325 underload, &e->e_remote.r_sa, in wg_handshake()
1326 if_getvnet(sc->sc_ifp)); in wg_handshake()
1335 wg_send_cookie(sc, &init->m, init->s_idx, e); in wg_handshake()
1341 if (noise_consume_initiation(sc->sc_local, &remote, in wg_handshake()
1342 init->s_idx, init->ue, init->es, init->ets) != 0) { in wg_handshake()
1349 DPRINTF(sc, "Receiving handshake initiation from peer %" PRIu64 "\n", peer->p_id); in wg_handshake()
1357 res = cookie_checker_validate_macs(&sc->sc_cookie, &resp->m, in wg_handshake()
1358 resp, sizeof(*resp) - sizeof(resp->m), in wg_handshake()
1359 underload, &e->e_remote.r_sa, in wg_handshake()
1360 if_getvnet(sc->sc_ifp)); in wg_handshake()
1369 wg_send_cookie(sc, &resp->m, resp->s_idx, e); in wg_handshake()
1375 if (noise_consume_response(sc->sc_local, &remote, in wg_handshake()
1376 resp->s_idx, resp->r_idx, resp->ue, resp->en) != 0) { in wg_handshake()
1382 DPRINTF(sc, "Receiving handshake response from peer %" PRIu64 "\n", peer->p_id); in wg_handshake()
1391 if ((remote = noise_remote_index(sc->sc_local, cook->r_idx)) == NULL) { in wg_handshake()
1398 if (cookie_maker_consume_payload(&peer->p_cookie, in wg_handshake()
1399 cook->nonce, cook->ec) == 0) { in wg_handshake()
1415 counter_u64_add(peer->p_rx_bytes, m->m_pkthdr.len); in wg_handshake()
1416 if_inc_counter(sc->sc_ifp, IFCOUNTER_IPACKETS, 1); in wg_handshake()
1417 if_inc_counter(sc->sc_ifp, IFCOUNTER_IBYTES, m->m_pkthdr.len); in wg_handshake()
1428 while ((pkt = wg_queue_dequeue_handshake(&sc->sc_handshake_queue)) != NULL) in wg_softc_handshake_receive()
1454 m->m_flags &= ~(M_BCAST|M_MCAST|M_VLANTAG|M_PROMISC|M_PROTOFLAGS); in wg_mbuf_reset()
1458 m->m_pkthdr.numa_domain = M_NODOM; in wg_mbuf_reset()
1460 SLIST_FOREACH_SAFE(t, &m->m_pkthdr.tags, m_tag_link, tmp) { in wg_mbuf_reset()
1461 if ((t->m_tag_id != 0 || t->m_tag_cookie != MTAG_WGLOOP) && in wg_mbuf_reset()
1462 t->m_tag_id != PACKET_TAG_MACLABEL) in wg_mbuf_reset()
1466 KASSERT((m->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0, in wg_mbuf_reset()
1469 m->m_pkthdr.csum_flags = 0; in wg_mbuf_reset()
1470 m->m_pkthdr.PH_per.sixtyfour[0] = 0; in wg_mbuf_reset()
1471 m->m_pkthdr.PH_loc.sixtyfour[0] = 0; in wg_mbuf_reset()
1477 unsigned int padded_size, last_unit = pkt->p_mbuf->m_pkthdr.len; in calculate_padding()
1480 if (__predict_false(pkt->p_mtu == 0)) { in calculate_padding()
1481 padded_size = (last_unit + (WG_PKT_PADDING - 1)) & in calculate_padding()
1482 ~(WG_PKT_PADDING - 1); in calculate_padding()
1483 return (padded_size - last_unit); in calculate_padding()
1486 if (__predict_false(last_unit > pkt->p_mtu)) in calculate_padding()
1487 last_unit %= pkt->p_mtu; in calculate_padding()
1489 padded_size = (last_unit + (WG_PKT_PADDING - 1)) & ~(WG_PKT_PADDING - 1); in calculate_padding()
1490 if (pkt->p_mtu < padded_size) in calculate_padding()
1491 padded_size = pkt->p_mtu; in calculate_padding()
1492 return (padded_size - last_unit); in calculate_padding()
1507 remote = noise_keypair_remote(pkt->p_keypair); in wg_encrypt()
1509 m = pkt->p_mbuf; in wg_encrypt()
1517 if (noise_keypair_encrypt(pkt->p_keypair, &idx, pkt->p_nonce, m) != 0) in wg_encrypt()
1525 data->t = WG_PKT_DATA; in wg_encrypt()
1526 data->r_idx = idx; in wg_encrypt()
1527 data->nonce = htole64(pkt->p_nonce); in wg_encrypt()
1532 pkt->p_mbuf = m; in wg_encrypt()
1533 atomic_store_rel_int(&pkt->p_state, state); in wg_encrypt()
1534 GROUPTASK_ENQUEUE(&peer->p_send); in wg_encrypt()
1547 remote = noise_keypair_remote(pkt->p_keypair); in wg_decrypt()
1549 m = pkt->p_mbuf; in wg_decrypt()
1552 pkt->p_nonce = le64toh(mtod(m, struct wg_pkt_data *)->nonce); in wg_decrypt()
1555 if (noise_keypair_decrypt(pkt->p_keypair, pkt->p_nonce, m) != 0) in wg_decrypt()
1559 if (__predict_false(m->m_pkthdr.len == 0)) { in wg_decrypt()
1561 "%" PRIu64 "\n", peer->p_id); in wg_decrypt()
1572 if (determine_af_and_pullup(&m, &pkt->p_af) == 0) { in wg_decrypt()
1573 if (pkt->p_af == AF_INET) { in wg_decrypt()
1575 allowed_peer = wg_aip_lookup(sc, AF_INET, &ip->ip_src); in wg_decrypt()
1576 len = ntohs(ip->ip_len); in wg_decrypt()
1577 if (len >= sizeof(struct ip) && len < m->m_pkthdr.len) in wg_decrypt()
1578 m_adj(m, len - m->m_pkthdr.len); in wg_decrypt()
1579 } else if (pkt->p_af == AF_INET6) { in wg_decrypt()
1581 allowed_peer = wg_aip_lookup(sc, AF_INET6, &ip6->ip6_src); in wg_decrypt()
1582 len = ntohs(ip6->ip6_plen) + sizeof(struct ip6_hdr); in wg_decrypt()
1583 if (len < m->m_pkthdr.len) in wg_decrypt()
1584 m_adj(m, len - m->m_pkthdr.len); in wg_decrypt()
1588 DPRINTF(sc, "Packet is neither ipv4 nor ipv6 from peer %" PRIu64 "\n", peer->p_id); in wg_decrypt()
1594 noise_remote_put(allowed_peer->p_remote); in wg_decrypt()
1597 DPRINTF(sc, "Packet has unallowed src IP from peer %" PRIu64 "\n", peer->p_id); in wg_decrypt()
1604 pkt->p_mbuf = m; in wg_decrypt()
1605 atomic_store_rel_int(&pkt->p_state, state); in wg_decrypt()
1606 GROUPTASK_ENQUEUE(&peer->p_recv); in wg_decrypt()
1615 while ((pkt = wg_queue_dequeue_parallel(&sc->sc_decrypt_parallel)) != NULL) in wg_softc_decrypt()
1624 while ((pkt = wg_queue_dequeue_parallel(&sc->sc_encrypt_parallel)) != NULL) in wg_softc_encrypt()
1636 u_int cpu = (sc->sc_encrypt_last_cpu + 1) % mp_ncpus; in wg_encrypt_dispatch()
1637 sc->sc_encrypt_last_cpu = cpu; in wg_encrypt_dispatch()
1638 GROUPTASK_ENQUEUE(&sc->sc_encrypt[cpu]); in wg_encrypt_dispatch()
1644 u_int cpu = (sc->sc_decrypt_last_cpu + 1) % mp_ncpus; in wg_decrypt_dispatch()
1645 sc->sc_decrypt_last_cpu = cpu; in wg_decrypt_dispatch()
1646 GROUPTASK_ENQUEUE(&sc->sc_decrypt[cpu]); in wg_decrypt_dispatch()
1653 struct wg_softc *sc = peer->p_sc; in wg_deliver_out()
1660 while ((pkt = wg_queue_dequeue_serial(&peer->p_encrypt_serial)) != NULL) { in wg_deliver_out()
1661 if (atomic_load_acq_int(&pkt->p_state) != WG_PACKET_CRYPTED) in wg_deliver_out()
1664 m = pkt->p_mbuf; in wg_deliver_out()
1665 pkt->p_mbuf = NULL; in wg_deliver_out()
1667 len = m->m_pkthdr.len; in wg_deliver_out()
1675 counter_u64_add(peer->p_tx_bytes, len); in wg_deliver_out()
1684 if (noise_keep_key_fresh_send(peer->p_remote)) in wg_deliver_out()
1688 if_inc_counter(sc->sc_ifp, IFCOUNTER_OERRORS, 1); in wg_deliver_out()
1695 * Hand a packet to the netmap RX ring, via netmap's
1710 eh->ether_type = af == AF_INET ? in wg_deliver_netmap()
1712 memcpy(eh->ether_shost, "\x02\x02\x02\x02\x02\x02", ETHER_ADDR_LEN); in wg_deliver_netmap()
1713 memcpy(eh->ether_dhost, "\xff\xff\xff\xff\xff\xff", ETHER_ADDR_LEN); in wg_deliver_netmap()
1721 struct wg_softc *sc = peer->p_sc; in wg_deliver_in()
1722 if_t ifp = sc->sc_ifp; in wg_deliver_in()
1728 while ((pkt = wg_queue_dequeue_serial(&peer->p_decrypt_serial)) != NULL) { in wg_deliver_in()
1729 if (atomic_load_acq_int(&pkt->p_state) != WG_PACKET_CRYPTED) in wg_deliver_in()
1732 m = pkt->p_mbuf; in wg_deliver_in()
1733 if (noise_keypair_nonce_check(pkt->p_keypair, pkt->p_nonce) != 0) in wg_deliver_in()
1736 if (noise_keypair_received_with(pkt->p_keypair) == ECONNRESET) in wg_deliver_in()
1741 wg_peer_set_endpoint(peer, &pkt->p_endpoint); in wg_deliver_in()
1743 counter_u64_add(peer->p_rx_bytes, m->m_pkthdr.len + in wg_deliver_in()
1745 if_inc_counter(sc->sc_ifp, IFCOUNTER_IPACKETS, 1); in wg_deliver_in()
1746 if_inc_counter(sc->sc_ifp, IFCOUNTER_IBYTES, m->m_pkthdr.len + in wg_deliver_in()
1749 if (m->m_pkthdr.len == 0) in wg_deliver_in()
1752 af = pkt->p_af; in wg_deliver_in()
1754 pkt->p_mbuf = NULL; in wg_deliver_in()
1756 m->m_pkthdr.rcvif = ifp; in wg_deliver_in()
1778 if (noise_keep_key_fresh_recv(peer->p_remote)) in wg_deliver_in()
1795 pkt->p_mbuf = m; in wg_packet_alloc()
1802 if (pkt->p_keypair != NULL) in wg_packet_free()
1803 noise_keypair_put(pkt->p_keypair); in wg_packet_free()
1804 if (pkt->p_mbuf != NULL) in wg_packet_free()
1805 m_freem(pkt->p_mbuf); in wg_packet_free()
1812 mtx_init(&queue->q_mtx, name, NULL, MTX_DEF); in wg_queue_init()
1813 STAILQ_INIT(&queue->q_queue); in wg_queue_init()
1814 queue->q_len = 0; in wg_queue_init()
1821 mtx_destroy(&queue->q_mtx); in wg_queue_deinit()
1827 return (queue->q_len); in wg_queue_len()
1834 mtx_lock(&hs->q_mtx); in wg_queue_enqueue_handshake()
1835 if (hs->q_len < MAX_QUEUED_HANDSHAKES) { in wg_queue_enqueue_handshake()
1836 STAILQ_INSERT_TAIL(&hs->q_queue, pkt, p_parallel); in wg_queue_enqueue_handshake()
1837 hs->q_len++; in wg_queue_enqueue_handshake()
1841 mtx_unlock(&hs->q_mtx); in wg_queue_enqueue_handshake()
1851 mtx_lock(&hs->q_mtx); in wg_queue_dequeue_handshake()
1852 if ((pkt = STAILQ_FIRST(&hs->q_queue)) != NULL) { in wg_queue_dequeue_handshake()
1853 STAILQ_REMOVE_HEAD(&hs->q_queue, p_parallel); in wg_queue_dequeue_handshake()
1854 hs->q_len--; in wg_queue_dequeue_handshake()
1856 mtx_unlock(&hs->q_mtx); in wg_queue_dequeue_handshake()
1865 mtx_lock(&staged->q_mtx); in wg_queue_push_staged()
1866 if (staged->q_len >= MAX_STAGED_PKT) { in wg_queue_push_staged()
1867 old = STAILQ_FIRST(&staged->q_queue); in wg_queue_push_staged()
1868 STAILQ_REMOVE_HEAD(&staged->q_queue, p_parallel); in wg_queue_push_staged()
1869 staged->q_len--; in wg_queue_push_staged()
1871 STAILQ_INSERT_TAIL(&staged->q_queue, pkt, p_parallel); in wg_queue_push_staged()
1872 staged->q_len++; in wg_queue_push_staged()
1873 mtx_unlock(&staged->q_mtx); in wg_queue_push_staged()
1891 mtx_lock(&staged->q_mtx); in wg_queue_delist_staged()
1892 STAILQ_CONCAT(list, &staged->q_queue); in wg_queue_delist_staged()
1893 staged->q_len = 0; in wg_queue_delist_staged()
1894 mtx_unlock(&staged->q_mtx); in wg_queue_delist_staged()
1910 pkt->p_state = WG_PACKET_UNCRYPTED; in wg_queue_both()
1912 mtx_lock(&serial->q_mtx); in wg_queue_both()
1913 if (serial->q_len < MAX_QUEUED_PKT) { in wg_queue_both()
1914 serial->q_len++; in wg_queue_both()
1915 STAILQ_INSERT_TAIL(&serial->q_queue, pkt, p_serial); in wg_queue_both()
1917 mtx_unlock(&serial->q_mtx); in wg_queue_both()
1921 mtx_unlock(&serial->q_mtx); in wg_queue_both()
1923 mtx_lock(¶llel->q_mtx); in wg_queue_both()
1924 if (parallel->q_len < MAX_QUEUED_PKT) { in wg_queue_both()
1925 parallel->q_len++; in wg_queue_both()
1926 STAILQ_INSERT_TAIL(¶llel->q_queue, pkt, p_parallel); in wg_queue_both()
1928 mtx_unlock(¶llel->q_mtx); in wg_queue_both()
1929 pkt->p_state = WG_PACKET_DEAD; in wg_queue_both()
1932 mtx_unlock(¶llel->q_mtx); in wg_queue_both()
1941 mtx_lock(&serial->q_mtx); in wg_queue_dequeue_serial()
1942 if (serial->q_len > 0 && STAILQ_FIRST(&serial->q_queue)->p_state != WG_PACKET_UNCRYPTED) { in wg_queue_dequeue_serial()
1943 serial->q_len--; in wg_queue_dequeue_serial()
1944 pkt = STAILQ_FIRST(&serial->q_queue); in wg_queue_dequeue_serial()
1945 STAILQ_REMOVE_HEAD(&serial->q_queue, p_serial); in wg_queue_dequeue_serial()
1947 mtx_unlock(&serial->q_mtx); in wg_queue_dequeue_serial()
1955 mtx_lock(¶llel->q_mtx); in wg_queue_dequeue_parallel()
1956 if (parallel->q_len > 0) { in wg_queue_dequeue_parallel()
1957 parallel->q_len--; in wg_queue_dequeue_parallel()
1958 pkt = STAILQ_FIRST(¶llel->q_queue); in wg_queue_dequeue_parallel()
1959 STAILQ_REMOVE_HEAD(¶llel->q_queue, p_parallel); in wg_queue_dequeue_parallel()
1961 mtx_unlock(¶llel->q_mtx); in wg_queue_dequeue_parallel()
1987 if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1); in wg_input()
1994 /* Pullup enough to read packet type */ in wg_input()
1996 if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1); in wg_input()
2001 if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1); in wg_input()
2007 switch (sa->sa_family) { in wg_input()
2011 pkt->p_endpoint.e_remote.r_sin = sin[0]; in wg_input()
2012 pkt->p_endpoint.e_local.l_in = sin[1].sin_addr; in wg_input()
2018 pkt->p_endpoint.e_remote.r_sin6 = sin6[0]; in wg_input()
2019 pkt->p_endpoint.e_local.l_in6 = sin6[1].sin6_addr; in wg_input()
2026 if ((m->m_pkthdr.len == sizeof(struct wg_pkt_initiation) && in wg_input()
2028 (m->m_pkthdr.len == sizeof(struct wg_pkt_response) && in wg_input()
2030 (m->m_pkthdr.len == sizeof(struct wg_pkt_cookie) && in wg_input()
2033 if (wg_queue_enqueue_handshake(&sc->sc_handshake_queue, pkt) != 0) { in wg_input()
2034 if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1); in wg_input()
2037 GROUPTASK_ENQUEUE(&sc->sc_handshake); in wg_input()
2038 } else if (m->m_pkthdr.len >= sizeof(struct wg_pkt_data) + in wg_input()
2041 /* Pullup whole header to read r_idx below. */ in wg_input()
2042 if ((pkt->p_mbuf = m_pullup(m, sizeof(struct wg_pkt_data))) == NULL) in wg_input()
2045 data = mtod(pkt->p_mbuf, struct wg_pkt_data *); in wg_input()
2046 if ((pkt->p_keypair = noise_keypair_lookup(sc->sc_local, data->r_idx)) == NULL) in wg_input()
2049 remote = noise_keypair_remote(pkt->p_keypair); in wg_input()
2051 if (wg_queue_both(&sc->sc_decrypt_parallel, &peer->p_decrypt_serial, pkt) != 0) in wg_input()
2052 if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1); in wg_input()
2060 if_inc_counter(sc->sc_ifp, IFCOUNTER_IERRORS, 1); in wg_input()
2071 struct wg_softc *sc = peer->p_sc; in wg_peer_send_staged()
2073 wg_queue_delist_staged(&peer->p_stage_queue, &list); in wg_peer_send_staged()
2078 if ((keypair = noise_keypair_current(peer->p_remote)) == NULL) in wg_peer_send_staged()
2082 if (noise_keypair_nonce_next(keypair, &pkt->p_nonce) != 0) in wg_peer_send_staged()
2086 pkt->p_keypair = noise_keypair_ref(keypair); in wg_peer_send_staged()
2087 if (wg_queue_both(&sc->sc_encrypt_parallel, &peer->p_encrypt_serial, pkt) != 0) in wg_peer_send_staged()
2088 if_inc_counter(sc->sc_ifp, IFCOUNTER_OQDROPS, 1); in wg_peer_send_staged()
2097 wg_queue_enlist_staged(&peer->p_stage_queue, &list); in wg_peer_send_staged()
2110 pkt->p_mbuf = NULL; in xmit_err()
2118 pkt->p_mbuf = NULL; in xmit_err()
2148 pkt->p_mtu = mtu; in wg_xmit()
2149 pkt->p_af = af; in wg_xmit()
2152 peer = wg_aip_lookup(sc, AF_INET, &mtod(m, struct ip *)->ip_dst); in wg_xmit()
2154 peer = wg_aip_lookup(sc, AF_INET6, &mtod(m, struct ip6_hdr *)->ip6_dst); in wg_xmit()
2160 BPF_MTAP2_AF(ifp, m, pkt->p_af); in wg_xmit()
2173 peer_af = peer->p_endpoint.e_remote.r_sa.sa_family; in wg_xmit()
2176 "discovered for peer %" PRIu64 "\n", peer->p_id); in wg_xmit()
2181 wg_queue_push_staged(&peer->p_stage_queue, pkt); in wg_xmit()
2183 noise_remote_put(peer->p_remote); in wg_xmit()
2187 noise_remote_put(peer->p_remote); in wg_xmit()
2197 if ((*m)->m_pkthdr.len >= sizeof(struct ip6_hdr)) in determine_af_and_pullup()
2199 else if ((*m)->m_pkthdr.len >= sizeof(struct ip)) in determine_af_and_pullup()
2205 ipv = mtod(*m, struct ip *)->ip_v; in determine_af_and_pullup()
2208 else if (ipv == 6 && (*m)->m_pkthdr.len >= sizeof(struct ip6_hdr)) in determine_af_and_pullup()
2224 *etp = ntohs(eh->ether_type); in determine_ethertype_and_pullup()
2232 * transmit packets from the netmap TX ring.
2268 * refusing to advance the transmit ring and retrying later. in wg_transmit()
2279 * packets from the host TX ring.
2311 * Deliver a packet to the host RX ring. Because the interface is in netmap
2334 eh->ether_type = af == AF_INET ? in wg_xmit_netmap()
2336 memcpy(eh->ether_shost, "\x06\x06\x06\x06\x06\x06", ETHER_ADDR_LEN); in wg_xmit_netmap()
2337 memcpy(eh->ether_dhost, "\xff\xff\xff\xff\xff\xff", ETHER_ADDR_LEN); in wg_xmit_netmap()
2351 if (dst->sa_family == AF_UNSPEC || dst->sa_family == pseudo_AF_HDRCMPLT) in wg_output()
2352 memcpy(&af, dst->sa_data, sizeof(af)); in wg_output()
2381 mtu = (ro != NULL && ro->ro_mtu > 0) ? ro->ro_mtu : if_getmtu(ifp); in wg_output()
2397 sx_assert(&sc->sc_lock, SX_XLOCKED); in wg_peer_add()
2399 if (!nvlist_exists_binary(nvl, "public-key")) { in wg_peer_add()
2402 pub_key = nvlist_get_binary(nvl, "public-key", &size); in wg_peer_add()
2406 if (noise_local_keys(sc->sc_local, public, NULL) == 0 && in wg_peer_add()
2410 if ((remote = noise_remote_lookup(sc->sc_local, pub_key)) != NULL) in wg_peer_add()
2420 if (nvlist_exists_bool(nvl, "replace-allowedips") && in wg_peer_add()
2421 nvlist_get_bool(nvl, "replace-allowedips") && in wg_peer_add()
2434 if (size > sizeof(peer->p_endpoint.e_remote)) { in wg_peer_add()
2438 memcpy(&peer->p_endpoint.e_remote, endpoint, size); in wg_peer_add()
2440 if (nvlist_exists_binary(nvl, "preshared-key")) { in wg_peer_add()
2441 preshared_key = nvlist_get_binary(nvl, "preshared-key", &size); in wg_peer_add()
2446 noise_remote_set_psk(peer->p_remote, preshared_key); in wg_peer_add()
2448 if (nvlist_exists_number(nvl, "persistent-keepalive-interval")) { in wg_peer_add()
2449 uint64_t pki = nvlist_get_number(nvl, "persistent-keepalive-interval"); in wg_peer_add()
2456 if (nvlist_exists_nvlist_array(nvl, "allowed-ips")) { in wg_peer_add()
2462 aipl = nvlist_get_nvlist_array(nvl, "allowed-ips", &allowedip_count); in wg_peer_add()
2509 ifp = sc->sc_ifp; in wgc_set()
2510 if (wgd->wgd_size == 0 || wgd->wgd_data == NULL) in wgc_set()
2515 if (wgd->wgd_size >= UINT32_MAX / 2) in wgc_set()
2518 nvlpacked = malloc(wgd->wgd_size, M_TEMP, M_WAITOK | M_ZERO); in wgc_set()
2520 err = copyin(wgd->wgd_data, nvlpacked, wgd->wgd_size); in wgc_set()
2523 nvl = nvlist_unpack(nvlpacked, wgd->wgd_size, 0); in wgc_set()
2528 sx_xlock(&sc->sc_lock); in wgc_set()
2529 if (nvlist_exists_bool(nvl, "replace-peers") && in wgc_set()
2530 nvlist_get_bool(nvl, "replace-peers")) in wgc_set()
2532 if (nvlist_exists_number(nvl, "listen-port")) { in wgc_set()
2533 uint64_t new_port = nvlist_get_number(nvl, "listen-port"); in wgc_set()
2538 if (new_port != sc->sc_socket.so_port) { in wgc_set()
2543 sc->sc_socket.so_port = new_port; in wgc_set()
2546 if (nvlist_exists_binary(nvl, "private-key")) { in wgc_set()
2547 const void *key = nvlist_get_binary(nvl, "private-key", &size); in wgc_set()
2553 if (noise_local_keys(sc->sc_local, NULL, private) != 0 || in wgc_set()
2560 if ((remote = noise_remote_lookup(sc->sc_local, in wgc_set()
2573 noise_local_private(sc->sc_local, key); in wgc_set()
2574 if (noise_local_keys(sc->sc_local, NULL, NULL) == 0) in wgc_set()
2575 cookie_checker_update(&sc->sc_cookie, public); in wgc_set()
2577 cookie_checker_update(&sc->sc_cookie, NULL); in wgc_set()
2580 if (nvlist_exists_number(nvl, "user-cookie")) { in wgc_set()
2581 uint64_t user_cookie = nvlist_get_number(nvl, "user-cookie"); in wgc_set()
2603 sx_xunlock(&sc->sc_lock); in wgc_set()
2628 sx_slock(&sc->sc_lock); in wgc_get()
2630 if (sc->sc_socket.so_port != 0) in wgc_get()
2631 nvlist_add_number(nvl, "listen-port", sc->sc_socket.so_port); in wgc_get()
2632 if (sc->sc_socket.so_user_cookie != 0) in wgc_get()
2633 nvlist_add_number(nvl, "user-cookie", sc->sc_socket.so_user_cookie); in wgc_get()
2634 if (noise_local_keys(sc->sc_local, public_key, private_key) == 0) { in wgc_get()
2635 nvlist_add_binary(nvl, "public-key", public_key, WG_KEY_SIZE); in wgc_get()
2637 nvlist_add_binary(nvl, "private-key", private_key, WG_KEY_SIZE); in wgc_get()
2640 peer_count = sc->sc_peers_num; in wgc_get()
2644 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) { in wgc_get()
2654 (void)noise_remote_keys(peer->p_remote, public_key, preshared_key); in wgc_get()
2655 nvlist_add_binary(nvl_peer, "public-key", public_key, sizeof(public_key)); in wgc_get()
2657 nvlist_add_binary(nvl_peer, "preshared-key", preshared_key, sizeof(preshared_key)); in wgc_get()
2659 if (peer->p_endpoint.e_remote.r_sa.sa_family == AF_INET) in wgc_get()
2660 nvlist_add_binary(nvl_peer, "endpoint", &peer->p_endpoint.e_remote, sizeof(struct sockaddr_in)); in wgc_get()
2661 else if (peer->p_endpoint.e_remote.r_sa.sa_family == AF_INET6) in wgc_get()
2662 … nvlist_add_binary(nvl_peer, "endpoint", &peer->p_endpoint.e_remote, sizeof(struct sockaddr_in6)); in wgc_get()
2664 nvlist_add_binary(nvl_peer, "last-handshake-time", &ts64, sizeof(ts64)); in wgc_get()
2665 …nvlist_add_number(nvl_peer, "persistent-keepalive-interval", peer->p_persistent_keepalive_interval… in wgc_get()
2666 nvlist_add_number(nvl_peer, "rx-bytes", counter_u64_fetch(peer->p_rx_bytes)); in wgc_get()
2667 nvlist_add_number(nvl_peer, "tx-bytes", counter_u64_fetch(peer->p_tx_bytes)); in wgc_get()
2669 aip_count = peer->p_aips_num; in wgc_get()
2673 LIST_FOREACH(aip, &peer->p_aips, a_entry) { in wgc_get()
2682 if (aip->a_af == AF_INET) { in wgc_get()
2683 nvlist_add_binary(nvl_aip, "ipv4", &aip->a_addr.in, sizeof(aip->a_addr.in)); in wgc_get()
2684 nvlist_add_number(nvl_aip, "cidr", bitcount32(aip->a_mask.ip)); in wgc_get()
2687 else if (aip->a_af == AF_INET6) { in wgc_get()
2688 nvlist_add_binary(nvl_aip, "ipv6", &aip->a_addr.in6, sizeof(aip->a_addr.in6)); in wgc_get()
2689 nvlist_add_number(nvl_aip, "cidr", in6_mask2len(&aip->a_mask.in6, NULL)); in wgc_get()
2693 nvlist_add_nvlist_array(nvl_peer, "allowed-ips", (const nvlist_t *const *)nvl_aips, aip_count); in wgc_get()
2708 sx_sunlock(&sc->sc_lock); in wgc_get()
2712 sx_sunlock(&sc->sc_lock); in wgc_get()
2718 if (!wgd->wgd_size) { in wgc_get()
2719 wgd->wgd_size = size; in wgc_get()
2722 if (wgd->wgd_size < size) { in wgc_get()
2726 err = copyout(packed, wgd->wgd_data, size); in wgc_get()
2727 wgd->wgd_size = size; in wgc_get()
2774 if (ifr->ifr_mtu <= 0 || ifr->ifr_mtu > MAX_MTU) in wg_ioctl()
2777 if_setmtu(ifp, ifr->ifr_mtu); in wg_ioctl()
2783 ifr->ifr_fib = sc->sc_socket.so_fibnum; in wg_ioctl()
2792 sx_xlock(&sc->sc_lock); in wg_ioctl()
2793 ret = wg_socket_set_fibnum(sc, ifr->ifr_fib); in wg_ioctl()
2794 sx_xunlock(&sc->sc_lock); in wg_ioctl()
2808 if_t ifp = sc->sc_ifp; in wg_up()
2812 sx_xlock(&sc->sc_lock); in wg_up()
2814 if ((sc->sc_flags & WGF_DYING) != 0) in wg_up()
2823 rc = wg_socket_init(sc, sc->sc_socket.so_port); in wg_up()
2825 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) in wg_up()
2827 if_link_state_change(sc->sc_ifp, LINK_STATE_UP); in wg_up()
2833 sx_xunlock(&sc->sc_lock); in wg_up()
2840 if_t ifp = sc->sc_ifp; in wg_down()
2843 sx_xlock(&sc->sc_lock); in wg_down()
2845 sx_xunlock(&sc->sc_lock); in wg_down()
2850 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) { in wg_down()
2851 wg_queue_purge(&peer->p_stage_queue); in wg_down()
2855 wg_queue_purge(&sc->sc_handshake_queue); in wg_down()
2857 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) { in wg_down()
2858 noise_remote_handshake_clear(peer->p_remote); in wg_down()
2859 noise_remote_keypairs_clear(peer->p_remote); in wg_down()
2862 if_link_state_change(sc->sc_ifp, LINK_STATE_DOWN); in wg_down()
2865 sx_xunlock(&sc->sc_lock); in wg_down()
2877 sc->sc_local = noise_local_alloc(sc); in wg_clone_create()
2879 sc->sc_encrypt = mallocarray(sizeof(struct grouptask), mp_ncpus, M_WG, M_WAITOK | M_ZERO); in wg_clone_create()
2881 sc->sc_decrypt = mallocarray(sizeof(struct grouptask), mp_ncpus, M_WG, M_WAITOK | M_ZERO); in wg_clone_create()
2883 if (!rn_inithead((void **)&sc->sc_aip4, offsetof(struct aip_addr, in) * NBBY)) in wg_clone_create()
2886 if (!rn_inithead((void **)&sc->sc_aip6, offsetof(struct aip_addr, in6) * NBBY)) in wg_clone_create()
2890 ifp = sc->sc_ifp = if_alloc(IFT_WIREGUARD); in wg_clone_create()
2892 sc->sc_ucred = crhold(curthread->td_ucred); in wg_clone_create()
2893 sc->sc_socket.so_fibnum = curthread->td_proc->p_fibnum; in wg_clone_create()
2894 sc->sc_socket.so_port = 0; in wg_clone_create()
2896 TAILQ_INIT(&sc->sc_peers); in wg_clone_create()
2897 sc->sc_peers_num = 0; in wg_clone_create()
2899 cookie_checker_init(&sc->sc_cookie); in wg_clone_create()
2901 RADIX_NODE_HEAD_LOCK_INIT(sc->sc_aip4); in wg_clone_create()
2902 RADIX_NODE_HEAD_LOCK_INIT(sc->sc_aip6); in wg_clone_create()
2904 GROUPTASK_INIT(&sc->sc_handshake, 0, (gtask_fn_t *)wg_softc_handshake_receive, sc); in wg_clone_create()
2905 taskqgroup_attach(qgroup_wg_tqg, &sc->sc_handshake, sc, NULL, NULL, "wg tx initiation"); in wg_clone_create()
2906 wg_queue_init(&sc->sc_handshake_queue, "hsq"); in wg_clone_create()
2909 GROUPTASK_INIT(&sc->sc_encrypt[i], 0, in wg_clone_create()
2911 taskqgroup_attach_cpu(qgroup_wg_tqg, &sc->sc_encrypt[i], sc, i, NULL, NULL, "wg encrypt"); in wg_clone_create()
2912 GROUPTASK_INIT(&sc->sc_decrypt[i], 0, in wg_clone_create()
2914 taskqgroup_attach_cpu(qgroup_wg_tqg, &sc->sc_decrypt[i], sc, i, NULL, NULL, "wg decrypt"); in wg_clone_create()
2917 wg_queue_init(&sc->sc_encrypt_parallel, "encp"); in wg_clone_create()
2918 wg_queue_init(&sc->sc_decrypt_parallel, "decp"); in wg_clone_create()
2920 sx_init(&sc->sc_lock, "wg softc lock"); in wg_clone_create()
2925 if_initname(ifp, wgname, ifd->unit); in wg_clone_create()
2941 ND_IFINFO(ifp)->flags &= ~ND6_IFF_AUTO_LINKLOCAL; in wg_clone_create()
2942 ND_IFINFO(ifp)->flags |= ND6_IFF_NO_DAD; in wg_clone_create()
2950 RADIX_NODE_HEAD_DESTROY(sc->sc_aip4); in wg_clone_create()
2951 free(sc->sc_aip4, M_RTABLE); in wg_clone_create()
2953 free(sc->sc_decrypt, M_WG); in wg_clone_create()
2954 free(sc->sc_encrypt, M_WG); in wg_clone_create()
2955 noise_local_free(sc->sc_local, NULL); in wg_clone_create()
2966 atomic_add_int(&clone_count, -1); in wg_clone_deferred_free()
2977 sx_xlock(&sc->sc_lock); in wg_clone_destroy()
2978 sc->sc_flags |= WGF_DYING; in wg_clone_destroy()
2979 cred = sc->sc_ucred; in wg_clone_destroy()
2980 sc->sc_ucred = NULL; in wg_clone_destroy()
2981 sx_xunlock(&sc->sc_lock); in wg_clone_destroy()
2985 if_link_state_change(sc->sc_ifp, LINK_STATE_DOWN); in wg_clone_destroy()
2986 CURVNET_SET(if_getvnet(sc->sc_ifp)); in wg_clone_destroy()
2987 if_purgeaddrs(sc->sc_ifp); in wg_clone_destroy()
2990 sx_xlock(&sc->sc_lock); in wg_clone_destroy()
2992 sx_xunlock(&sc->sc_lock); in wg_clone_destroy()
3001 sx_xlock(&sc->sc_lock); in wg_clone_destroy()
3004 sx_xunlock(&sc->sc_lock); in wg_clone_destroy()
3005 sx_destroy(&sc->sc_lock); in wg_clone_destroy()
3006 taskqgroup_detach(qgroup_wg_tqg, &sc->sc_handshake); in wg_clone_destroy()
3008 taskqgroup_detach(qgroup_wg_tqg, &sc->sc_encrypt[i]); in wg_clone_destroy()
3009 taskqgroup_detach(qgroup_wg_tqg, &sc->sc_decrypt[i]); in wg_clone_destroy()
3011 free(sc->sc_encrypt, M_WG); in wg_clone_destroy()
3012 free(sc->sc_decrypt, M_WG); in wg_clone_destroy()
3013 wg_queue_deinit(&sc->sc_handshake_queue); in wg_clone_destroy()
3014 wg_queue_deinit(&sc->sc_encrypt_parallel); in wg_clone_destroy()
3015 wg_queue_deinit(&sc->sc_decrypt_parallel); in wg_clone_destroy()
3017 RADIX_NODE_HEAD_DESTROY(sc->sc_aip4); in wg_clone_destroy()
3018 RADIX_NODE_HEAD_DESTROY(sc->sc_aip6); in wg_clone_destroy()
3019 rn_detachhead((void **)&sc->sc_aip4); in wg_clone_destroy()
3020 rn_detachhead((void **)&sc->sc_aip6); in wg_clone_destroy()
3022 cookie_checker_free(&sc->sc_cookie); in wg_clone_destroy()
3026 bpfdetach(sc->sc_ifp); in wg_clone_destroy()
3027 if_detach(sc->sc_ifp); in wg_clone_destroy()
3028 if_free(sc->sc_ifp); in wg_clone_destroy()
3030 noise_local_free(sc->sc_local, wg_clone_deferred_free); in wg_clone_destroy()
3041 * Privileged information (private-key, preshared-key) are only exported for
3107 sx_xlock(&sc->sc_lock); in wg_prison_remove()
3108 if (!(sc->sc_flags & WGF_DYING) && sc->sc_ucred && sc->sc_ucred->cr_prison == pr) { in wg_prison_remove()
3109 struct ucred *cred = sc->sc_ucred; in wg_prison_remove()
3111 if_link_state_change(sc->sc_ifp, LINK_STATE_DOWN); in wg_prison_remove()
3113 sc->sc_ucred = NULL; in wg_prison_remove()
3115 sc->sc_flags |= WGF_DYING; in wg_prison_remove()
3117 sx_xunlock(&sc->sc_lock); in wg_prison_remove()