Lines Matching +full:ring +full:- +full:disable +full:- +full:pullup
1 /* SPDX-License-Identifier: ISC
3 * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
4 * Copyright (C) 2019-2021 Matt Dunwoodie <ncon@noconroy.net>
5 * Copyright (c) 2019-2020 Rubicon Communications, LLC (Netgate)
56 #define DEFAULT_MTU (ETHERMTU - 80)
57 #define MAX_MTU (IF_MAXMTU - 80)
61 #define MAX_QUEUED_PKT_MASK (MAX_QUEUED_PKT - 1)
70 #define DPRINTF(sc, ...) if (if_getflags(sc->sc_ifp) & IFF_DEBUG) if_printf(sc->sc_ifp, ##__VA_ARGS…
256 gtaskqueue_drain((gtask)->gt_taskqueue, &(gtask)->gt_task)
385 sx_assert(&sc->sc_lock, SX_XLOCKED); in wg_peer_alloc()
388 peer->p_remote = noise_remote_alloc(sc->sc_local, peer, pub_key); in wg_peer_alloc()
389 peer->p_tx_bytes = counter_u64_alloc(M_WAITOK); in wg_peer_alloc()
390 peer->p_rx_bytes = counter_u64_alloc(M_WAITOK); in wg_peer_alloc()
391 peer->p_id = peer_counter++; in wg_peer_alloc()
392 peer->p_sc = sc; in wg_peer_alloc()
394 cookie_maker_init(&peer->p_cookie, pub_key); in wg_peer_alloc()
396 rw_init(&peer->p_endpoint_lock, "wg_peer_endpoint"); in wg_peer_alloc()
398 wg_queue_init(&peer->p_stage_queue, "stageq"); in wg_peer_alloc()
399 wg_queue_init(&peer->p_encrypt_serial, "txq"); in wg_peer_alloc()
400 wg_queue_init(&peer->p_decrypt_serial, "rxq"); in wg_peer_alloc()
402 peer->p_enabled = false; in wg_peer_alloc()
403 peer->p_need_another_keepalive = false; in wg_peer_alloc()
404 peer->p_persistent_keepalive_interval = 0; in wg_peer_alloc()
405 callout_init(&peer->p_new_handshake, true); in wg_peer_alloc()
406 callout_init(&peer->p_send_keepalive, true); in wg_peer_alloc()
407 callout_init(&peer->p_retry_handshake, true); in wg_peer_alloc()
408 callout_init(&peer->p_persistent_keepalive, true); in wg_peer_alloc()
409 callout_init(&peer->p_zero_key_material, true); in wg_peer_alloc()
411 mtx_init(&peer->p_handshake_mtx, "peer handshake", NULL, MTX_DEF); in wg_peer_alloc()
412 bzero(&peer->p_handshake_complete, sizeof(peer->p_handshake_complete)); in wg_peer_alloc()
413 peer->p_handshake_retries = 0; in wg_peer_alloc()
415 GROUPTASK_INIT(&peer->p_send, 0, (gtask_fn_t *)wg_deliver_out, peer); in wg_peer_alloc()
416 taskqgroup_attach(qgroup_wg_tqg, &peer->p_send, peer, NULL, NULL, "wg send"); in wg_peer_alloc()
417 GROUPTASK_INIT(&peer->p_recv, 0, (gtask_fn_t *)wg_deliver_in, peer); in wg_peer_alloc()
418 taskqgroup_attach(qgroup_wg_tqg, &peer->p_recv, peer, NULL, NULL, "wg recv"); in wg_peer_alloc()
420 LIST_INIT(&peer->p_aips); in wg_peer_alloc()
421 peer->p_aips_num = 0; in wg_peer_alloc()
434 GROUPTASK_DRAIN(&peer->p_recv); in wg_peer_free_deferred()
435 GROUPTASK_DRAIN(&peer->p_send); in wg_peer_free_deferred()
436 taskqgroup_detach(qgroup_wg_tqg, &peer->p_recv); in wg_peer_free_deferred()
437 taskqgroup_detach(qgroup_wg_tqg, &peer->p_send); in wg_peer_free_deferred()
439 wg_queue_deinit(&peer->p_decrypt_serial); in wg_peer_free_deferred()
440 wg_queue_deinit(&peer->p_encrypt_serial); in wg_peer_free_deferred()
441 wg_queue_deinit(&peer->p_stage_queue); in wg_peer_free_deferred()
443 counter_u64_free(peer->p_tx_bytes); in wg_peer_free_deferred()
444 counter_u64_free(peer->p_rx_bytes); in wg_peer_free_deferred()
445 rw_destroy(&peer->p_endpoint_lock); in wg_peer_free_deferred()
446 mtx_destroy(&peer->p_handshake_mtx); in wg_peer_free_deferred()
448 cookie_maker_free(&peer->p_cookie); in wg_peer_free_deferred()
456 struct wg_softc *sc = peer->p_sc; in wg_peer_destroy()
457 sx_assert(&sc->sc_lock, SX_XLOCKED); in wg_peer_destroy()
459 /* Disable remote and timers. This will prevent any new handshakes in wg_peer_destroy()
461 noise_remote_disable(peer->p_remote); in wg_peer_destroy()
471 sc->sc_peers_num--; in wg_peer_destroy()
472 TAILQ_REMOVE(&sc->sc_peers, peer, p_entry); in wg_peer_destroy()
473 DPRINTF(sc, "Peer %" PRIu64 " destroyed\n", peer->p_id); in wg_peer_destroy()
474 noise_remote_free(peer->p_remote, wg_peer_free_deferred); in wg_peer_destroy()
481 TAILQ_FOREACH_SAFE(peer, &sc->sc_peers, p_entry, tpeer) in wg_peer_destroy_all()
488 MPASS(e->e_remote.r_sa.sa_family != 0); in wg_peer_set_endpoint()
489 if (memcmp(e, &peer->p_endpoint, sizeof(*e)) == 0) in wg_peer_set_endpoint()
492 rw_wlock(&peer->p_endpoint_lock); in wg_peer_set_endpoint()
493 peer->p_endpoint = *e; in wg_peer_set_endpoint()
494 rw_wunlock(&peer->p_endpoint_lock); in wg_peer_set_endpoint()
500 rw_wlock(&peer->p_endpoint_lock); in wg_peer_clear_src()
501 bzero(&peer->p_endpoint.e_local, sizeof(peer->p_endpoint.e_local)); in wg_peer_clear_src()
502 rw_wunlock(&peer->p_endpoint_lock); in wg_peer_clear_src()
508 rw_rlock(&peer->p_endpoint_lock); in wg_peer_get_endpoint()
509 *e = peer->p_endpoint; in wg_peer_get_endpoint()
510 rw_runlock(&peer->p_endpoint_lock); in wg_peer_get_endpoint()
523 aip->a_peer = peer; in wg_aip_add()
524 aip->a_af = af; in wg_aip_add()
530 root = sc->sc_aip4; in wg_aip_add()
531 aip->a_addr.in = *(const struct in_addr *)addr; in wg_aip_add()
532 aip->a_mask.ip = htonl(~((1LL << (32 - cidr)) - 1) & 0xffffffff); in wg_aip_add()
533 aip->a_addr.ip &= aip->a_mask.ip; in wg_aip_add()
534 aip->a_addr.length = aip->a_mask.length = offsetof(struct aip_addr, in) + sizeof(struct in_addr); in wg_aip_add()
540 root = sc->sc_aip6; in wg_aip_add()
541 aip->a_addr.in6 = *(const struct in6_addr *)addr; in wg_aip_add()
542 in6_prefixlen2mask(&aip->a_mask.in6, cidr); in wg_aip_add()
544 aip->a_addr.ip6[i] &= aip->a_mask.ip6[i]; in wg_aip_add()
545 …aip->a_addr.length = aip->a_mask.length = offsetof(struct aip_addr, in6) + sizeof(struct in6_addr); in wg_aip_add()
554 node = root->rnh_addaddr(&aip->a_addr, &aip->a_mask, &root->rh, aip->a_nodes); in wg_aip_add()
555 if (node == aip->a_nodes) { in wg_aip_add()
556 LIST_INSERT_HEAD(&peer->p_aips, aip, a_entry); in wg_aip_add()
557 peer->p_aips_num++; in wg_aip_add()
559 node = root->rnh_lookup(&aip->a_addr, &aip->a_mask, &root->rh); in wg_aip_add()
563 } else if (node != aip->a_nodes) { in wg_aip_add()
566 if (aip->a_peer != peer) { in wg_aip_add()
568 aip->a_peer->p_aips_num--; in wg_aip_add()
569 aip->a_peer = peer; in wg_aip_add()
570 LIST_INSERT_HEAD(&peer->p_aips, aip, a_entry); in wg_aip_add()
571 aip->a_peer->p_aips_num++; in wg_aip_add()
589 root = sc->sc_aip4; in wg_aip_lookup()
594 root = sc->sc_aip6; in wg_aip_lookup()
603 node = root->rnh_matchaddr(&addr, &root->rh); in wg_aip_lookup()
605 peer = ((struct wg_aip *)node)->a_peer; in wg_aip_lookup()
606 noise_remote_ref(peer->p_remote); in wg_aip_lookup()
620 RADIX_NODE_HEAD_LOCK(sc->sc_aip4); in wg_aip_remove_all()
621 LIST_FOREACH_SAFE(aip, &peer->p_aips, a_entry, taip) { in wg_aip_remove_all()
622 if (aip->a_af == AF_INET) { in wg_aip_remove_all()
623 if (sc->sc_aip4->rnh_deladdr(&aip->a_addr, &aip->a_mask, &sc->sc_aip4->rh) == NULL) in wg_aip_remove_all()
626 peer->p_aips_num--; in wg_aip_remove_all()
630 RADIX_NODE_HEAD_UNLOCK(sc->sc_aip4); in wg_aip_remove_all()
632 RADIX_NODE_HEAD_LOCK(sc->sc_aip6); in wg_aip_remove_all()
633 LIST_FOREACH_SAFE(aip, &peer->p_aips, a_entry, taip) { in wg_aip_remove_all()
634 if (aip->a_af == AF_INET6) { in wg_aip_remove_all()
635 if (sc->sc_aip6->rnh_deladdr(&aip->a_addr, &aip->a_mask, &sc->sc_aip6->rh) == NULL) in wg_aip_remove_all()
638 peer->p_aips_num--; in wg_aip_remove_all()
642 RADIX_NODE_HEAD_UNLOCK(sc->sc_aip6); in wg_aip_remove_all()
644 if (!LIST_EMPTY(&peer->p_aips) || peer->p_aips_num != 0) in wg_aip_remove_all()
651 struct ucred *cred = sc->sc_ucred; in wg_socket_init()
655 sx_assert(&sc->sc_lock, SX_XLOCKED); in wg_socket_init()
663 * WireGuard has on Linux with network namespaces -- that the sockets in wg_socket_init()
689 if (sc->sc_socket.so_user_cookie) { in wg_socket_init()
690 …rc = wg_socket_set_sockopt(so4, so6, SO_USER_COOKIE, &sc->sc_socket.so_user_cookie, sizeof(sc->sc_… in wg_socket_init()
694 …rc = wg_socket_set_sockopt(so4, so6, SO_SETFIB, &sc->sc_socket.so_fibnum, sizeof(sc->sc_socket.so_… in wg_socket_init()
700 sc->sc_socket.so_port = port; in wg_socket_init()
733 struct wg_socket *so = &sc->sc_socket; in wg_socket_set_cookie()
736 sx_assert(&sc->sc_lock, SX_XLOCKED); in wg_socket_set_cookie()
737 …ret = wg_socket_set_sockopt(so->so_so4, so->so_so6, SO_USER_COOKIE, &user_cookie, sizeof(user_cook… in wg_socket_set_cookie()
739 so->so_user_cookie = user_cookie; in wg_socket_set_cookie()
745 struct wg_socket *so = &sc->sc_socket; in wg_socket_set_fibnum()
748 sx_assert(&sc->sc_lock, SX_XLOCKED); in wg_socket_set_fibnum()
750 ret = wg_socket_set_sockopt(so->so_so4, so->so_so6, SO_SETFIB, &fibnum, sizeof(fibnum)); in wg_socket_set_fibnum()
752 so->so_fibnum = fibnum; in wg_socket_set_fibnum()
765 struct wg_socket *so = &sc->sc_socket; in wg_socket_set()
768 sx_assert(&sc->sc_lock, SX_XLOCKED); in wg_socket_set()
770 so4 = atomic_load_ptr(&so->so_so4); in wg_socket_set()
771 so6 = atomic_load_ptr(&so->so_so6); in wg_socket_set()
772 atomic_store_ptr(&so->so_so4, new_so4); in wg_socket_set()
773 atomic_store_ptr(&so->so_so6, new_so6); in wg_socket_set()
852 struct wg_socket *so = &sc->sc_socket; in wg_send()
856 size_t len = m->m_pkthdr.len; in wg_send()
859 if (e->e_remote.r_sa.sa_family == AF_INET) { in wg_send()
860 if (e->e_local.l_in.s_addr != INADDR_ANY) in wg_send()
861 control = sbcreatecontrol((caddr_t)&e->e_local.l_in, in wg_send()
865 } else if (e->e_remote.r_sa.sa_family == AF_INET6) { in wg_send()
866 if (!IN6_IS_ADDR_UNSPECIFIED(&e->e_local.l_in6)) in wg_send()
867 control = sbcreatecontrol((caddr_t)&e->e_local.l_pktinfo6, in wg_send()
877 sa = &e->e_remote.r_sa; in wg_send()
880 so4 = atomic_load_ptr(&so->so_so4); in wg_send()
881 so6 = atomic_load_ptr(&so->so_so6); in wg_send()
882 if (e->e_remote.r_sa.sa_family == AF_INET && so4 != NULL) in wg_send()
884 else if (e->e_remote.r_sa.sa_family == AF_INET6 && so6 != NULL) in wg_send()
893 if_inc_counter(sc->sc_ifp, IFCOUNTER_OPACKETS, 1); in wg_send()
894 if_inc_counter(sc->sc_ifp, IFCOUNTER_OBYTES, len); in wg_send()
916 /* Retry if we couldn't bind to e->e_local */ in wg_send_buf()
918 bzero(&e->e_local, sizeof(e->e_local)); in wg_send_buf()
934 atomic_store_bool(&peer->p_enabled, true); in wg_timers_enable()
953 atomic_store_bool(&peer->p_enabled, false); in wg_timers_disable()
955 atomic_store_bool(&peer->p_need_another_keepalive, false); in wg_timers_disable()
957 callout_stop(&peer->p_new_handshake); in wg_timers_disable()
958 callout_stop(&peer->p_send_keepalive); in wg_timers_disable()
959 callout_stop(&peer->p_retry_handshake); in wg_timers_disable()
960 callout_stop(&peer->p_persistent_keepalive); in wg_timers_disable()
961 callout_stop(&peer->p_zero_key_material); in wg_timers_disable()
968 if (interval != peer->p_persistent_keepalive_interval) { in wg_timers_set_persistent_keepalive()
969 atomic_store_16(&peer->p_persistent_keepalive_interval, interval); in wg_timers_set_persistent_keepalive()
971 if (atomic_load_bool(&peer->p_enabled)) in wg_timers_set_persistent_keepalive()
980 mtx_lock(&peer->p_handshake_mtx); in wg_timers_get_last_handshake()
981 time->tv_sec = peer->p_handshake_complete.tv_sec; in wg_timers_get_last_handshake()
982 time->tv_nsec = peer->p_handshake_complete.tv_nsec; in wg_timers_get_last_handshake()
983 mtx_unlock(&peer->p_handshake_mtx); in wg_timers_get_last_handshake()
991 if (atomic_load_bool(&peer->p_enabled) && in wg_timers_event_data_sent()
992 !callout_pending(&peer->p_new_handshake)) in wg_timers_event_data_sent()
993 callout_reset(&peer->p_new_handshake, MSEC_2_TICKS( in wg_timers_event_data_sent()
1005 if (atomic_load_bool(&peer->p_enabled)) { in wg_timers_event_data_received()
1006 if (!callout_pending(&peer->p_send_keepalive)) in wg_timers_event_data_received()
1007 callout_reset(&peer->p_send_keepalive, in wg_timers_event_data_received()
1011 atomic_store_bool(&peer->p_need_another_keepalive, in wg_timers_event_data_received()
1020 callout_stop(&peer->p_send_keepalive); in wg_timers_event_any_authenticated_packet_sent()
1026 callout_stop(&peer->p_new_handshake); in wg_timers_event_any_authenticated_packet_received()
1035 interval = atomic_load_16(&peer->p_persistent_keepalive_interval); in wg_timers_event_any_authenticated_packet_traversal()
1036 if (atomic_load_bool(&peer->p_enabled) && interval > 0) in wg_timers_event_any_authenticated_packet_traversal()
1037 callout_reset(&peer->p_persistent_keepalive, in wg_timers_event_any_authenticated_packet_traversal()
1048 if (atomic_load_bool(&peer->p_enabled)) in wg_timers_event_handshake_initiated()
1049 callout_reset(&peer->p_retry_handshake, MSEC_2_TICKS( in wg_timers_event_handshake_initiated()
1061 if (atomic_load_bool(&peer->p_enabled)) { in wg_timers_event_handshake_complete()
1062 mtx_lock(&peer->p_handshake_mtx); in wg_timers_event_handshake_complete()
1063 callout_stop(&peer->p_retry_handshake); in wg_timers_event_handshake_complete()
1064 peer->p_handshake_retries = 0; in wg_timers_event_handshake_complete()
1065 getnanotime(&peer->p_handshake_complete); in wg_timers_event_handshake_complete()
1066 mtx_unlock(&peer->p_handshake_mtx); in wg_timers_event_handshake_complete()
1077 if (atomic_load_bool(&peer->p_enabled)) in wg_timers_event_session_derived()
1078 callout_reset(&peer->p_zero_key_material, in wg_timers_event_session_derived()
1089 if (atomic_load_bool(&peer->p_enabled)) in wg_timers_event_want_initiation()
1098 peer->p_handshake_retries = 0; in wg_timers_run_send_initiation()
1099 if (noise_remote_initiation_expired(peer->p_remote) == ETIMEDOUT) in wg_timers_run_send_initiation()
1109 mtx_lock(&peer->p_handshake_mtx); in wg_timers_run_retry_handshake()
1110 if (peer->p_handshake_retries <= MAX_TIMER_HANDSHAKES) { in wg_timers_run_retry_handshake()
1111 peer->p_handshake_retries++; in wg_timers_run_retry_handshake()
1112 mtx_unlock(&peer->p_handshake_mtx); in wg_timers_run_retry_handshake()
1114 DPRINTF(peer->p_sc, "Handshake for peer %" PRIu64 " did not complete " in wg_timers_run_retry_handshake()
1115 "after %d seconds, retrying (try %d)\n", peer->p_id, in wg_timers_run_retry_handshake()
1116 REKEY_TIMEOUT, peer->p_handshake_retries + 1); in wg_timers_run_retry_handshake()
1120 mtx_unlock(&peer->p_handshake_mtx); in wg_timers_run_retry_handshake()
1122 DPRINTF(peer->p_sc, "Handshake for peer %" PRIu64 " did not complete " in wg_timers_run_retry_handshake()
1123 "after %d retries, giving up\n", peer->p_id, in wg_timers_run_retry_handshake()
1126 callout_stop(&peer->p_send_keepalive); in wg_timers_run_retry_handshake()
1127 wg_queue_purge(&peer->p_stage_queue); in wg_timers_run_retry_handshake()
1129 if (atomic_load_bool(&peer->p_enabled) && in wg_timers_run_retry_handshake()
1130 !callout_pending(&peer->p_zero_key_material)) in wg_timers_run_retry_handshake()
1131 callout_reset(&peer->p_zero_key_material, in wg_timers_run_retry_handshake()
1146 if (atomic_load_bool(&peer->p_enabled) && in wg_timers_run_send_keepalive()
1147 atomic_load_bool(&peer->p_need_another_keepalive)) { in wg_timers_run_send_keepalive()
1148 atomic_store_bool(&peer->p_need_another_keepalive, false); in wg_timers_run_send_keepalive()
1149 callout_reset(&peer->p_send_keepalive, in wg_timers_run_send_keepalive()
1161 DPRINTF(peer->p_sc, "Retrying handshake with peer %" PRIu64 " because we " in wg_timers_run_new_handshake()
1163 peer->p_id, NEW_HANDSHAKE_TIMEOUT); in wg_timers_run_new_handshake()
1174 DPRINTF(peer->p_sc, "Zeroing out keys for peer %" PRIu64 ", since we " in wg_timers_run_zero_key_material()
1176 peer->p_id, REJECT_AFTER_TIME * 3); in wg_timers_run_zero_key_material()
1177 noise_remote_keypairs_clear(peer->p_remote); in wg_timers_run_zero_key_material()
1185 if (atomic_load_16(&peer->p_persistent_keepalive_interval) > 0) in wg_timers_run_persistent_keepalive()
1195 counter_u64_add(peer->p_tx_bytes, len); in wg_peer_send_buf()
1199 wg_send_buf(peer->p_sc, &endpoint, buf, len); in wg_peer_send_buf()
1207 if (noise_create_initiation(peer->p_remote, &pkt.s_idx, pkt.ue, in wg_send_initiation()
1211 DPRINTF(peer->p_sc, "Sending handshake initiation to peer %" PRIu64 "\n", peer->p_id); in wg_send_initiation()
1214 cookie_maker_mac(&peer->p_cookie, &pkt.m, &pkt, in wg_send_initiation()
1215 sizeof(pkt) - sizeof(pkt.m)); in wg_send_initiation()
1225 if (noise_create_response(peer->p_remote, &pkt.s_idx, &pkt.r_idx, in wg_send_response()
1229 DPRINTF(peer->p_sc, "Sending handshake response to peer %" PRIu64 "\n", peer->p_id); in wg_send_response()
1233 cookie_maker_mac(&peer->p_cookie, &pkt.m, &pkt, in wg_send_response()
1234 sizeof(pkt)-sizeof(pkt.m)); in wg_send_response()
1249 cookie_checker_create_payload(&sc->sc_cookie, cm, pkt.nonce, in wg_send_cookie()
1250 pkt.ec, &e->e_remote.r_sa); in wg_send_cookie()
1260 if (wg_queue_len(&peer->p_stage_queue) > 0) in wg_send_keepalive()
1268 wg_queue_push_staged(&peer->p_stage_queue, pkt); in wg_send_keepalive()
1269 DPRINTF(peer->p_sc, "Sending keepalive packet to peer %" PRIu64 "\n", peer->p_id); in wg_send_keepalive()
1288 underload = wg_queue_len(&sc->sc_handshake_queue) >= MAX_QUEUED_HANDSHAKES / 8; in wg_handshake()
1297 m = pkt->p_mbuf; in wg_handshake()
1298 e = &pkt->p_endpoint; in wg_handshake()
1300 if ((pkt->p_mbuf = m = m_pullup(m, m->m_pkthdr.len)) == NULL) in wg_handshake()
1307 res = cookie_checker_validate_macs(&sc->sc_cookie, &init->m, in wg_handshake()
1308 init, sizeof(*init) - sizeof(init->m), in wg_handshake()
1309 underload, &e->e_remote.r_sa, in wg_handshake()
1310 if_getvnet(sc->sc_ifp)); in wg_handshake()
1319 wg_send_cookie(sc, &init->m, init->s_idx, e); in wg_handshake()
1325 if (noise_consume_initiation(sc->sc_local, &remote, in wg_handshake()
1326 init->s_idx, init->ue, init->es, init->ets) != 0) { in wg_handshake()
1333 DPRINTF(sc, "Receiving handshake initiation from peer %" PRIu64 "\n", peer->p_id); in wg_handshake()
1341 res = cookie_checker_validate_macs(&sc->sc_cookie, &resp->m, in wg_handshake()
1342 resp, sizeof(*resp) - sizeof(resp->m), in wg_handshake()
1343 underload, &e->e_remote.r_sa, in wg_handshake()
1344 if_getvnet(sc->sc_ifp)); in wg_handshake()
1353 wg_send_cookie(sc, &resp->m, resp->s_idx, e); in wg_handshake()
1359 if (noise_consume_response(sc->sc_local, &remote, in wg_handshake()
1360 resp->s_idx, resp->r_idx, resp->ue, resp->en) != 0) { in wg_handshake()
1366 DPRINTF(sc, "Receiving handshake response from peer %" PRIu64 "\n", peer->p_id); in wg_handshake()
1375 if ((remote = noise_remote_index(sc->sc_local, cook->r_idx)) == NULL) { in wg_handshake()
1382 if (cookie_maker_consume_payload(&peer->p_cookie, in wg_handshake()
1383 cook->nonce, cook->ec) == 0) { in wg_handshake()
1399 counter_u64_add(peer->p_rx_bytes, m->m_pkthdr.len); in wg_handshake()
1400 if_inc_counter(sc->sc_ifp, IFCOUNTER_IPACKETS, 1); in wg_handshake()
1401 if_inc_counter(sc->sc_ifp, IFCOUNTER_IBYTES, m->m_pkthdr.len); in wg_handshake()
1412 while ((pkt = wg_queue_dequeue_handshake(&sc->sc_handshake_queue)) != NULL) in wg_softc_handshake_receive()
1438 m->m_flags &= ~(M_BCAST|M_MCAST|M_VLANTAG|M_PROMISC|M_PROTOFLAGS); in wg_mbuf_reset()
1442 m->m_pkthdr.numa_domain = M_NODOM; in wg_mbuf_reset()
1444 SLIST_FOREACH_SAFE(t, &m->m_pkthdr.tags, m_tag_link, tmp) { in wg_mbuf_reset()
1445 if ((t->m_tag_id != 0 || t->m_tag_cookie != MTAG_WGLOOP) && in wg_mbuf_reset()
1446 t->m_tag_id != PACKET_TAG_MACLABEL) in wg_mbuf_reset()
1450 KASSERT((m->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0, in wg_mbuf_reset()
1453 m->m_pkthdr.csum_flags = 0; in wg_mbuf_reset()
1454 m->m_pkthdr.PH_per.sixtyfour[0] = 0; in wg_mbuf_reset()
1455 m->m_pkthdr.PH_loc.sixtyfour[0] = 0; in wg_mbuf_reset()
1461 unsigned int padded_size, last_unit = pkt->p_mbuf->m_pkthdr.len; in calculate_padding()
1464 if (__predict_false(pkt->p_mtu == 0)) { in calculate_padding()
1465 padded_size = (last_unit + (WG_PKT_PADDING - 1)) & in calculate_padding()
1466 ~(WG_PKT_PADDING - 1); in calculate_padding()
1467 return (padded_size - last_unit); in calculate_padding()
1470 if (__predict_false(last_unit > pkt->p_mtu)) in calculate_padding()
1471 last_unit %= pkt->p_mtu; in calculate_padding()
1473 padded_size = (last_unit + (WG_PKT_PADDING - 1)) & ~(WG_PKT_PADDING - 1); in calculate_padding()
1474 if (pkt->p_mtu < padded_size) in calculate_padding()
1475 padded_size = pkt->p_mtu; in calculate_padding()
1476 return (padded_size - last_unit); in calculate_padding()
1491 remote = noise_keypair_remote(pkt->p_keypair); in wg_encrypt()
1493 m = pkt->p_mbuf; in wg_encrypt()
1501 if (noise_keypair_encrypt(pkt->p_keypair, &idx, pkt->p_nonce, m) != 0) in wg_encrypt()
1509 data->t = WG_PKT_DATA; in wg_encrypt()
1510 data->r_idx = idx; in wg_encrypt()
1511 data->nonce = htole64(pkt->p_nonce); in wg_encrypt()
1516 pkt->p_mbuf = m; in wg_encrypt()
1517 atomic_store_rel_int(&pkt->p_state, state); in wg_encrypt()
1518 GROUPTASK_ENQUEUE(&peer->p_send); in wg_encrypt()
1531 remote = noise_keypair_remote(pkt->p_keypair); in wg_decrypt()
1533 m = pkt->p_mbuf; in wg_decrypt()
1536 pkt->p_nonce = le64toh(mtod(m, struct wg_pkt_data *)->nonce); in wg_decrypt()
1539 if (noise_keypair_decrypt(pkt->p_keypair, pkt->p_nonce, m) != 0) in wg_decrypt()
1543 if (__predict_false(m->m_pkthdr.len == 0)) { in wg_decrypt()
1545 "%" PRIu64 "\n", peer->p_id); in wg_decrypt()
1556 if (determine_af_and_pullup(&m, &pkt->p_af) == 0) { in wg_decrypt()
1557 if (pkt->p_af == AF_INET) { in wg_decrypt()
1559 allowed_peer = wg_aip_lookup(sc, AF_INET, &ip->ip_src); in wg_decrypt()
1560 len = ntohs(ip->ip_len); in wg_decrypt()
1561 if (len >= sizeof(struct ip) && len < m->m_pkthdr.len) in wg_decrypt()
1562 m_adj(m, len - m->m_pkthdr.len); in wg_decrypt()
1563 } else if (pkt->p_af == AF_INET6) { in wg_decrypt()
1565 allowed_peer = wg_aip_lookup(sc, AF_INET6, &ip6->ip6_src); in wg_decrypt()
1566 len = ntohs(ip6->ip6_plen) + sizeof(struct ip6_hdr); in wg_decrypt()
1567 if (len < m->m_pkthdr.len) in wg_decrypt()
1568 m_adj(m, len - m->m_pkthdr.len); in wg_decrypt()
1572 DPRINTF(sc, "Packet is neither ipv4 nor ipv6 from peer %" PRIu64 "\n", peer->p_id); in wg_decrypt()
1578 noise_remote_put(allowed_peer->p_remote); in wg_decrypt()
1581 DPRINTF(sc, "Packet has unallowed src IP from peer %" PRIu64 "\n", peer->p_id); in wg_decrypt()
1588 pkt->p_mbuf = m; in wg_decrypt()
1589 atomic_store_rel_int(&pkt->p_state, state); in wg_decrypt()
1590 GROUPTASK_ENQUEUE(&peer->p_recv); in wg_decrypt()
1599 while ((pkt = wg_queue_dequeue_parallel(&sc->sc_decrypt_parallel)) != NULL) in wg_softc_decrypt()
1608 while ((pkt = wg_queue_dequeue_parallel(&sc->sc_encrypt_parallel)) != NULL) in wg_softc_encrypt()
1620 u_int cpu = (sc->sc_encrypt_last_cpu + 1) % mp_ncpus; in wg_encrypt_dispatch()
1621 sc->sc_encrypt_last_cpu = cpu; in wg_encrypt_dispatch()
1622 GROUPTASK_ENQUEUE(&sc->sc_encrypt[cpu]); in wg_encrypt_dispatch()
1628 u_int cpu = (sc->sc_decrypt_last_cpu + 1) % mp_ncpus; in wg_decrypt_dispatch()
1629 sc->sc_decrypt_last_cpu = cpu; in wg_decrypt_dispatch()
1630 GROUPTASK_ENQUEUE(&sc->sc_decrypt[cpu]); in wg_decrypt_dispatch()
1637 struct wg_softc *sc = peer->p_sc; in wg_deliver_out()
1644 while ((pkt = wg_queue_dequeue_serial(&peer->p_encrypt_serial)) != NULL) { in wg_deliver_out()
1645 if (atomic_load_acq_int(&pkt->p_state) != WG_PACKET_CRYPTED) in wg_deliver_out()
1648 m = pkt->p_mbuf; in wg_deliver_out()
1649 pkt->p_mbuf = NULL; in wg_deliver_out()
1651 len = m->m_pkthdr.len; in wg_deliver_out()
1659 counter_u64_add(peer->p_tx_bytes, len); in wg_deliver_out()
1668 if (noise_keep_key_fresh_send(peer->p_remote)) in wg_deliver_out()
1672 if_inc_counter(sc->sc_ifp, IFCOUNTER_OERRORS, 1); in wg_deliver_out()
1679 * Hand a packet to the netmap RX ring, via netmap's
1694 eh->ether_type = af == AF_INET ? in wg_deliver_netmap()
1696 memcpy(eh->ether_shost, "\x02\x02\x02\x02\x02\x02", ETHER_ADDR_LEN); in wg_deliver_netmap()
1697 memcpy(eh->ether_dhost, "\xff\xff\xff\xff\xff\xff", ETHER_ADDR_LEN); in wg_deliver_netmap()
1705 struct wg_softc *sc = peer->p_sc; in wg_deliver_in()
1706 if_t ifp = sc->sc_ifp; in wg_deliver_in()
1712 while ((pkt = wg_queue_dequeue_serial(&peer->p_decrypt_serial)) != NULL) { in wg_deliver_in()
1713 if (atomic_load_acq_int(&pkt->p_state) != WG_PACKET_CRYPTED) in wg_deliver_in()
1716 m = pkt->p_mbuf; in wg_deliver_in()
1717 if (noise_keypair_nonce_check(pkt->p_keypair, pkt->p_nonce) != 0) in wg_deliver_in()
1720 if (noise_keypair_received_with(pkt->p_keypair) == ECONNRESET) in wg_deliver_in()
1725 wg_peer_set_endpoint(peer, &pkt->p_endpoint); in wg_deliver_in()
1727 counter_u64_add(peer->p_rx_bytes, m->m_pkthdr.len + in wg_deliver_in()
1729 if_inc_counter(sc->sc_ifp, IFCOUNTER_IPACKETS, 1); in wg_deliver_in()
1730 if_inc_counter(sc->sc_ifp, IFCOUNTER_IBYTES, m->m_pkthdr.len + in wg_deliver_in()
1733 if (m->m_pkthdr.len == 0) in wg_deliver_in()
1736 af = pkt->p_af; in wg_deliver_in()
1738 pkt->p_mbuf = NULL; in wg_deliver_in()
1740 m->m_pkthdr.rcvif = ifp; in wg_deliver_in()
1762 if (noise_keep_key_fresh_recv(peer->p_remote)) in wg_deliver_in()
1779 pkt->p_mbuf = m; in wg_packet_alloc()
1786 if (pkt->p_keypair != NULL) in wg_packet_free()
1787 noise_keypair_put(pkt->p_keypair); in wg_packet_free()
1788 if (pkt->p_mbuf != NULL) in wg_packet_free()
1789 m_freem(pkt->p_mbuf); in wg_packet_free()
1796 mtx_init(&queue->q_mtx, name, NULL, MTX_DEF); in wg_queue_init()
1797 STAILQ_INIT(&queue->q_queue); in wg_queue_init()
1798 queue->q_len = 0; in wg_queue_init()
1805 mtx_destroy(&queue->q_mtx); in wg_queue_deinit()
1811 return (queue->q_len); in wg_queue_len()
1818 mtx_lock(&hs->q_mtx); in wg_queue_enqueue_handshake()
1819 if (hs->q_len < MAX_QUEUED_HANDSHAKES) { in wg_queue_enqueue_handshake()
1820 STAILQ_INSERT_TAIL(&hs->q_queue, pkt, p_parallel); in wg_queue_enqueue_handshake()
1821 hs->q_len++; in wg_queue_enqueue_handshake()
1825 mtx_unlock(&hs->q_mtx); in wg_queue_enqueue_handshake()
1835 mtx_lock(&hs->q_mtx); in wg_queue_dequeue_handshake()
1836 if ((pkt = STAILQ_FIRST(&hs->q_queue)) != NULL) { in wg_queue_dequeue_handshake()
1837 STAILQ_REMOVE_HEAD(&hs->q_queue, p_parallel); in wg_queue_dequeue_handshake()
1838 hs->q_len--; in wg_queue_dequeue_handshake()
1840 mtx_unlock(&hs->q_mtx); in wg_queue_dequeue_handshake()
1849 mtx_lock(&staged->q_mtx); in wg_queue_push_staged()
1850 if (staged->q_len >= MAX_STAGED_PKT) { in wg_queue_push_staged()
1851 old = STAILQ_FIRST(&staged->q_queue); in wg_queue_push_staged()
1852 STAILQ_REMOVE_HEAD(&staged->q_queue, p_parallel); in wg_queue_push_staged()
1853 staged->q_len--; in wg_queue_push_staged()
1855 STAILQ_INSERT_TAIL(&staged->q_queue, pkt, p_parallel); in wg_queue_push_staged()
1856 staged->q_len++; in wg_queue_push_staged()
1857 mtx_unlock(&staged->q_mtx); in wg_queue_push_staged()
1875 mtx_lock(&staged->q_mtx); in wg_queue_delist_staged()
1876 STAILQ_CONCAT(list, &staged->q_queue); in wg_queue_delist_staged()
1877 staged->q_len = 0; in wg_queue_delist_staged()
1878 mtx_unlock(&staged->q_mtx); in wg_queue_delist_staged()
1894 pkt->p_state = WG_PACKET_UNCRYPTED; in wg_queue_both()
1896 mtx_lock(&serial->q_mtx); in wg_queue_both()
1897 if (serial->q_len < MAX_QUEUED_PKT) { in wg_queue_both()
1898 serial->q_len++; in wg_queue_both()
1899 STAILQ_INSERT_TAIL(&serial->q_queue, pkt, p_serial); in wg_queue_both()
1901 mtx_unlock(&serial->q_mtx); in wg_queue_both()
1905 mtx_unlock(&serial->q_mtx); in wg_queue_both()
1907 mtx_lock(¶llel->q_mtx); in wg_queue_both()
1908 if (parallel->q_len < MAX_QUEUED_PKT) { in wg_queue_both()
1909 parallel->q_len++; in wg_queue_both()
1910 STAILQ_INSERT_TAIL(¶llel->q_queue, pkt, p_parallel); in wg_queue_both()
1912 mtx_unlock(¶llel->q_mtx); in wg_queue_both()
1913 pkt->p_state = WG_PACKET_DEAD; in wg_queue_both()
1916 mtx_unlock(¶llel->q_mtx); in wg_queue_both()
1925 mtx_lock(&serial->q_mtx); in wg_queue_dequeue_serial()
1926 if (serial->q_len > 0 && STAILQ_FIRST(&serial->q_queue)->p_state != WG_PACKET_UNCRYPTED) { in wg_queue_dequeue_serial()
1927 serial->q_len--; in wg_queue_dequeue_serial()
1928 pkt = STAILQ_FIRST(&serial->q_queue); in wg_queue_dequeue_serial()
1929 STAILQ_REMOVE_HEAD(&serial->q_queue, p_serial); in wg_queue_dequeue_serial()
1931 mtx_unlock(&serial->q_mtx); in wg_queue_dequeue_serial()
1939 mtx_lock(¶llel->q_mtx); in wg_queue_dequeue_parallel()
1940 if (parallel->q_len > 0) { in wg_queue_dequeue_parallel()
1941 parallel->q_len--; in wg_queue_dequeue_parallel()
1942 pkt = STAILQ_FIRST(¶llel->q_queue); in wg_queue_dequeue_parallel()
1943 STAILQ_REMOVE_HEAD(¶llel->q_queue, p_parallel); in wg_queue_dequeue_parallel()
1945 mtx_unlock(¶llel->q_mtx); in wg_queue_dequeue_parallel()
1971 if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1); in wg_input()
1978 /* Pullup enough to read packet type */ in wg_input()
1980 if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1); in wg_input()
1985 if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1); in wg_input()
1991 switch (sa->sa_family) { in wg_input()
1995 pkt->p_endpoint.e_remote.r_sin = sin[0]; in wg_input()
1996 pkt->p_endpoint.e_local.l_in = sin[1].sin_addr; in wg_input()
2002 pkt->p_endpoint.e_remote.r_sin6 = sin6[0]; in wg_input()
2003 pkt->p_endpoint.e_local.l_in6 = sin6[1].sin6_addr; in wg_input()
2010 if ((m->m_pkthdr.len == sizeof(struct wg_pkt_initiation) && in wg_input()
2012 (m->m_pkthdr.len == sizeof(struct wg_pkt_response) && in wg_input()
2014 (m->m_pkthdr.len == sizeof(struct wg_pkt_cookie) && in wg_input()
2017 if (wg_queue_enqueue_handshake(&sc->sc_handshake_queue, pkt) != 0) { in wg_input()
2018 if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1); in wg_input()
2021 GROUPTASK_ENQUEUE(&sc->sc_handshake); in wg_input()
2022 } else if (m->m_pkthdr.len >= sizeof(struct wg_pkt_data) + in wg_input()
2025 /* Pullup whole header to read r_idx below. */ in wg_input()
2026 if ((pkt->p_mbuf = m_pullup(m, sizeof(struct wg_pkt_data))) == NULL) in wg_input()
2029 data = mtod(pkt->p_mbuf, struct wg_pkt_data *); in wg_input()
2030 if ((pkt->p_keypair = noise_keypair_lookup(sc->sc_local, data->r_idx)) == NULL) in wg_input()
2033 remote = noise_keypair_remote(pkt->p_keypair); in wg_input()
2035 if (wg_queue_both(&sc->sc_decrypt_parallel, &peer->p_decrypt_serial, pkt) != 0) in wg_input()
2036 if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1); in wg_input()
2044 if_inc_counter(sc->sc_ifp, IFCOUNTER_IERRORS, 1); in wg_input()
2055 struct wg_softc *sc = peer->p_sc; in wg_peer_send_staged()
2057 wg_queue_delist_staged(&peer->p_stage_queue, &list); in wg_peer_send_staged()
2062 if ((keypair = noise_keypair_current(peer->p_remote)) == NULL) in wg_peer_send_staged()
2066 if (noise_keypair_nonce_next(keypair, &pkt->p_nonce) != 0) in wg_peer_send_staged()
2070 pkt->p_keypair = noise_keypair_ref(keypair); in wg_peer_send_staged()
2071 if (wg_queue_both(&sc->sc_encrypt_parallel, &peer->p_encrypt_serial, pkt) != 0) in wg_peer_send_staged()
2072 if_inc_counter(sc->sc_ifp, IFCOUNTER_OQDROPS, 1); in wg_peer_send_staged()
2081 wg_queue_enlist_staged(&peer->p_stage_queue, &list); in wg_peer_send_staged()
2094 pkt->p_mbuf = NULL; in xmit_err()
2102 pkt->p_mbuf = NULL; in xmit_err()
2132 pkt->p_mtu = mtu; in wg_xmit()
2133 pkt->p_af = af; in wg_xmit()
2136 peer = wg_aip_lookup(sc, AF_INET, &mtod(m, struct ip *)->ip_dst); in wg_xmit()
2138 peer = wg_aip_lookup(sc, AF_INET6, &mtod(m, struct ip6_hdr *)->ip6_dst); in wg_xmit()
2144 BPF_MTAP2_AF(ifp, m, pkt->p_af); in wg_xmit()
2157 peer_af = peer->p_endpoint.e_remote.r_sa.sa_family; in wg_xmit()
2160 "discovered for peer %" PRIu64 "\n", peer->p_id); in wg_xmit()
2165 wg_queue_push_staged(&peer->p_stage_queue, pkt); in wg_xmit()
2167 noise_remote_put(peer->p_remote); in wg_xmit()
2171 noise_remote_put(peer->p_remote); in wg_xmit()
2181 if ((*m)->m_pkthdr.len >= sizeof(struct ip6_hdr)) in determine_af_and_pullup()
2183 else if ((*m)->m_pkthdr.len >= sizeof(struct ip)) in determine_af_and_pullup()
2189 ipv = mtod(*m, struct ip *)->ip_v; in determine_af_and_pullup()
2192 else if (ipv == 6 && (*m)->m_pkthdr.len >= sizeof(struct ip6_hdr)) in determine_af_and_pullup()
2208 *etp = ntohs(eh->ether_type); in determine_ethertype_and_pullup()
2216 * transmit packets from the netmap TX ring.
2252 * refusing to advance the transmit ring and retrying later. in wg_transmit()
2263 * packets from the host TX ring.
2295 * Deliver a packet to the host RX ring. Because the interface is in netmap
2318 eh->ether_type = af == AF_INET ? in wg_xmit_netmap()
2320 memcpy(eh->ether_shost, "\x06\x06\x06\x06\x06\x06", ETHER_ADDR_LEN); in wg_xmit_netmap()
2321 memcpy(eh->ether_dhost, "\xff\xff\xff\xff\xff\xff", ETHER_ADDR_LEN); in wg_xmit_netmap()
2335 if (dst->sa_family == AF_UNSPEC || dst->sa_family == pseudo_AF_HDRCMPLT) in wg_output()
2336 memcpy(&af, dst->sa_data, sizeof(af)); in wg_output()
2365 mtu = (ro != NULL && ro->ro_mtu > 0) ? ro->ro_mtu : if_getmtu(ifp); in wg_output()
2381 sx_assert(&sc->sc_lock, SX_XLOCKED); in wg_peer_add()
2383 if (!nvlist_exists_binary(nvl, "public-key")) { in wg_peer_add()
2386 pub_key = nvlist_get_binary(nvl, "public-key", &size); in wg_peer_add()
2390 if (noise_local_keys(sc->sc_local, public, NULL) == 0 && in wg_peer_add()
2394 if ((remote = noise_remote_lookup(sc->sc_local, pub_key)) != NULL) in wg_peer_add()
2404 if (nvlist_exists_bool(nvl, "replace-allowedips") && in wg_peer_add()
2405 nvlist_get_bool(nvl, "replace-allowedips") && in wg_peer_add()
2416 if (size > sizeof(peer->p_endpoint.e_remote)) { in wg_peer_add()
2420 memcpy(&peer->p_endpoint.e_remote, endpoint, size); in wg_peer_add()
2422 if (nvlist_exists_binary(nvl, "preshared-key")) { in wg_peer_add()
2423 preshared_key = nvlist_get_binary(nvl, "preshared-key", &size); in wg_peer_add()
2428 noise_remote_set_psk(peer->p_remote, preshared_key); in wg_peer_add()
2430 if (nvlist_exists_number(nvl, "persistent-keepalive-interval")) { in wg_peer_add()
2431 uint64_t pki = nvlist_get_number(nvl, "persistent-keepalive-interval"); in wg_peer_add()
2438 if (nvlist_exists_nvlist_array(nvl, "allowed-ips")) { in wg_peer_add()
2444 aipl = nvlist_get_nvlist_array(nvl, "allowed-ips", &allowedip_count); in wg_peer_add()
2471 if ((err = noise_remote_enable(peer->p_remote)) != 0) in wg_peer_add()
2473 TAILQ_INSERT_TAIL(&sc->sc_peers, peer, p_entry); in wg_peer_add()
2474 sc->sc_peers_num++; in wg_peer_add()
2475 if (if_getlinkstate(sc->sc_ifp) == LINK_STATE_UP) in wg_peer_add()
2499 ifp = sc->sc_ifp; in wgc_set()
2500 if (wgd->wgd_size == 0 || wgd->wgd_data == NULL) in wgc_set()
2505 if (wgd->wgd_size >= UINT32_MAX / 2) in wgc_set()
2508 nvlpacked = malloc(wgd->wgd_size, M_TEMP, M_WAITOK | M_ZERO); in wgc_set()
2510 err = copyin(wgd->wgd_data, nvlpacked, wgd->wgd_size); in wgc_set()
2513 nvl = nvlist_unpack(nvlpacked, wgd->wgd_size, 0); in wgc_set()
2518 sx_xlock(&sc->sc_lock); in wgc_set()
2519 if (nvlist_exists_bool(nvl, "replace-peers") && in wgc_set()
2520 nvlist_get_bool(nvl, "replace-peers")) in wgc_set()
2522 if (nvlist_exists_number(nvl, "listen-port")) { in wgc_set()
2523 uint64_t new_port = nvlist_get_number(nvl, "listen-port"); in wgc_set()
2528 if (new_port != sc->sc_socket.so_port) { in wgc_set()
2533 sc->sc_socket.so_port = new_port; in wgc_set()
2536 if (nvlist_exists_binary(nvl, "private-key")) { in wgc_set()
2537 const void *key = nvlist_get_binary(nvl, "private-key", &size); in wgc_set()
2543 if (noise_local_keys(sc->sc_local, NULL, private) != 0 || in wgc_set()
2550 if ((remote = noise_remote_lookup(sc->sc_local, in wgc_set()
2563 noise_local_private(sc->sc_local, key); in wgc_set()
2564 if (noise_local_keys(sc->sc_local, NULL, NULL) == 0) in wgc_set()
2565 cookie_checker_update(&sc->sc_cookie, public); in wgc_set()
2567 cookie_checker_update(&sc->sc_cookie, NULL); in wgc_set()
2570 if (nvlist_exists_number(nvl, "user-cookie")) { in wgc_set()
2571 uint64_t user_cookie = nvlist_get_number(nvl, "user-cookie"); in wgc_set()
2593 sx_xunlock(&sc->sc_lock); in wgc_set()
2618 sx_slock(&sc->sc_lock); in wgc_get()
2620 if (sc->sc_socket.so_port != 0) in wgc_get()
2621 nvlist_add_number(nvl, "listen-port", sc->sc_socket.so_port); in wgc_get()
2622 if (sc->sc_socket.so_user_cookie != 0) in wgc_get()
2623 nvlist_add_number(nvl, "user-cookie", sc->sc_socket.so_user_cookie); in wgc_get()
2624 if (noise_local_keys(sc->sc_local, public_key, private_key) == 0) { in wgc_get()
2625 nvlist_add_binary(nvl, "public-key", public_key, WG_KEY_SIZE); in wgc_get()
2627 nvlist_add_binary(nvl, "private-key", private_key, WG_KEY_SIZE); in wgc_get()
2630 peer_count = sc->sc_peers_num; in wgc_get()
2634 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) { in wgc_get()
2644 (void)noise_remote_keys(peer->p_remote, public_key, preshared_key); in wgc_get()
2645 nvlist_add_binary(nvl_peer, "public-key", public_key, sizeof(public_key)); in wgc_get()
2647 nvlist_add_binary(nvl_peer, "preshared-key", preshared_key, sizeof(preshared_key)); in wgc_get()
2649 if (peer->p_endpoint.e_remote.r_sa.sa_family == AF_INET) in wgc_get()
2650 nvlist_add_binary(nvl_peer, "endpoint", &peer->p_endpoint.e_remote, sizeof(struct sockaddr_in)); in wgc_get()
2651 else if (peer->p_endpoint.e_remote.r_sa.sa_family == AF_INET6) in wgc_get()
2652 … nvlist_add_binary(nvl_peer, "endpoint", &peer->p_endpoint.e_remote, sizeof(struct sockaddr_in6)); in wgc_get()
2654 nvlist_add_binary(nvl_peer, "last-handshake-time", &ts64, sizeof(ts64)); in wgc_get()
2655 …nvlist_add_number(nvl_peer, "persistent-keepalive-interval", peer->p_persistent_keepalive_interval… in wgc_get()
2656 nvlist_add_number(nvl_peer, "rx-bytes", counter_u64_fetch(peer->p_rx_bytes)); in wgc_get()
2657 nvlist_add_number(nvl_peer, "tx-bytes", counter_u64_fetch(peer->p_tx_bytes)); in wgc_get()
2659 aip_count = peer->p_aips_num; in wgc_get()
2663 LIST_FOREACH(aip, &peer->p_aips, a_entry) { in wgc_get()
2672 if (aip->a_af == AF_INET) { in wgc_get()
2673 nvlist_add_binary(nvl_aip, "ipv4", &aip->a_addr.in, sizeof(aip->a_addr.in)); in wgc_get()
2674 nvlist_add_number(nvl_aip, "cidr", bitcount32(aip->a_mask.ip)); in wgc_get()
2677 else if (aip->a_af == AF_INET6) { in wgc_get()
2678 nvlist_add_binary(nvl_aip, "ipv6", &aip->a_addr.in6, sizeof(aip->a_addr.in6)); in wgc_get()
2679 nvlist_add_number(nvl_aip, "cidr", in6_mask2len(&aip->a_mask.in6, NULL)); in wgc_get()
2683 nvlist_add_nvlist_array(nvl_peer, "allowed-ips", (const nvlist_t *const *)nvl_aips, aip_count); in wgc_get()
2698 sx_sunlock(&sc->sc_lock); in wgc_get()
2702 sx_sunlock(&sc->sc_lock); in wgc_get()
2708 if (!wgd->wgd_size) { in wgc_get()
2709 wgd->wgd_size = size; in wgc_get()
2712 if (wgd->wgd_size < size) { in wgc_get()
2716 err = copyout(packed, wgd->wgd_data, size); in wgc_get()
2717 wgd->wgd_size = size; in wgc_get()
2764 if (ifr->ifr_mtu <= 0 || ifr->ifr_mtu > MAX_MTU) in wg_ioctl()
2767 if_setmtu(ifp, ifr->ifr_mtu); in wg_ioctl()
2773 ifr->ifr_fib = sc->sc_socket.so_fibnum; in wg_ioctl()
2782 sx_xlock(&sc->sc_lock); in wg_ioctl()
2783 ret = wg_socket_set_fibnum(sc, ifr->ifr_fib); in wg_ioctl()
2784 sx_xunlock(&sc->sc_lock); in wg_ioctl()
2798 if_t ifp = sc->sc_ifp; in wg_up()
2802 sx_xlock(&sc->sc_lock); in wg_up()
2804 if ((sc->sc_flags & WGF_DYING) != 0) in wg_up()
2813 rc = wg_socket_init(sc, sc->sc_socket.so_port); in wg_up()
2815 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) in wg_up()
2817 if_link_state_change(sc->sc_ifp, LINK_STATE_UP); in wg_up()
2823 sx_xunlock(&sc->sc_lock); in wg_up()
2830 if_t ifp = sc->sc_ifp; in wg_down()
2833 sx_xlock(&sc->sc_lock); in wg_down()
2835 sx_xunlock(&sc->sc_lock); in wg_down()
2840 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) { in wg_down()
2841 wg_queue_purge(&peer->p_stage_queue); in wg_down()
2845 wg_queue_purge(&sc->sc_handshake_queue); in wg_down()
2847 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) { in wg_down()
2848 noise_remote_handshake_clear(peer->p_remote); in wg_down()
2849 noise_remote_keypairs_clear(peer->p_remote); in wg_down()
2852 if_link_state_change(sc->sc_ifp, LINK_STATE_DOWN); in wg_down()
2855 sx_xunlock(&sc->sc_lock); in wg_down()
2867 sc->sc_local = noise_local_alloc(sc); in wg_clone_create()
2869 sc->sc_encrypt = mallocarray(sizeof(struct grouptask), mp_ncpus, M_WG, M_WAITOK | M_ZERO); in wg_clone_create()
2871 sc->sc_decrypt = mallocarray(sizeof(struct grouptask), mp_ncpus, M_WG, M_WAITOK | M_ZERO); in wg_clone_create()
2873 if (!rn_inithead((void **)&sc->sc_aip4, offsetof(struct aip_addr, in) * NBBY)) in wg_clone_create()
2876 if (!rn_inithead((void **)&sc->sc_aip6, offsetof(struct aip_addr, in6) * NBBY)) in wg_clone_create()
2880 ifp = sc->sc_ifp = if_alloc(IFT_WIREGUARD); in wg_clone_create()
2882 sc->sc_ucred = crhold(curthread->td_ucred); in wg_clone_create()
2883 sc->sc_socket.so_fibnum = curthread->td_proc->p_fibnum; in wg_clone_create()
2884 sc->sc_socket.so_port = 0; in wg_clone_create()
2886 TAILQ_INIT(&sc->sc_peers); in wg_clone_create()
2887 sc->sc_peers_num = 0; in wg_clone_create()
2889 cookie_checker_init(&sc->sc_cookie); in wg_clone_create()
2891 RADIX_NODE_HEAD_LOCK_INIT(sc->sc_aip4); in wg_clone_create()
2892 RADIX_NODE_HEAD_LOCK_INIT(sc->sc_aip6); in wg_clone_create()
2894 GROUPTASK_INIT(&sc->sc_handshake, 0, (gtask_fn_t *)wg_softc_handshake_receive, sc); in wg_clone_create()
2895 taskqgroup_attach(qgroup_wg_tqg, &sc->sc_handshake, sc, NULL, NULL, "wg tx initiation"); in wg_clone_create()
2896 wg_queue_init(&sc->sc_handshake_queue, "hsq"); in wg_clone_create()
2899 GROUPTASK_INIT(&sc->sc_encrypt[i], 0, in wg_clone_create()
2901 taskqgroup_attach_cpu(qgroup_wg_tqg, &sc->sc_encrypt[i], sc, i, NULL, NULL, "wg encrypt"); in wg_clone_create()
2902 GROUPTASK_INIT(&sc->sc_decrypt[i], 0, in wg_clone_create()
2904 taskqgroup_attach_cpu(qgroup_wg_tqg, &sc->sc_decrypt[i], sc, i, NULL, NULL, "wg decrypt"); in wg_clone_create()
2907 wg_queue_init(&sc->sc_encrypt_parallel, "encp"); in wg_clone_create()
2908 wg_queue_init(&sc->sc_decrypt_parallel, "decp"); in wg_clone_create()
2910 sx_init(&sc->sc_lock, "wg softc lock"); in wg_clone_create()
2915 if_initname(ifp, wgname, ifd->unit); in wg_clone_create()
2931 ND_IFINFO(ifp)->flags &= ~ND6_IFF_AUTO_LINKLOCAL; in wg_clone_create()
2932 ND_IFINFO(ifp)->flags |= ND6_IFF_NO_DAD; in wg_clone_create()
2940 RADIX_NODE_HEAD_DESTROY(sc->sc_aip4); in wg_clone_create()
2941 free(sc->sc_aip4, M_RTABLE); in wg_clone_create()
2943 free(sc->sc_decrypt, M_WG); in wg_clone_create()
2944 free(sc->sc_encrypt, M_WG); in wg_clone_create()
2945 noise_local_free(sc->sc_local, NULL); in wg_clone_create()
2956 atomic_add_int(&clone_count, -1); in wg_clone_deferred_free()
2967 sx_xlock(&sc->sc_lock); in wg_clone_destroy()
2968 sc->sc_flags |= WGF_DYING; in wg_clone_destroy()
2969 cred = sc->sc_ucred; in wg_clone_destroy()
2970 sc->sc_ucred = NULL; in wg_clone_destroy()
2971 sx_xunlock(&sc->sc_lock); in wg_clone_destroy()
2975 if_link_state_change(sc->sc_ifp, LINK_STATE_DOWN); in wg_clone_destroy()
2976 CURVNET_SET(if_getvnet(sc->sc_ifp)); in wg_clone_destroy()
2977 if_purgeaddrs(sc->sc_ifp); in wg_clone_destroy()
2980 sx_xlock(&sc->sc_lock); in wg_clone_destroy()
2982 sx_xunlock(&sc->sc_lock); in wg_clone_destroy()
2991 sx_xlock(&sc->sc_lock); in wg_clone_destroy()
2994 sx_xunlock(&sc->sc_lock); in wg_clone_destroy()
2995 sx_destroy(&sc->sc_lock); in wg_clone_destroy()
2996 taskqgroup_detach(qgroup_wg_tqg, &sc->sc_handshake); in wg_clone_destroy()
2998 taskqgroup_detach(qgroup_wg_tqg, &sc->sc_encrypt[i]); in wg_clone_destroy()
2999 taskqgroup_detach(qgroup_wg_tqg, &sc->sc_decrypt[i]); in wg_clone_destroy()
3001 free(sc->sc_encrypt, M_WG); in wg_clone_destroy()
3002 free(sc->sc_decrypt, M_WG); in wg_clone_destroy()
3003 wg_queue_deinit(&sc->sc_handshake_queue); in wg_clone_destroy()
3004 wg_queue_deinit(&sc->sc_encrypt_parallel); in wg_clone_destroy()
3005 wg_queue_deinit(&sc->sc_decrypt_parallel); in wg_clone_destroy()
3007 RADIX_NODE_HEAD_DESTROY(sc->sc_aip4); in wg_clone_destroy()
3008 RADIX_NODE_HEAD_DESTROY(sc->sc_aip6); in wg_clone_destroy()
3009 rn_detachhead((void **)&sc->sc_aip4); in wg_clone_destroy()
3010 rn_detachhead((void **)&sc->sc_aip6); in wg_clone_destroy()
3012 cookie_checker_free(&sc->sc_cookie); in wg_clone_destroy()
3016 bpfdetach(sc->sc_ifp); in wg_clone_destroy()
3017 if_detach(sc->sc_ifp); in wg_clone_destroy()
3018 if_free(sc->sc_ifp); in wg_clone_destroy()
3020 noise_local_free(sc->sc_local, wg_clone_deferred_free); in wg_clone_destroy()
3031 * Privileged information (private-key, preshared-key) are only exported for
3097 sx_xlock(&sc->sc_lock); in wg_prison_remove()
3098 if (!(sc->sc_flags & WGF_DYING) && sc->sc_ucred && sc->sc_ucred->cr_prison == pr) { in wg_prison_remove()
3099 struct ucred *cred = sc->sc_ucred; in wg_prison_remove()
3101 if_link_state_change(sc->sc_ifp, LINK_STATE_DOWN); in wg_prison_remove()
3103 sc->sc_ucred = NULL; in wg_prison_remove()
3105 sc->sc_flags |= WGF_DYING; in wg_prison_remove()
3107 sx_xunlock(&sc->sc_lock); in wg_prison_remove()