1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2021-2022 Rubicon Communications, LLC (Netgate) 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 */ 28 #include "opt_inet.h" 29 #include "opt_inet6.h" 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/buf_ring.h> 34 #include <sys/epoch.h> 35 #include <sys/file.h> 36 #include <sys/filedesc.h> 37 #include <sys/kernel.h> 38 #include <sys/malloc.h> 39 #include <sys/mbuf.h> 40 #include <sys/module.h> 41 #include <sys/nv.h> 42 #include <sys/priv.h> 43 #include <sys/protosw.h> 44 #include <sys/rmlock.h> 45 #include <sys/sdt.h> 46 #include <sys/smp.h> 47 #include <sys/socket.h> 48 #include <sys/socketvar.h> 49 #include <sys/sockio.h> 50 #include <sys/sysctl.h> 51 #include <sys/time.h> 52 53 #include <machine/atomic.h> 54 55 #include <net/bpf.h> 56 #include <net/if.h> 57 #include <net/if_clone.h> 58 #include <net/if_types.h> 59 #include <net/if_var.h> 60 #include <net/if_private.h> 61 #include <net/netisr.h> 62 #include <net/route/nhop.h> 63 64 #include <netinet/in.h> 65 #include <netinet/in_fib.h> 66 #include <netinet/ip.h> 67 #include <netinet/ip6.h> 68 #include <netinet/ip_var.h> 69 #include <netinet/udp.h> 70 #include <netinet/udp_var.h> 71 72 #include <netinet6/ip6_var.h> 73 #include <netinet6/in6_fib.h> 74 75 #include <machine/in_cksum.h> 76 77 #include <opencrypto/cryptodev.h> 78 79 #include "if_ovpn.h" 80 81 struct ovpn_kkey_dir { 82 int refcount; 83 uint8_t key[32]; 84 uint8_t keylen; 85 uint8_t nonce[8]; 86 uint8_t noncelen; 87 enum ovpn_key_cipher cipher; 88 crypto_session_t cryptoid; 89 90 struct mtx replay_mtx; 91 /* 92 * Last seen gapless sequence number. New rx seq numbers must be 93 * strictly higher than this. 94 */ 95 uint32_t rx_seq; 96 uint64_t tx_seq; 97 98 /* Seen packets, relative to rx_seq. bit(0) will always be 0. */ 99 uint64_t rx_window; 100 }; 101 102 struct ovpn_kkey { 103 struct ovpn_kkey_dir *encrypt; 104 struct ovpn_kkey_dir *decrypt; 105 uint8_t keyid; 106 uint32_t peerid; 107 }; 108 109 struct ovpn_keepalive { 110 uint32_t interval; 111 uint32_t timeout; 112 }; 113 114 struct ovpn_wire_header { 115 uint32_t opcode; /* opcode, key id, peer id */ 116 uint32_t seq; 117 uint8_t auth_tag[16]; 118 }; 119 120 struct ovpn_peer_counters { 121 uint64_t pkt_in; 122 uint64_t pkt_out; 123 uint64_t bytes_in; 124 uint64_t bytes_out; 125 }; 126 #define OVPN_PEER_COUNTER_SIZE (sizeof(struct ovpn_peer_counters)/sizeof(uint64_t)) 127 128 struct ovpn_notification { 129 enum ovpn_notif_type type; 130 uint32_t peerid; 131 132 /* Delete notification */ 133 enum ovpn_del_reason del_reason; 134 struct ovpn_peer_counters counters; 135 }; 136 137 struct ovpn_softc; 138 139 struct ovpn_kpeer { 140 RB_ENTRY(ovpn_kpeer) tree; 141 int refcount; 142 uint32_t peerid; 143 144 struct ovpn_softc *sc; 145 struct sockaddr_storage local; 146 struct sockaddr_storage remote; 147 148 struct in_addr vpn4; 149 struct in6_addr vpn6; 150 151 struct ovpn_kkey keys[2]; 152 153 enum ovpn_del_reason del_reason; 154 struct ovpn_keepalive keepalive; 155 uint32_t *last_active; 156 struct callout ping_send; 157 struct callout ping_rcv; 158 159 counter_u64_t counters[OVPN_PEER_COUNTER_SIZE]; 160 }; 161 162 struct ovpn_counters { 163 uint64_t lost_ctrl_pkts_in; 164 uint64_t lost_ctrl_pkts_out; 165 uint64_t lost_data_pkts_in; 166 uint64_t lost_data_pkts_out; 167 uint64_t nomem_data_pkts_in; 168 uint64_t nomem_data_pkts_out; 169 uint64_t received_ctrl_pkts; 170 uint64_t received_data_pkts; 171 uint64_t sent_ctrl_pkts; 172 uint64_t sent_data_pkts; 173 174 uint64_t transport_bytes_sent; 175 uint64_t transport_bytes_received; 176 uint64_t tunnel_bytes_sent; 177 uint64_t tunnel_bytes_received; 178 }; 179 #define OVPN_COUNTER_SIZE (sizeof(struct ovpn_counters)/sizeof(uint64_t)) 180 181 RB_HEAD(ovpn_kpeers, ovpn_kpeer); 182 183 struct ovpn_softc { 184 int refcount; 185 struct rmlock lock; 186 struct ifnet *ifp; 187 struct socket *so; 188 int peercount; 189 struct ovpn_kpeers peers; 190 191 /* Pending notification */ 192 struct buf_ring *notifring; 193 194 counter_u64_t counters[OVPN_COUNTER_SIZE]; 195 196 struct epoch_context epoch_ctx; 197 }; 198 199 static struct ovpn_kpeer *ovpn_find_peer(struct ovpn_softc *, uint32_t); 200 static bool ovpn_udp_input(struct mbuf *, int, struct inpcb *, 201 const struct sockaddr *, void *); 202 static int ovpn_transmit_to_peer(struct ifnet *, struct mbuf *, 203 struct ovpn_kpeer *, struct rm_priotracker *); 204 static int ovpn_encap(struct ovpn_softc *, uint32_t, struct mbuf *); 205 static int ovpn_get_af(struct mbuf *); 206 static void ovpn_free_kkey_dir(struct ovpn_kkey_dir *); 207 static bool ovpn_check_replay(struct ovpn_kkey_dir *, uint32_t); 208 static int ovpn_peer_compare(struct ovpn_kpeer *, struct ovpn_kpeer *); 209 210 static RB_PROTOTYPE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare); 211 static RB_GENERATE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare); 212 213 #define OVPN_MTU_MIN 576 214 #define OVPN_MTU_MAX (IP_MAXPACKET - sizeof(struct ip) - \ 215 sizeof(struct udphdr) - sizeof(struct ovpn_wire_header)) 216 217 #define OVPN_OP_DATA_V2 0x09 218 #define OVPN_OP_SHIFT 3 219 #define OVPN_SEQ_ROTATE 0x80000000 220 221 VNET_DEFINE_STATIC(struct if_clone *, ovpn_cloner); 222 #define V_ovpn_cloner VNET(ovpn_cloner) 223 224 #define OVPN_RLOCK_TRACKER struct rm_priotracker _ovpn_lock_tracker; \ 225 struct rm_priotracker *_ovpn_lock_trackerp = &_ovpn_lock_tracker 226 #define OVPN_RLOCK(sc) rm_rlock(&(sc)->lock, _ovpn_lock_trackerp) 227 #define OVPN_RUNLOCK(sc) rm_runlock(&(sc)->lock, _ovpn_lock_trackerp) 228 #define OVPN_WLOCK(sc) rm_wlock(&(sc)->lock) 229 #define OVPN_WUNLOCK(sc) rm_wunlock(&(sc)->lock) 230 #define OVPN_ASSERT(sc) rm_assert(&(sc)->lock, RA_LOCKED) 231 #define OVPN_RASSERT(sc) rm_assert(&(sc)->lock, RA_RLOCKED) 232 #define OVPN_WASSERT(sc) rm_assert(&(sc)->lock, RA_WLOCKED) 233 #define OVPN_UNLOCK_ASSERT(sc) rm_assert(&(sc)->lock, RA_UNLOCKED) 234 235 #define OVPN_COUNTER(sc, name) \ 236 ((sc)->counters[offsetof(struct ovpn_counters, name)/sizeof(uint64_t)]) 237 #define OVPN_PEER_COUNTER(peer, name) \ 238 ((peer)->counters[offsetof(struct ovpn_peer_counters, name) / \ 239 sizeof(uint64_t)]) 240 241 #define OVPN_COUNTER_ADD(sc, name, val) \ 242 counter_u64_add(OVPN_COUNTER(sc, name), val) 243 #define OVPN_PEER_COUNTER_ADD(p, name, val) \ 244 counter_u64_add(OVPN_PEER_COUNTER(p, name), val) 245 246 #define TO_IN(x) ((struct sockaddr_in *)(x)) 247 #define TO_IN6(x) ((struct sockaddr_in6 *)(x)) 248 249 SDT_PROVIDER_DEFINE(if_ovpn); 250 SDT_PROBE_DEFINE1(if_ovpn, tx, transmit, start, "struct mbuf *"); 251 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip4, "struct in_addr *", "struct ovpn_kpeer *"); 252 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip6, "struct in6_addr *", "struct ovpn_kpeer *"); 253 254 static const char ovpnname[] = "ovpn"; 255 static const char ovpngroupname[] = "openvpn"; 256 257 static MALLOC_DEFINE(M_OVPN, ovpnname, "OpenVPN DCO Interface"); 258 259 SYSCTL_DECL(_net_link); 260 static SYSCTL_NODE(_net_link, IFT_OTHER, openvpn, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 261 "OpenVPN DCO Interface"); 262 VNET_DEFINE_STATIC(int, replay_protection) = 0; 263 #define V_replay_protection VNET(replay_protection) 264 SYSCTL_INT(_net_link_openvpn, OID_AUTO, replay_protection, CTLFLAG_VNET | CTLFLAG_RW, 265 &VNET_NAME(replay_protection), 0, "Validate sequence numbers"); 266 267 VNET_DEFINE_STATIC(int, async_crypto); 268 #define V_async_crypto VNET(async_crypto) 269 SYSCTL_INT(_net_link_openvpn, OID_AUTO, async_crypto, 270 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_crypto), 0, 271 "Use asynchronous mode to parallelize crypto jobs."); 272 273 VNET_DEFINE_STATIC(int, async_netisr_queue); 274 #define V_async_netisr_queue VNET(async_netisr_queue) 275 SYSCTL_INT(_net_link_openvpn, OID_AUTO, netisr_queue, 276 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_netisr_queue), 0, 277 "Use netisr_queue() rather than netisr_dispatch()."); 278 279 static int 280 ovpn_peer_compare(struct ovpn_kpeer *a, struct ovpn_kpeer *b) 281 { 282 return (a->peerid - b->peerid); 283 } 284 285 static struct ovpn_kpeer * 286 ovpn_find_peer(struct ovpn_softc *sc, uint32_t peerid) 287 { 288 struct ovpn_kpeer p; 289 290 OVPN_ASSERT(sc); 291 292 p.peerid = peerid; 293 294 return (RB_FIND(ovpn_kpeers, &sc->peers, &p)); 295 } 296 297 static struct ovpn_kpeer * 298 ovpn_find_only_peer(struct ovpn_softc *sc) 299 { 300 OVPN_ASSERT(sc); 301 302 return (RB_ROOT(&sc->peers)); 303 } 304 305 static uint16_t 306 ovpn_get_port(struct sockaddr_storage *s) 307 { 308 switch (s->ss_family) { 309 case AF_INET: { 310 struct sockaddr_in *in = (struct sockaddr_in *)s; 311 return (in->sin_port); 312 } 313 case AF_INET6: { 314 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)s; 315 return (in6->sin6_port); 316 } 317 default: 318 panic("Unsupported address family %d", s->ss_family); 319 } 320 } 321 322 static int 323 ovpn_nvlist_to_sockaddr(const nvlist_t *nvl, struct sockaddr_storage *sa) 324 { 325 int af; 326 327 if (! nvlist_exists_number(nvl, "af")) 328 return (EINVAL); 329 if (! nvlist_exists_binary(nvl, "address")) 330 return (EINVAL); 331 if (! nvlist_exists_number(nvl, "port")) 332 return (EINVAL); 333 334 af = nvlist_get_number(nvl, "af"); 335 336 switch (af) { 337 #ifdef INET 338 case AF_INET: { 339 struct sockaddr_in *in = (struct sockaddr_in *)sa; 340 size_t len; 341 const void *addr = nvlist_get_binary(nvl, "address", &len); 342 in->sin_family = af; 343 if (len != sizeof(in->sin_addr)) 344 return (EINVAL); 345 346 memcpy(&in->sin_addr, addr, sizeof(in->sin_addr)); 347 in->sin_port = nvlist_get_number(nvl, "port"); 348 break; 349 } 350 #endif 351 #ifdef INET6 352 case AF_INET6: { 353 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)sa; 354 size_t len; 355 const void *addr = nvlist_get_binary(nvl, "address", &len); 356 in6->sin6_family = af; 357 if (len != sizeof(in6->sin6_addr)) 358 return (EINVAL); 359 360 memcpy(&in6->sin6_addr, addr, sizeof(in6->sin6_addr)); 361 in6->sin6_port = nvlist_get_number(nvl, "port"); 362 break; 363 } 364 #endif 365 default: 366 return (EINVAL); 367 } 368 369 return (0); 370 } 371 372 static bool 373 ovpn_has_peers(struct ovpn_softc *sc) 374 { 375 OVPN_ASSERT(sc); 376 377 return (sc->peercount > 0); 378 } 379 380 static void 381 ovpn_rele_so(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 382 { 383 bool has_peers; 384 385 OVPN_WASSERT(sc); 386 387 if (sc->so == NULL) 388 return; 389 390 has_peers = ovpn_has_peers(sc); 391 392 /* Only remove the tunnel function if we're releasing the socket for 393 * the last peer. */ 394 if (! has_peers) 395 (void)udp_set_kernel_tunneling(sc->so, NULL, NULL, NULL); 396 397 sorele(sc->so); 398 399 if (! has_peers) 400 sc->so = NULL; 401 } 402 403 static void 404 ovpn_notify_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 405 { 406 struct ovpn_notification *n; 407 408 OVPN_WASSERT(sc); 409 410 n = malloc(sizeof(*n), M_OVPN, M_NOWAIT); 411 if (n == NULL) 412 return; 413 414 n->peerid = peer->peerid; 415 n->type = OVPN_NOTIF_DEL_PEER; 416 n->del_reason = peer->del_reason; 417 418 n->counters.pkt_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_in)); 419 n->counters.pkt_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_out)); 420 n->counters.bytes_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_in)); 421 n->counters.bytes_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_out)); 422 423 if (buf_ring_enqueue(sc->notifring, n) != 0) { 424 free(n, M_OVPN); 425 } else if (sc->so != NULL) { 426 /* Wake up userspace */ 427 sc->so->so_error = EAGAIN; 428 sorwakeup(sc->so); 429 sowwakeup(sc->so); 430 } 431 } 432 433 static void 434 ovpn_notify_key_rotation(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 435 { 436 struct ovpn_notification *n; 437 438 n = malloc(sizeof(*n), M_OVPN, M_NOWAIT | M_ZERO); 439 if (n == NULL) 440 return; 441 442 n->peerid = peer->peerid; 443 n->type = OVPN_NOTIF_ROTATE_KEY; 444 445 if (buf_ring_enqueue(sc->notifring, n) != 0) { 446 free(n, M_OVPN); 447 } else if (sc->so != NULL) { 448 /* Wake up userspace */ 449 sc->so->so_error = EAGAIN; 450 sorwakeup(sc->so); 451 sowwakeup(sc->so); 452 } 453 } 454 455 static void 456 ovpn_peer_release_ref(struct ovpn_kpeer *peer, bool locked) 457 { 458 struct ovpn_softc *sc; 459 460 CURVNET_ASSERT_SET(); 461 462 atomic_add_int(&peer->refcount, -1); 463 464 if (atomic_load_int(&peer->refcount) > 0) 465 return; 466 467 sc = peer->sc; 468 469 if (! locked) { 470 OVPN_WLOCK(sc); 471 472 /* Might have changed before we acquired the lock. */ 473 if (atomic_load_int(&peer->refcount) > 0) { 474 OVPN_WUNLOCK(sc); 475 return; 476 } 477 } 478 479 OVPN_ASSERT(sc); 480 481 /* The peer should have been removed from the list already. */ 482 MPASS(ovpn_find_peer(sc, peer->peerid) == NULL); 483 484 ovpn_notify_del_peer(sc, peer); 485 486 for (int i = 0; i < 2; i++) { 487 ovpn_free_kkey_dir(peer->keys[i].encrypt); 488 ovpn_free_kkey_dir(peer->keys[i].decrypt); 489 } 490 491 ovpn_rele_so(sc, peer); 492 493 callout_stop(&peer->ping_send); 494 callout_stop(&peer->ping_rcv); 495 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 496 free(peer, M_OVPN); 497 498 if (! locked) 499 OVPN_WUNLOCK(sc); 500 } 501 502 static int 503 ovpn_new_peer(struct ifnet *ifp, const nvlist_t *nvl) 504 { 505 #ifdef INET6 506 struct epoch_tracker et; 507 #endif 508 struct sockaddr_storage remote; 509 struct ovpn_kpeer *peer = NULL; 510 struct file *fp = NULL; 511 struct ovpn_softc *sc = ifp->if_softc; 512 struct thread *td = curthread; 513 struct socket *so = NULL; 514 int fd; 515 uint32_t peerid; 516 int ret = 0; 517 518 if (nvl == NULL) 519 return (EINVAL); 520 521 if (! nvlist_exists_number(nvl, "peerid")) 522 return (EINVAL); 523 524 if (! nvlist_exists_number(nvl, "fd")) 525 return (EINVAL); 526 527 if (! nvlist_exists_nvlist(nvl, "remote")) 528 return (EINVAL); 529 530 peerid = nvlist_get_number(nvl, "peerid"); 531 532 ret = ovpn_nvlist_to_sockaddr(nvlist_get_nvlist(nvl, "remote"), 533 &remote); 534 if (ret != 0) 535 return (ret); 536 537 fd = nvlist_get_number(nvl, "fd"); 538 539 /* Look up the userspace process and use the fd to find the socket. */ 540 ret = getsock(td, fd, &cap_connect_rights, &fp); 541 if (ret != 0) 542 return (ret); 543 544 so = fp->f_data; 545 546 peer = malloc(sizeof(*peer), M_OVPN, M_WAITOK | M_ZERO); 547 peer->peerid = peerid; 548 peer->sc = sc; 549 peer->refcount = 1; 550 peer->last_active = uma_zalloc_pcpu(pcpu_zone_4, M_WAITOK | M_ZERO); 551 COUNTER_ARRAY_ALLOC(peer->counters, OVPN_PEER_COUNTER_SIZE, M_WAITOK); 552 553 if (nvlist_exists_binary(nvl, "vpn_ipv4")) { 554 size_t len; 555 const void *addr = nvlist_get_binary(nvl, "vpn_ipv4", &len); 556 if (len != sizeof(peer->vpn4)) { 557 ret = EINVAL; 558 goto error; 559 } 560 memcpy(&peer->vpn4, addr, len); 561 } 562 563 if (nvlist_exists_binary(nvl, "vpn_ipv6")) { 564 size_t len; 565 const void *addr = nvlist_get_binary(nvl, "vpn_ipv6", &len); 566 if (len != sizeof(peer->vpn6)) { 567 ret = EINVAL; 568 goto error; 569 } 570 memcpy(&peer->vpn6, addr, len); 571 } 572 573 callout_init_rm(&peer->ping_send, &sc->lock, CALLOUT_SHAREDLOCK); 574 callout_init_rm(&peer->ping_rcv, &sc->lock, 0); 575 576 peer->local.ss_len = sizeof(peer->local); 577 ret = sosockaddr(so, (struct sockaddr *)&peer->local); 578 if (ret) 579 goto error; 580 581 if (ovpn_get_port(&peer->local) == 0) { 582 ret = EINVAL; 583 goto error; 584 } 585 if (peer->local.ss_family != remote.ss_family) { 586 ret = EINVAL; 587 goto error; 588 } 589 590 memcpy(&peer->remote, &remote, sizeof(remote)); 591 592 if (peer->local.ss_family == AF_INET6 && 593 IN6_IS_ADDR_V4MAPPED(&TO_IN6(&peer->remote)->sin6_addr)) { 594 /* V4 mapped address, so treat this as v4, not v6. */ 595 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->local); 596 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->remote); 597 } 598 599 #ifdef INET6 600 if (peer->local.ss_family == AF_INET6 && 601 IN6_IS_ADDR_UNSPECIFIED(&TO_IN6(&peer->local)->sin6_addr)) { 602 NET_EPOCH_ENTER(et); 603 ret = in6_selectsrc_addr(curthread->td_proc->p_fibnum, 604 &TO_IN6(&peer->remote)->sin6_addr, 605 0, NULL, &TO_IN6(&peer->local)->sin6_addr, NULL); 606 NET_EPOCH_EXIT(et); 607 if (ret != 0) { 608 goto error; 609 } 610 } 611 #endif 612 OVPN_WLOCK(sc); 613 614 /* Disallow peer id re-use. */ 615 if (ovpn_find_peer(sc, peerid) != NULL) { 616 ret = EEXIST; 617 goto error_locked; 618 } 619 620 /* Make sure this is really a UDP socket. */ 621 if (so->so_type != SOCK_DGRAM || so->so_proto->pr_type != SOCK_DGRAM) { 622 ret = EPROTOTYPE; 623 goto error_locked; 624 } 625 626 /* Must be the same socket as for other peers on this interface. */ 627 if (sc->so != NULL && so != sc->so) 628 goto error_locked; 629 630 if (sc->so == NULL) 631 sc->so = so; 632 633 /* Insert the peer into the list. */ 634 RB_INSERT(ovpn_kpeers, &sc->peers, peer); 635 sc->peercount++; 636 soref(sc->so); 637 638 ret = udp_set_kernel_tunneling(sc->so, ovpn_udp_input, NULL, sc); 639 if (ret == EBUSY) { 640 /* Fine, another peer already set the input function. */ 641 ret = 0; 642 } 643 if (ret != 0) { 644 RB_REMOVE(ovpn_kpeers, &sc->peers, peer); 645 sc->peercount--; 646 goto error_locked; 647 } 648 649 OVPN_WUNLOCK(sc); 650 651 goto done; 652 653 error_locked: 654 OVPN_WUNLOCK(sc); 655 error: 656 COUNTER_ARRAY_FREE(peer->counters, OVPN_PEER_COUNTER_SIZE); 657 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 658 free(peer, M_OVPN); 659 done: 660 if (fp != NULL) 661 fdrop(fp, td); 662 663 return (ret); 664 } 665 666 static int 667 _ovpn_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 668 { 669 struct ovpn_kpeer *tmp __diagused; 670 671 OVPN_WASSERT(sc); 672 CURVNET_ASSERT_SET(); 673 674 MPASS(RB_FIND(ovpn_kpeers, &sc->peers, peer) == peer); 675 676 tmp = RB_REMOVE(ovpn_kpeers, &sc->peers, peer); 677 MPASS(tmp != NULL); 678 679 sc->peercount--; 680 681 ovpn_peer_release_ref(peer, true); 682 683 return (0); 684 } 685 686 static int 687 ovpn_del_peer(struct ifnet *ifp, nvlist_t *nvl) 688 { 689 struct ovpn_softc *sc = ifp->if_softc; 690 struct ovpn_kpeer *peer; 691 uint32_t peerid; 692 int ret; 693 694 OVPN_WASSERT(sc); 695 696 if (nvl == NULL) 697 return (EINVAL); 698 699 if (! nvlist_exists_number(nvl, "peerid")) 700 return (EINVAL); 701 702 peerid = nvlist_get_number(nvl, "peerid"); 703 704 peer = ovpn_find_peer(sc, peerid); 705 if (peer == NULL) 706 return (ENOENT); 707 708 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 709 ret = _ovpn_del_peer(sc, peer); 710 711 return (ret); 712 } 713 714 static int 715 ovpn_create_kkey_dir(struct ovpn_kkey_dir **kdirp, 716 const nvlist_t *nvl) 717 { 718 struct crypto_session_params csp; 719 struct ovpn_kkey_dir *kdir; 720 const char *ciphername; 721 enum ovpn_key_cipher cipher; 722 const void *key, *iv; 723 size_t keylen = 0, ivlen = 0; 724 int error; 725 726 if (! nvlist_exists_string(nvl, "cipher")) 727 return (EINVAL); 728 ciphername = nvlist_get_string(nvl, "cipher"); 729 730 if (strcmp(ciphername, "none") == 0) 731 cipher = OVPN_CIPHER_ALG_NONE; 732 else if (strcmp(ciphername, "AES-256-GCM") == 0 || 733 strcmp(ciphername, "AES-192-GCM") == 0 || 734 strcmp(ciphername, "AES-128-GCM") == 0) 735 cipher = OVPN_CIPHER_ALG_AES_GCM; 736 else if (strcmp(ciphername, "CHACHA20-POLY1305") == 0) 737 cipher = OVPN_CIPHER_ALG_CHACHA20_POLY1305; 738 else 739 return (EINVAL); 740 741 if (cipher != OVPN_CIPHER_ALG_NONE) { 742 if (! nvlist_exists_binary(nvl, "key")) 743 return (EINVAL); 744 key = nvlist_get_binary(nvl, "key", &keylen); 745 if (keylen > sizeof(kdir->key)) 746 return (E2BIG); 747 748 if (! nvlist_exists_binary(nvl, "iv")) 749 return (EINVAL); 750 iv = nvlist_get_binary(nvl, "iv", &ivlen); 751 if (ivlen != 8) 752 return (E2BIG); 753 } 754 755 kdir = malloc(sizeof(struct ovpn_kkey_dir), M_OVPN, 756 M_WAITOK | M_ZERO); 757 758 kdir->cipher = cipher; 759 kdir->keylen = keylen; 760 kdir->tx_seq = 1; 761 memcpy(kdir->key, key, keylen); 762 kdir->noncelen = ivlen; 763 memcpy(kdir->nonce, iv, ivlen); 764 765 if (kdir->cipher != OVPN_CIPHER_ALG_NONE) { 766 /* Crypto init */ 767 bzero(&csp, sizeof(csp)); 768 csp.csp_mode = CSP_MODE_AEAD; 769 770 if (kdir->cipher == OVPN_CIPHER_ALG_CHACHA20_POLY1305) 771 csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305; 772 else 773 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 774 775 csp.csp_flags |= CSP_F_SEPARATE_AAD; 776 777 csp.csp_cipher_klen = kdir->keylen; 778 csp.csp_cipher_key = kdir->key; 779 csp.csp_ivlen = 96 / 8; 780 781 error = crypto_newsession(&kdir->cryptoid, &csp, 782 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE); 783 if (error) { 784 free(kdir, M_OVPN); 785 return (error); 786 } 787 } 788 789 mtx_init(&kdir->replay_mtx, "if_ovpn rx replay", NULL, MTX_DEF); 790 *kdirp = kdir; 791 792 return (0); 793 } 794 795 static void 796 ovpn_free_kkey_dir(struct ovpn_kkey_dir *kdir) 797 { 798 if (kdir == NULL) 799 return; 800 801 mtx_destroy(&kdir->replay_mtx); 802 803 crypto_freesession(kdir->cryptoid); 804 free(kdir, M_OVPN); 805 } 806 807 static int 808 ovpn_set_key(struct ifnet *ifp, const nvlist_t *nvl) 809 { 810 struct ovpn_softc *sc = ifp->if_softc; 811 struct ovpn_kkey_dir *enc, *dec; 812 struct ovpn_kpeer *peer; 813 int slot, keyid, peerid; 814 int error; 815 816 if (nvl == NULL) 817 return (EINVAL); 818 819 if (! nvlist_exists_number(nvl, "slot")) 820 return (EINVAL); 821 slot = nvlist_get_number(nvl, "slot"); 822 823 if (! nvlist_exists_number(nvl, "keyid")) 824 return (EINVAL); 825 keyid = nvlist_get_number(nvl, "keyid"); 826 827 if (! nvlist_exists_number(nvl, "peerid")) 828 return (EINVAL); 829 peerid = nvlist_get_number(nvl, "peerid"); 830 831 if (slot != OVPN_KEY_SLOT_PRIMARY && 832 slot != OVPN_KEY_SLOT_SECONDARY) 833 return (EINVAL); 834 835 if (! nvlist_exists_nvlist(nvl, "encrypt") || 836 ! nvlist_exists_nvlist(nvl, "decrypt")) 837 return (EINVAL); 838 839 error = ovpn_create_kkey_dir(&enc, nvlist_get_nvlist(nvl, "encrypt")); 840 if (error) 841 return (error); 842 843 error = ovpn_create_kkey_dir(&dec, nvlist_get_nvlist(nvl, "decrypt")); 844 if (error) { 845 ovpn_free_kkey_dir(enc); 846 return (error); 847 } 848 849 OVPN_WLOCK(sc); 850 851 peer = ovpn_find_peer(sc, peerid); 852 if (peer == NULL) { 853 ovpn_free_kkey_dir(dec); 854 ovpn_free_kkey_dir(enc); 855 OVPN_WUNLOCK(sc); 856 return (ENOENT); 857 } 858 859 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 860 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 861 862 peer->keys[slot].encrypt = enc; 863 peer->keys[slot].decrypt = dec; 864 865 peer->keys[slot].keyid = keyid; 866 peer->keys[slot].peerid = peerid; 867 868 OVPN_WUNLOCK(sc); 869 870 return (0); 871 } 872 873 static int 874 ovpn_check_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, enum ovpn_key_slot slot) 875 { 876 OVPN_ASSERT(sc); 877 878 if (peer->keys[slot].encrypt == NULL) 879 return (ENOLINK); 880 881 if (peer->keys[slot].decrypt == NULL) 882 return (ENOLINK); 883 884 return (0); 885 } 886 887 static int 888 ovpn_start(struct ifnet *ifp) 889 { 890 struct ovpn_softc *sc = ifp->if_softc; 891 892 OVPN_WLOCK(sc); 893 894 ifp->if_flags |= IFF_UP; 895 ifp->if_drv_flags |= IFF_DRV_RUNNING; 896 if_link_state_change(ifp, LINK_STATE_UP); 897 898 OVPN_WUNLOCK(sc); 899 900 return (0); 901 } 902 903 static int 904 ovpn_swap_keys(struct ifnet *ifp, nvlist_t *nvl) 905 { 906 struct ovpn_softc *sc = ifp->if_softc; 907 struct ovpn_kpeer *peer; 908 struct ovpn_kkey tmpkey; 909 int error; 910 911 if (nvl == NULL) 912 return (EINVAL); 913 914 if (! nvlist_exists_number(nvl, "peerid")) 915 return (EINVAL); 916 917 OVPN_WLOCK(sc); 918 919 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 920 if (peer == NULL) { 921 OVPN_WUNLOCK(sc); 922 return (ENOENT); 923 } 924 925 /* Check that we have a second key to swap to. */ 926 error = ovpn_check_key(sc, peer, OVPN_KEY_SLOT_SECONDARY); 927 if (error) { 928 OVPN_WUNLOCK(sc); 929 return (error); 930 } 931 932 tmpkey = peer->keys[0]; 933 peer->keys[0] = peer->keys[1]; 934 peer->keys[1] = tmpkey; 935 936 OVPN_WUNLOCK(sc); 937 938 return (0); 939 } 940 941 static int 942 ovpn_del_key(struct ifnet *ifp, const nvlist_t *nvl) 943 { 944 enum ovpn_key_slot slot; 945 struct ovpn_kpeer *peer; 946 struct ovpn_softc *sc = ifp->if_softc; 947 948 if (nvl == NULL) 949 return (EINVAL); 950 951 if (! nvlist_exists_number(nvl, "peerid")) 952 return (EINVAL); 953 954 if (! nvlist_exists_number(nvl, "slot")) 955 return (EINVAL); 956 slot = nvlist_get_number(nvl, "slot"); 957 958 if (slot != OVPN_KEY_SLOT_PRIMARY && 959 slot != OVPN_KEY_SLOT_SECONDARY) 960 return (EINVAL); 961 962 OVPN_WLOCK(sc); 963 964 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 965 if (peer == NULL) { 966 OVPN_WUNLOCK(sc); 967 return (ENOENT); 968 } 969 970 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 971 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 972 973 peer->keys[slot].encrypt = NULL; 974 peer->keys[slot].decrypt = NULL; 975 976 peer->keys[slot].keyid = 0; 977 peer->keys[slot].peerid = 0; 978 979 OVPN_WUNLOCK(sc); 980 981 return (0); 982 } 983 984 static void 985 ovpn_send_ping(void *arg) 986 { 987 static const uint8_t ping_str[] = { 988 0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb, 989 0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48 990 }; 991 992 struct epoch_tracker et; 993 struct ovpn_kpeer *peer = arg; 994 struct ovpn_softc *sc = peer->sc; 995 struct mbuf *m; 996 997 OVPN_RASSERT(sc); 998 999 /* Ensure we repeat! */ 1000 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 1001 ovpn_send_ping, peer); 1002 1003 m = m_get2(sizeof(ping_str), M_NOWAIT, MT_DATA, M_PKTHDR); 1004 if (m == NULL) 1005 return; 1006 1007 m_copyback(m, 0, sizeof(ping_str), ping_str); 1008 m->m_len = m->m_pkthdr.len = sizeof(ping_str); 1009 1010 CURVNET_SET(sc->ifp->if_vnet); 1011 NET_EPOCH_ENTER(et); 1012 (void)ovpn_transmit_to_peer(sc->ifp, m, peer, NULL); 1013 NET_EPOCH_EXIT(et); 1014 CURVNET_RESTORE(); 1015 } 1016 1017 static void 1018 ovpn_timeout(void *arg) 1019 { 1020 struct ovpn_kpeer *peer = arg; 1021 struct ovpn_softc *sc = peer->sc; 1022 uint32_t last, _last_active; 1023 int ret __diagused; 1024 int cpu; 1025 1026 OVPN_WASSERT(sc); 1027 1028 last = 0; 1029 CPU_FOREACH(cpu) { 1030 _last_active = *zpcpu_get_cpu(peer->last_active, cpu); 1031 if (_last_active > last) 1032 last = _last_active; 1033 } 1034 1035 if (last + peer->keepalive.timeout > time_uptime) { 1036 callout_reset(&peer->ping_rcv, 1037 (peer->keepalive.timeout - (time_uptime - last)) * hz, 1038 ovpn_timeout, peer); 1039 return; 1040 } 1041 1042 CURVNET_SET(sc->ifp->if_vnet); 1043 peer->del_reason = OVPN_DEL_REASON_TIMEOUT; 1044 ret = _ovpn_del_peer(sc, peer); 1045 MPASS(ret == 0); 1046 CURVNET_RESTORE(); 1047 } 1048 1049 static int 1050 ovpn_set_peer(struct ifnet *ifp, const nvlist_t *nvl) 1051 { 1052 struct ovpn_softc *sc = ifp->if_softc; 1053 struct ovpn_kpeer *peer; 1054 1055 if (nvl == NULL) 1056 return (EINVAL); 1057 1058 if (! nvlist_exists_number(nvl, "interval") || 1059 ! nvlist_exists_number(nvl, "timeout") || 1060 ! nvlist_exists_number(nvl, "peerid")) 1061 return (EINVAL); 1062 1063 OVPN_WLOCK(sc); 1064 1065 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 1066 if (peer == NULL) { 1067 OVPN_WUNLOCK(sc); 1068 return (ENOENT); 1069 } 1070 1071 peer->keepalive.interval = nvlist_get_number(nvl, "interval"); 1072 peer->keepalive.timeout = nvlist_get_number(nvl, "timeout"); 1073 1074 if (peer->keepalive.interval > 0) 1075 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 1076 ovpn_send_ping, peer); 1077 if (peer->keepalive.timeout > 0) 1078 callout_reset(&peer->ping_rcv, peer->keepalive.timeout * hz, 1079 ovpn_timeout, peer); 1080 1081 OVPN_WUNLOCK(sc); 1082 1083 return (0); 1084 } 1085 1086 static int 1087 ovpn_set_ifmode(struct ifnet *ifp, const nvlist_t *nvl) 1088 { 1089 struct ovpn_softc *sc = ifp->if_softc; 1090 int ifmode; 1091 1092 if (nvl == NULL) 1093 return (EINVAL); 1094 1095 if (! nvlist_exists_number(nvl, "ifmode") ) 1096 return (EINVAL); 1097 1098 ifmode = nvlist_get_number(nvl, "ifmode"); 1099 1100 OVPN_WLOCK(sc); 1101 1102 /* deny this if UP */ 1103 if (ifp->if_flags & IFF_UP) { 1104 OVPN_WUNLOCK(sc); 1105 return (EBUSY); 1106 } 1107 1108 switch (ifmode & ~IFF_MULTICAST) { 1109 case IFF_POINTOPOINT: 1110 case IFF_BROADCAST: 1111 ifp->if_flags &= 1112 ~(IFF_BROADCAST|IFF_POINTOPOINT|IFF_MULTICAST); 1113 ifp->if_flags |= ifmode; 1114 break; 1115 default: 1116 OVPN_WUNLOCK(sc); 1117 return (EINVAL); 1118 } 1119 1120 OVPN_WUNLOCK(sc); 1121 1122 return (0); 1123 } 1124 1125 static int 1126 ovpn_ioctl_set(struct ifnet *ifp, struct ifdrv *ifd) 1127 { 1128 struct ovpn_softc *sc = ifp->if_softc; 1129 uint8_t *buf = NULL; 1130 nvlist_t *nvl = NULL; 1131 int ret; 1132 1133 if (ifd->ifd_len != 0) { 1134 if (ifd->ifd_len > OVPN_MAX_REQUEST_SIZE) 1135 return (E2BIG); 1136 1137 buf = malloc(ifd->ifd_len, M_OVPN, M_WAITOK); 1138 1139 ret = copyin(ifd->ifd_data, buf, ifd->ifd_len); 1140 if (ret != 0) { 1141 free(buf, M_OVPN); 1142 return (ret); 1143 } 1144 1145 nvl = nvlist_unpack(buf, ifd->ifd_len, 0); 1146 free(buf, M_OVPN); 1147 if (nvl == NULL) { 1148 return (EINVAL); 1149 } 1150 } 1151 1152 switch (ifd->ifd_cmd) { 1153 case OVPN_NEW_PEER: 1154 ret = ovpn_new_peer(ifp, nvl); 1155 break; 1156 case OVPN_DEL_PEER: 1157 OVPN_WLOCK(sc); 1158 ret = ovpn_del_peer(ifp, nvl); 1159 OVPN_WUNLOCK(sc); 1160 break; 1161 case OVPN_NEW_KEY: 1162 ret = ovpn_set_key(ifp, nvl); 1163 break; 1164 case OVPN_START_VPN: 1165 ret = ovpn_start(ifp); 1166 break; 1167 case OVPN_SWAP_KEYS: 1168 ret = ovpn_swap_keys(ifp, nvl); 1169 break; 1170 case OVPN_DEL_KEY: 1171 ret = ovpn_del_key(ifp, nvl); 1172 break; 1173 case OVPN_SET_PEER: 1174 ret = ovpn_set_peer(ifp, nvl); 1175 break; 1176 case OVPN_SET_IFMODE: 1177 ret = ovpn_set_ifmode(ifp, nvl); 1178 break; 1179 default: 1180 ret = ENOTSUP; 1181 } 1182 1183 nvlist_destroy(nvl); 1184 return (ret); 1185 } 1186 1187 static int 1188 ovpn_add_counters(nvlist_t *parent, const char *name, counter_u64_t in, 1189 counter_u64_t out) 1190 { 1191 nvlist_t *nvl; 1192 1193 nvl = nvlist_create(0); 1194 if (nvl == NULL) 1195 return (ENOMEM); 1196 1197 nvlist_add_number(nvl, "in", counter_u64_fetch(in)); 1198 nvlist_add_number(nvl, "out", counter_u64_fetch(out)); 1199 1200 nvlist_add_nvlist(parent, name, nvl); 1201 1202 nvlist_destroy(nvl); 1203 1204 return (0); 1205 } 1206 1207 static int 1208 ovpn_get_stats(struct ovpn_softc *sc, nvlist_t **onvl) 1209 { 1210 nvlist_t *nvl; 1211 int ret; 1212 1213 nvl = nvlist_create(0); 1214 if (nvl == NULL) 1215 return (ENOMEM); 1216 1217 #define OVPN_COUNTER_OUT(name, in, out) \ 1218 do { \ 1219 ret = ovpn_add_counters(nvl, name, OVPN_COUNTER(sc, in), \ 1220 OVPN_COUNTER(sc, out)); \ 1221 if (ret != 0) \ 1222 goto error; \ 1223 } while(0) 1224 1225 OVPN_COUNTER_OUT("lost_ctrl", lost_ctrl_pkts_in, lost_ctrl_pkts_out); 1226 OVPN_COUNTER_OUT("lost_data", lost_data_pkts_in, lost_data_pkts_out); 1227 OVPN_COUNTER_OUT("nomem_data", nomem_data_pkts_in, 1228 nomem_data_pkts_out); 1229 OVPN_COUNTER_OUT("data", received_data_pkts, sent_data_pkts); 1230 OVPN_COUNTER_OUT("ctrl", received_ctrl_pkts, sent_ctrl_pkts); 1231 OVPN_COUNTER_OUT("tunnel", tunnel_bytes_received, 1232 tunnel_bytes_received); 1233 OVPN_COUNTER_OUT("transport", transport_bytes_received, 1234 transport_bytes_received); 1235 #undef OVPN_COUNTER_OUT 1236 1237 *onvl = nvl; 1238 1239 return (0); 1240 1241 error: 1242 nvlist_destroy(nvl); 1243 return (ret); 1244 } 1245 1246 static int 1247 ovpn_get_peer_stats(struct ovpn_softc *sc, nvlist_t **nvl) 1248 { 1249 struct ovpn_kpeer *peer; 1250 nvlist_t *nvpeer = NULL; 1251 int ret; 1252 1253 OVPN_RLOCK_TRACKER; 1254 1255 *nvl = nvlist_create(0); 1256 if (*nvl == NULL) 1257 return (ENOMEM); 1258 1259 #define OVPN_PEER_COUNTER_OUT(name, in, out) \ 1260 do { \ 1261 ret = ovpn_add_counters(nvpeer, name, \ 1262 OVPN_PEER_COUNTER(peer, in), OVPN_PEER_COUNTER(peer, out)); \ 1263 if (ret != 0) \ 1264 goto error; \ 1265 } while(0) 1266 1267 OVPN_RLOCK(sc); 1268 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1269 nvpeer = nvlist_create(0); 1270 if (nvpeer == NULL) { 1271 OVPN_RUNLOCK(sc); 1272 nvlist_destroy(*nvl); 1273 *nvl = NULL; 1274 return (ENOMEM); 1275 } 1276 1277 nvlist_add_number(nvpeer, "peerid", peer->peerid); 1278 1279 OVPN_PEER_COUNTER_OUT("packets", pkt_in, pkt_out); 1280 OVPN_PEER_COUNTER_OUT("bytes", bytes_in, bytes_out); 1281 1282 nvlist_append_nvlist_array(*nvl, "peers", nvpeer); 1283 nvlist_destroy(nvpeer); 1284 } 1285 #undef OVPN_PEER_COUNTER_OUT 1286 OVPN_RUNLOCK(sc); 1287 1288 return (0); 1289 1290 error: 1291 nvlist_destroy(nvpeer); 1292 nvlist_destroy(*nvl); 1293 *nvl = NULL; 1294 return (ret); 1295 } 1296 1297 static int 1298 ovpn_poll_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1299 { 1300 nvlist_t *nvl; 1301 1302 nvl = nvlist_create(0); 1303 if (nvl == NULL) 1304 return (ENOMEM); 1305 1306 nvlist_add_number(nvl, "pending", buf_ring_count(sc->notifring)); 1307 1308 *onvl = nvl; 1309 1310 return (0); 1311 } 1312 1313 static void 1314 ovpn_notif_add_counters(nvlist_t *parent, struct ovpn_notification *n) 1315 { 1316 nvlist_t *nvl; 1317 1318 nvl = nvlist_create(0); 1319 if (nvl == NULL) 1320 return; 1321 1322 nvlist_add_number(nvl, "in", n->counters.pkt_in); 1323 nvlist_add_number(nvl, "out", n->counters.pkt_out); 1324 1325 nvlist_add_nvlist(parent, "packets", nvl); 1326 nvlist_destroy(nvl); 1327 1328 nvl = nvlist_create(0); 1329 if (nvl == NULL) 1330 return; 1331 1332 nvlist_add_number(nvl, "in", n->counters.bytes_in); 1333 nvlist_add_number(nvl, "out", n->counters.bytes_out); 1334 1335 nvlist_add_nvlist(parent, "bytes", nvl); 1336 nvlist_destroy(nvl); 1337 } 1338 1339 static int 1340 opvn_get_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1341 { 1342 struct ovpn_notification *n; 1343 nvlist_t *nvl; 1344 1345 /* Check if we have notifications pending. */ 1346 n = buf_ring_dequeue_mc(sc->notifring); 1347 if (n == NULL) 1348 return (ENOENT); 1349 1350 nvl = nvlist_create(0); 1351 if (nvl == NULL) { 1352 free(n, M_OVPN); 1353 return (ENOMEM); 1354 } 1355 nvlist_add_number(nvl, "peerid", n->peerid); 1356 nvlist_add_number(nvl, "notification", n->type); 1357 if (n->type == OVPN_NOTIF_DEL_PEER) { 1358 nvlist_add_number(nvl, "del_reason", n->del_reason); 1359 1360 /* No error handling, because we want to send the notification 1361 * even if we can't attach the counters. */ 1362 ovpn_notif_add_counters(nvl, n); 1363 } 1364 free(n, M_OVPN); 1365 1366 *onvl = nvl; 1367 1368 return (0); 1369 } 1370 1371 static int 1372 ovpn_ioctl_get(struct ifnet *ifp, struct ifdrv *ifd) 1373 { 1374 struct ovpn_softc *sc = ifp->if_softc; 1375 nvlist_t *nvl = NULL; 1376 int error; 1377 1378 switch (ifd->ifd_cmd) { 1379 case OVPN_GET_STATS: 1380 error = ovpn_get_stats(sc, &nvl); 1381 break; 1382 case OVPN_GET_PEER_STATS: 1383 error = ovpn_get_peer_stats(sc, &nvl); 1384 break; 1385 case OVPN_POLL_PKT: 1386 error = ovpn_poll_pkt(sc, &nvl); 1387 break; 1388 case OVPN_GET_PKT: 1389 error = opvn_get_pkt(sc, &nvl); 1390 break; 1391 default: 1392 error = ENOTSUP; 1393 break; 1394 } 1395 1396 if (error == 0) { 1397 void *packed = NULL; 1398 size_t len; 1399 1400 MPASS(nvl != NULL); 1401 1402 packed = nvlist_pack(nvl, &len); 1403 if (! packed) { 1404 nvlist_destroy(nvl); 1405 return (ENOMEM); 1406 } 1407 1408 if (len > ifd->ifd_len) { 1409 free(packed, M_NVLIST); 1410 nvlist_destroy(nvl); 1411 return (ENOSPC); 1412 } 1413 1414 error = copyout(packed, ifd->ifd_data, len); 1415 ifd->ifd_len = len; 1416 1417 free(packed, M_NVLIST); 1418 nvlist_destroy(nvl); 1419 } 1420 1421 return (error); 1422 } 1423 1424 static int 1425 ovpn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 1426 { 1427 struct ifdrv *ifd; 1428 int error; 1429 1430 CURVNET_ASSERT_SET(); 1431 1432 switch (cmd) { 1433 case SIOCSDRVSPEC: 1434 case SIOCGDRVSPEC: 1435 error = priv_check(curthread, PRIV_NET_OVPN); 1436 if (error) 1437 return (error); 1438 break; 1439 } 1440 1441 switch (cmd) { 1442 case SIOCSDRVSPEC: 1443 ifd = (struct ifdrv *)data; 1444 error = ovpn_ioctl_set(ifp, ifd); 1445 break; 1446 case SIOCGDRVSPEC: 1447 ifd = (struct ifdrv *)data; 1448 error = ovpn_ioctl_get(ifp, ifd); 1449 break; 1450 case SIOCSIFMTU: { 1451 struct ifreq *ifr = (struct ifreq *)data; 1452 if (ifr->ifr_mtu < OVPN_MTU_MIN || ifr->ifr_mtu > OVPN_MTU_MAX) 1453 return (EINVAL); 1454 1455 ifp->if_mtu = ifr->ifr_mtu; 1456 return (0); 1457 } 1458 case SIOCSIFADDR: 1459 case SIOCADDMULTI: 1460 case SIOCDELMULTI: 1461 case SIOCGIFMTU: 1462 case SIOCSIFFLAGS: 1463 return (0); 1464 default: 1465 error = EINVAL; 1466 } 1467 1468 return (error); 1469 } 1470 1471 static int 1472 ovpn_encrypt_tx_cb(struct cryptop *crp) 1473 { 1474 struct epoch_tracker et; 1475 struct ovpn_kpeer *peer = crp->crp_opaque; 1476 struct ovpn_softc *sc = peer->sc; 1477 struct mbuf *m = crp->crp_buf.cb_mbuf; 1478 int tunnel_len; 1479 int ret; 1480 1481 CURVNET_SET(sc->ifp->if_vnet); 1482 NET_EPOCH_ENTER(et); 1483 1484 if (crp->crp_etype != 0) { 1485 crypto_freereq(crp); 1486 ovpn_peer_release_ref(peer, false); 1487 NET_EPOCH_EXIT(et); 1488 CURVNET_RESTORE(); 1489 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1490 m_freem(m); 1491 return (0); 1492 } 1493 1494 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1495 1496 tunnel_len = m->m_pkthdr.len - sizeof(struct ovpn_wire_header); 1497 ret = ovpn_encap(sc, peer->peerid, m); 1498 if (ret == 0) { 1499 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1500 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 1501 } 1502 1503 crypto_freereq(crp); 1504 ovpn_peer_release_ref(peer, false); 1505 1506 NET_EPOCH_EXIT(et); 1507 CURVNET_RESTORE(); 1508 1509 return (0); 1510 } 1511 1512 static void 1513 ovpn_finish_rx(struct ovpn_softc *sc, struct mbuf *m, 1514 struct ovpn_kpeer *peer, struct ovpn_kkey *key, uint32_t seq, 1515 struct rm_priotracker *_ovpn_lock_trackerp) 1516 { 1517 uint32_t af; 1518 1519 OVPN_RASSERT(sc); 1520 NET_EPOCH_ASSERT(); 1521 1522 /* Replay protection. */ 1523 if (V_replay_protection && ! ovpn_check_replay(key->decrypt, seq)) { 1524 OVPN_RUNLOCK(sc); 1525 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1526 m_freem(m); 1527 return; 1528 } 1529 1530 critical_enter(); 1531 *zpcpu_get(peer->last_active) = time_uptime; 1532 critical_exit(); 1533 1534 OVPN_RUNLOCK(sc); 1535 1536 OVPN_COUNTER_ADD(sc, received_data_pkts, 1); 1537 OVPN_COUNTER_ADD(sc, tunnel_bytes_received, m->m_pkthdr.len); 1538 OVPN_PEER_COUNTER_ADD(peer, pkt_in, 1); 1539 OVPN_PEER_COUNTER_ADD(peer, bytes_in, m->m_pkthdr.len); 1540 1541 /* Receive the packet on our interface. */ 1542 m->m_pkthdr.rcvif = sc->ifp; 1543 1544 /* Clear checksum flags in case the real hardware set them. */ 1545 m->m_pkthdr.csum_flags = 0; 1546 1547 /* Clear mbuf tags & flags */ 1548 m_tag_delete_nonpersistent(m); 1549 m_clrprotoflags(m); 1550 1551 /* Ensure we can read the first byte. */ 1552 m = m_pullup(m, 1); 1553 if (m == NULL) { 1554 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 1555 return; 1556 } 1557 1558 /* 1559 * Check for address family, and disregard any control packets (e.g. 1560 * keepalive). 1561 */ 1562 af = ovpn_get_af(m); 1563 if (af != 0) { 1564 BPF_MTAP2(sc->ifp, &af, sizeof(af), m); 1565 if (V_async_netisr_queue) 1566 netisr_queue(af == AF_INET ? NETISR_IP : NETISR_IPV6, m); 1567 else 1568 netisr_dispatch(af == AF_INET ? NETISR_IP : NETISR_IPV6, m); 1569 } else { 1570 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1571 m_freem(m); 1572 } 1573 } 1574 1575 static struct ovpn_kkey * 1576 ovpn_find_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, 1577 const struct ovpn_wire_header *ohdr) 1578 { 1579 struct ovpn_kkey *key = NULL; 1580 uint8_t keyid; 1581 1582 OVPN_RASSERT(sc); 1583 1584 keyid = (ntohl(ohdr->opcode) >> 24) & 0x07; 1585 1586 if (peer->keys[0].keyid == keyid) 1587 key = &peer->keys[0]; 1588 else if (peer->keys[1].keyid == keyid) 1589 key = &peer->keys[1]; 1590 1591 return (key); 1592 } 1593 1594 static int 1595 ovpn_decrypt_rx_cb(struct cryptop *crp) 1596 { 1597 struct epoch_tracker et; 1598 struct ovpn_softc *sc = crp->crp_opaque; 1599 struct mbuf *m = crp->crp_buf.cb_mbuf; 1600 struct ovpn_kkey *key; 1601 struct ovpn_kpeer *peer; 1602 struct ovpn_wire_header *ohdr; 1603 uint32_t peerid; 1604 1605 OVPN_RLOCK_TRACKER; 1606 1607 OVPN_RLOCK(sc); 1608 1609 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1610 1611 if (crp->crp_etype != 0) { 1612 crypto_freereq(crp); 1613 atomic_add_int(&sc->refcount, -1); 1614 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1615 OVPN_RUNLOCK(sc); 1616 m_freem(m); 1617 return (0); 1618 } 1619 1620 CURVNET_SET(sc->ifp->if_vnet); 1621 1622 ohdr = mtodo(m, sizeof(struct udphdr)); 1623 1624 peerid = ntohl(ohdr->opcode) & 0x00ffffff; 1625 peer = ovpn_find_peer(sc, peerid); 1626 if (peer == NULL) { 1627 /* No such peer. Drop packet. */ 1628 crypto_freereq(crp); 1629 atomic_add_int(&sc->refcount, -1); 1630 OVPN_RUNLOCK(sc); 1631 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1632 m_freem(m); 1633 CURVNET_RESTORE(); 1634 return (0); 1635 } 1636 1637 key = ovpn_find_key(sc, peer, ohdr); 1638 if (key == NULL) { 1639 crypto_freereq(crp); 1640 atomic_add_int(&sc->refcount, -1); 1641 /* 1642 * Has this key been removed between us starting the decrypt 1643 * and finishing it? 1644 */ 1645 OVPN_RUNLOCK(sc); 1646 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1647 m_freem(m); 1648 CURVNET_RESTORE(); 1649 return (0); 1650 } 1651 1652 /* Now remove the outer headers */ 1653 m_adj_decap(m, sizeof(struct udphdr) + 1654 sizeof(struct ovpn_wire_header)); 1655 1656 NET_EPOCH_ENTER(et); 1657 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), _ovpn_lock_trackerp); 1658 NET_EPOCH_EXIT(et); 1659 OVPN_UNLOCK_ASSERT(sc); 1660 1661 CURVNET_RESTORE(); 1662 1663 crypto_freereq(crp); 1664 atomic_add_int(&sc->refcount, -1); 1665 1666 return (0); 1667 } 1668 1669 static int 1670 ovpn_get_af(struct mbuf *m) 1671 { 1672 struct ip *ip; 1673 struct ip6_hdr *ip6; 1674 1675 /* 1676 * We should pullup, but we're only interested in the first byte, so 1677 * that'll always be contiguous. 1678 */ 1679 ip = mtod(m, struct ip *); 1680 if (ip->ip_v == IPVERSION) 1681 return (AF_INET); 1682 1683 ip6 = mtod(m, struct ip6_hdr *); 1684 if ((ip6->ip6_vfc & IPV6_VERSION_MASK) == IPV6_VERSION) 1685 return (AF_INET6); 1686 1687 return (0); 1688 } 1689 1690 #ifdef INET 1691 static struct ovpn_kpeer * 1692 ovpn_find_peer_by_ip(struct ovpn_softc *sc, const struct in_addr addr) 1693 { 1694 struct ovpn_kpeer *peer = NULL; 1695 1696 OVPN_ASSERT(sc); 1697 1698 /* TODO: Add a second RB so we can look up by IP. */ 1699 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1700 if (addr.s_addr == peer->vpn4.s_addr) 1701 return (peer); 1702 } 1703 1704 return (peer); 1705 } 1706 #endif 1707 1708 #ifdef INET6 1709 static struct ovpn_kpeer * 1710 ovpn_find_peer_by_ip6(struct ovpn_softc *sc, const struct in6_addr *addr) 1711 { 1712 struct ovpn_kpeer *peer = NULL; 1713 1714 OVPN_ASSERT(sc); 1715 1716 /* TODO: Add a third RB so we can look up by IPv6 address. */ 1717 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1718 if (memcmp(addr, &peer->vpn6, sizeof(*addr)) == 0) 1719 return (peer); 1720 } 1721 1722 return (peer); 1723 } 1724 #endif 1725 1726 static struct ovpn_kpeer * 1727 ovpn_route_peer(struct ovpn_softc *sc, struct mbuf **m0, 1728 const struct sockaddr *dst) 1729 { 1730 struct ovpn_kpeer *peer = NULL; 1731 int af; 1732 1733 NET_EPOCH_ASSERT(); 1734 OVPN_ASSERT(sc); 1735 1736 /* Shortcut if we're a client (or are a server and have only one client). */ 1737 if (sc->peercount == 1) 1738 return (ovpn_find_only_peer(sc)); 1739 1740 if (dst != NULL) 1741 af = dst->sa_family; 1742 else 1743 af = ovpn_get_af(*m0); 1744 1745 switch (af) { 1746 #ifdef INET 1747 case AF_INET: { 1748 const struct sockaddr_in *sa = (const struct sockaddr_in *)dst; 1749 struct nhop_object *nh; 1750 const struct in_addr *ip_dst; 1751 1752 if (sa != NULL) { 1753 ip_dst = &sa->sin_addr; 1754 } else { 1755 struct ip *ip; 1756 1757 *m0 = m_pullup(*m0, sizeof(struct ip)); 1758 if (*m0 == NULL) 1759 return (NULL); 1760 ip = mtod(*m0, struct ip *); 1761 ip_dst = &ip->ip_dst; 1762 } 1763 1764 peer = ovpn_find_peer_by_ip(sc, *ip_dst); 1765 SDT_PROBE2(if_ovpn, tx, route, ip4, ip_dst, peer); 1766 if (peer == NULL) { 1767 nh = fib4_lookup(M_GETFIB(*m0), *ip_dst, 0, 1768 NHR_NONE, 0); 1769 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1770 peer = ovpn_find_peer_by_ip(sc, 1771 nh->gw4_sa.sin_addr); 1772 SDT_PROBE2(if_ovpn, tx, route, ip4, 1773 &nh->gw4_sa.sin_addr, peer); 1774 } 1775 } 1776 break; 1777 } 1778 #endif 1779 #ifdef INET6 1780 case AF_INET6: { 1781 const struct sockaddr_in6 *sa6 = 1782 (const struct sockaddr_in6 *)dst; 1783 struct nhop_object *nh; 1784 const struct in6_addr *ip6_dst; 1785 1786 if (sa6 != NULL) { 1787 ip6_dst = &sa6->sin6_addr; 1788 } else { 1789 struct ip6_hdr *ip6; 1790 1791 *m0 = m_pullup(*m0, sizeof(struct ip6_hdr)); 1792 if (*m0 == NULL) 1793 return (NULL); 1794 ip6 = mtod(*m0, struct ip6_hdr *); 1795 ip6_dst = &ip6->ip6_dst; 1796 } 1797 1798 peer = ovpn_find_peer_by_ip6(sc, ip6_dst); 1799 SDT_PROBE2(if_ovpn, tx, route, ip6, ip6_dst, peer); 1800 if (peer == NULL) { 1801 nh = fib6_lookup(M_GETFIB(*m0), ip6_dst, 0, 1802 NHR_NONE, 0); 1803 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1804 peer = ovpn_find_peer_by_ip6(sc, 1805 &nh->gw6_sa.sin6_addr); 1806 SDT_PROBE2(if_ovpn, tx, route, ip6, 1807 &nh->gw6_sa.sin6_addr, peer); 1808 } 1809 } 1810 break; 1811 } 1812 #endif 1813 } 1814 1815 return (peer); 1816 } 1817 1818 static int 1819 ovpn_transmit(struct ifnet *ifp, struct mbuf *m) 1820 { 1821 return (ifp->if_output(ifp, m, NULL, NULL)); 1822 } 1823 1824 static int 1825 ovpn_transmit_to_peer(struct ifnet *ifp, struct mbuf *m, 1826 struct ovpn_kpeer *peer, struct rm_priotracker *_ovpn_lock_trackerp) 1827 { 1828 struct ovpn_wire_header *ohdr; 1829 struct ovpn_kkey *key; 1830 struct ovpn_softc *sc; 1831 struct cryptop *crp; 1832 uint32_t af, seq; 1833 uint64_t seq64; 1834 size_t len, ovpn_hdr_len; 1835 int tunnel_len; 1836 int ret; 1837 1838 sc = ifp->if_softc; 1839 1840 OVPN_RASSERT(sc); 1841 1842 tunnel_len = m->m_pkthdr.len; 1843 1844 key = &peer->keys[OVPN_KEY_SLOT_PRIMARY]; 1845 if (key->encrypt == NULL) { 1846 if (_ovpn_lock_trackerp != NULL) 1847 OVPN_RUNLOCK(sc); 1848 m_freem(m); 1849 return (ENOLINK); 1850 } 1851 1852 af = ovpn_get_af(m); 1853 /* Don't capture control packets. */ 1854 if (af != 0) 1855 BPF_MTAP2(ifp, &af, sizeof(af), m); 1856 1857 len = m->m_pkthdr.len; 1858 MPASS(len <= ifp->if_mtu); 1859 1860 ovpn_hdr_len = sizeof(struct ovpn_wire_header); 1861 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) 1862 ovpn_hdr_len -= 16; /* No auth tag. */ 1863 1864 M_PREPEND(m, ovpn_hdr_len, M_NOWAIT); 1865 if (m == NULL) { 1866 if (_ovpn_lock_trackerp != NULL) 1867 OVPN_RUNLOCK(sc); 1868 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1869 return (ENOBUFS); 1870 } 1871 ohdr = mtod(m, struct ovpn_wire_header *); 1872 ohdr->opcode = (OVPN_OP_DATA_V2 << OVPN_OP_SHIFT) | key->keyid; 1873 ohdr->opcode <<= 24; 1874 ohdr->opcode |= key->peerid; 1875 ohdr->opcode = htonl(ohdr->opcode); 1876 1877 seq64 = atomic_fetchadd_64(&peer->keys[OVPN_KEY_SLOT_PRIMARY].encrypt->tx_seq, 1); 1878 if (seq64 == OVPN_SEQ_ROTATE) { 1879 ovpn_notify_key_rotation(sc, peer); 1880 } else if (seq64 > UINT32_MAX) { 1881 /* We've wrapped, give up on this packet. */ 1882 if (_ovpn_lock_trackerp != NULL) 1883 OVPN_RUNLOCK(sc); 1884 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1885 1886 /* Let's avoid (very unlikely, but still) wraparounds of the 1887 * 64-bit counter taking us back to 0. */ 1888 atomic_store_64(&peer->keys[OVPN_KEY_SLOT_PRIMARY].encrypt->tx_seq, 1889 UINT32_MAX); 1890 1891 return (ENOBUFS); 1892 } 1893 1894 seq = htonl(seq64 & UINT32_MAX); 1895 ohdr->seq = seq; 1896 1897 OVPN_PEER_COUNTER_ADD(peer, pkt_out, 1); 1898 OVPN_PEER_COUNTER_ADD(peer, bytes_out, len); 1899 1900 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) { 1901 ret = ovpn_encap(sc, peer->peerid, m); 1902 if (_ovpn_lock_trackerp != NULL) 1903 OVPN_RUNLOCK(sc); 1904 if (ret == 0) { 1905 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1906 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 1907 } 1908 return (ret); 1909 } 1910 1911 crp = crypto_getreq(key->encrypt->cryptoid, M_NOWAIT); 1912 if (crp == NULL) { 1913 if (_ovpn_lock_trackerp != NULL) 1914 OVPN_RUNLOCK(sc); 1915 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1916 m_freem(m); 1917 return (ENOBUFS); 1918 } 1919 1920 /* Encryption covers only the payload, not the header. */ 1921 crp->crp_payload_start = sizeof(*ohdr); 1922 crp->crp_payload_length = len; 1923 crp->crp_op = CRYPTO_OP_ENCRYPT; 1924 1925 /* 1926 * AAD data covers the ovpn_wire_header minus the auth 1927 * tag. 1928 */ 1929 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 1930 crp->crp_aad = ohdr; 1931 crp->crp_aad_start = 0; 1932 crp->crp_op |= CRYPTO_OP_COMPUTE_DIGEST; 1933 crp->crp_digest_start = offsetof(struct ovpn_wire_header, auth_tag); 1934 1935 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 1936 memcpy(crp->crp_iv, &seq, sizeof(seq)); 1937 memcpy(crp->crp_iv + sizeof(seq), key->encrypt->nonce, 1938 key->encrypt->noncelen); 1939 1940 crypto_use_mbuf(crp, m); 1941 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 1942 crp->crp_callback = ovpn_encrypt_tx_cb; 1943 crp->crp_opaque = peer; 1944 1945 atomic_add_int(&peer->refcount, 1); 1946 if (_ovpn_lock_trackerp != NULL) 1947 OVPN_RUNLOCK(sc); 1948 if (V_async_crypto) 1949 ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); 1950 else 1951 ret = crypto_dispatch(crp); 1952 if (ret) { 1953 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1954 } 1955 1956 return (ret); 1957 } 1958 1959 /* 1960 * Note: Expects to hold the read lock on entry, and will release it itself. 1961 */ 1962 static int 1963 ovpn_encap(struct ovpn_softc *sc, uint32_t peerid, struct mbuf *m) 1964 { 1965 struct udphdr *udp; 1966 struct ovpn_kpeer *peer; 1967 int len; 1968 1969 OVPN_RLOCK_TRACKER; 1970 1971 OVPN_RLOCK(sc); 1972 NET_EPOCH_ASSERT(); 1973 1974 peer = ovpn_find_peer(sc, peerid); 1975 if (peer == NULL || sc->ifp->if_link_state != LINK_STATE_UP) { 1976 OVPN_RUNLOCK(sc); 1977 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1978 m_freem(m); 1979 return (ENETDOWN); 1980 } 1981 1982 len = m->m_pkthdr.len; 1983 1984 M_PREPEND(m, sizeof(struct udphdr), M_NOWAIT); 1985 if (m == NULL) { 1986 OVPN_RUNLOCK(sc); 1987 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1988 m_freem(m); 1989 return (ENOBUFS); 1990 } 1991 udp = mtod(m, struct udphdr *); 1992 1993 MPASS(peer->local.ss_family == peer->remote.ss_family); 1994 1995 udp->uh_sport = ovpn_get_port(&peer->local); 1996 udp->uh_dport = ovpn_get_port(&peer->remote); 1997 udp->uh_ulen = htons(sizeof(struct udphdr) + len); 1998 1999 switch (peer->remote.ss_family) { 2000 #ifdef INET 2001 case AF_INET: { 2002 struct sockaddr_in *in_local = TO_IN(&peer->local); 2003 struct sockaddr_in *in_remote = TO_IN(&peer->remote); 2004 struct ip *ip; 2005 2006 /* 2007 * This requires knowing the source IP, which we don't. Happily 2008 * we're allowed to keep this at 0, and the checksum won't do 2009 * anything the crypto won't already do. 2010 */ 2011 udp->uh_sum = 0; 2012 2013 /* Set the checksum flags so we recalculate checksums. */ 2014 m->m_pkthdr.csum_flags |= CSUM_IP; 2015 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 2016 2017 M_PREPEND(m, sizeof(struct ip), M_NOWAIT); 2018 if (m == NULL) { 2019 OVPN_RUNLOCK(sc); 2020 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2021 return (ENOBUFS); 2022 } 2023 ip = mtod(m, struct ip *); 2024 2025 ip->ip_tos = 0; 2026 ip->ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) + 2027 len); 2028 ip->ip_off = 0; 2029 ip->ip_ttl = V_ip_defttl; 2030 ip->ip_p = IPPROTO_UDP; 2031 ip->ip_sum = 0; 2032 if (in_local->sin_port != 0) 2033 ip->ip_src = in_local->sin_addr; 2034 else 2035 ip->ip_src.s_addr = INADDR_ANY; 2036 ip->ip_dst = in_remote->sin_addr; 2037 2038 OVPN_RUNLOCK(sc); 2039 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 2040 2041 return (ip_output(m, NULL, NULL, 0, NULL, NULL)); 2042 } 2043 #endif 2044 #ifdef INET6 2045 case AF_INET6: { 2046 struct sockaddr_in6 *in6_local = TO_IN6(&peer->local); 2047 struct sockaddr_in6 *in6_remote = TO_IN6(&peer->remote); 2048 struct ip6_hdr *ip6; 2049 2050 M_PREPEND(m, sizeof(struct ip6_hdr), M_NOWAIT); 2051 if (m == NULL) { 2052 OVPN_RUNLOCK(sc); 2053 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2054 return (ENOBUFS); 2055 } 2056 m = m_pullup(m, sizeof(*ip6) + sizeof(*udp)); 2057 if (m == NULL) { 2058 OVPN_RUNLOCK(sc); 2059 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2060 return (ENOBUFS); 2061 } 2062 2063 ip6 = mtod(m, struct ip6_hdr *); 2064 2065 ip6->ip6_vfc = IPV6_VERSION; 2066 ip6->ip6_flow &= ~IPV6_FLOWINFO_MASK; 2067 ip6->ip6_plen = htons(sizeof(*ip6) + sizeof(struct udphdr) + 2068 len); 2069 ip6->ip6_nxt = IPPROTO_UDP; 2070 ip6->ip6_hlim = V_ip6_defhlim; 2071 2072 memcpy(&ip6->ip6_src, &in6_local->sin6_addr, 2073 sizeof(ip6->ip6_src)); 2074 memcpy(&ip6->ip6_dst, &in6_remote->sin6_addr, 2075 sizeof(ip6->ip6_dst)); 2076 2077 udp = mtodo(m, sizeof(*ip6)); 2078 udp->uh_sum = in6_cksum_pseudo(ip6, 2079 m->m_pkthdr.len - sizeof(struct ip6_hdr), 2080 IPPROTO_UDP, 0); 2081 2082 m->m_pkthdr.csum_flags |= CSUM_UDP_IPV6; 2083 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 2084 2085 OVPN_RUNLOCK(sc); 2086 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 2087 2088 return (ip6_output(m, NULL, NULL, IPV6_UNSPECSRC, NULL, NULL, 2089 NULL)); 2090 } 2091 #endif 2092 default: 2093 panic("Unsupported address family %d", 2094 peer->remote.ss_family); 2095 } 2096 } 2097 2098 static int 2099 ovpn_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 2100 struct route *ro) 2101 { 2102 struct ovpn_softc *sc; 2103 struct ovpn_kpeer *peer; 2104 2105 OVPN_RLOCK_TRACKER; 2106 2107 sc = ifp->if_softc; 2108 2109 OVPN_RLOCK(sc); 2110 2111 SDT_PROBE1(if_ovpn, tx, transmit, start, m); 2112 2113 if (__predict_false(ifp->if_link_state != LINK_STATE_UP)) { 2114 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2115 OVPN_RUNLOCK(sc); 2116 m_freem(m); 2117 return (ENETDOWN); 2118 } 2119 2120 /** 2121 * Only obey 'dst' (i.e. the gateway) if no route is supplied. 2122 * That's our indication that we're being called through pf's route-to, 2123 * and we should route according to 'dst' instead. We can't do so 2124 * consistently, because the usual openvpn configuration sets the first 2125 * non-server IP in the subnet as the gateway. If we always use that 2126 * one we'd end up routing all traffic to the first client. 2127 * tl;dr: 'ro == NULL' tells us pf is doing a route-to, and then but 2128 * only then, we should treat 'dst' as the destination. */ 2129 peer = ovpn_route_peer(sc, &m, ro == NULL ? dst : NULL); 2130 if (peer == NULL) { 2131 /* No destination. */ 2132 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2133 OVPN_RUNLOCK(sc); 2134 m_freem(m); 2135 return (ENETDOWN); 2136 } 2137 2138 return (ovpn_transmit_to_peer(ifp, m, peer, _ovpn_lock_trackerp)); 2139 } 2140 2141 static bool 2142 ovpn_check_replay(struct ovpn_kkey_dir *key, uint32_t seq) 2143 { 2144 uint32_t d; 2145 2146 mtx_lock(&key->replay_mtx); 2147 2148 /* Sequence number must be strictly greater than rx_seq */ 2149 if (seq <= key->rx_seq) { 2150 mtx_unlock(&key->replay_mtx); 2151 return (false); 2152 } 2153 2154 /* Large jump. The packet authenticated okay, so just accept that. */ 2155 if (seq > (key->rx_seq + (sizeof(key->rx_window) * 8))) { 2156 key->rx_seq = seq; 2157 key->rx_window = 0; 2158 mtx_unlock(&key->replay_mtx); 2159 return (true); 2160 } 2161 2162 /* Happy case. */ 2163 if ((seq == key->rx_seq + 1) && key->rx_window == 0) { 2164 key->rx_seq++; 2165 mtx_unlock(&key->replay_mtx); 2166 return (true); 2167 } 2168 2169 d = seq - key->rx_seq - 1; 2170 2171 if (key->rx_window & ((uint64_t)1 << d)) { 2172 /* Dupe! */ 2173 mtx_unlock(&key->replay_mtx); 2174 return (false); 2175 } 2176 2177 key->rx_window |= (uint64_t)1 << d; 2178 2179 while (key->rx_window & 1) { 2180 key->rx_seq++; 2181 key->rx_window >>= 1; 2182 } 2183 2184 mtx_unlock(&key->replay_mtx); 2185 2186 return (true); 2187 } 2188 2189 static struct ovpn_kpeer * 2190 ovpn_peer_from_mbuf(struct ovpn_softc *sc, struct mbuf *m, int off) 2191 { 2192 struct ovpn_wire_header ohdr; 2193 uint32_t peerid; 2194 const size_t hdrlen = sizeof(ohdr) - sizeof(ohdr.auth_tag); 2195 2196 OVPN_RASSERT(sc); 2197 2198 if (m_length(m, NULL) < (off + sizeof(struct udphdr) + hdrlen)) 2199 return (NULL); 2200 2201 m_copydata(m, off + sizeof(struct udphdr), hdrlen, (caddr_t)&ohdr); 2202 2203 peerid = ntohl(ohdr.opcode) & 0x00ffffff; 2204 2205 return (ovpn_find_peer(sc, peerid)); 2206 } 2207 2208 static bool 2209 ovpn_udp_input(struct mbuf *m, int off, struct inpcb *inp, 2210 const struct sockaddr *sa, void *ctx) 2211 { 2212 struct ovpn_softc *sc = ctx; 2213 struct ovpn_wire_header tmphdr; 2214 struct ovpn_wire_header *ohdr; 2215 struct udphdr *uhdr; 2216 struct ovpn_kkey *key; 2217 struct cryptop *crp; 2218 struct ovpn_kpeer *peer; 2219 size_t ohdrlen; 2220 int ret; 2221 uint8_t op; 2222 2223 OVPN_RLOCK_TRACKER; 2224 2225 M_ASSERTPKTHDR(m); 2226 2227 OVPN_COUNTER_ADD(sc, transport_bytes_received, m->m_pkthdr.len - off); 2228 2229 ohdrlen = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2230 2231 OVPN_RLOCK(sc); 2232 2233 peer = ovpn_peer_from_mbuf(sc, m, off); 2234 if (peer == NULL) { 2235 OVPN_RUNLOCK(sc); 2236 return (false); 2237 } 2238 2239 if (m_length(m, NULL) < (off + sizeof(*uhdr) + ohdrlen)) { 2240 /* Short packet. */ 2241 OVPN_RUNLOCK(sc); 2242 return (false); 2243 } 2244 2245 m_copydata(m, off + sizeof(*uhdr), ohdrlen, (caddr_t)&tmphdr); 2246 2247 op = ntohl(tmphdr.opcode) >> 24 >> OVPN_OP_SHIFT; 2248 if (op != OVPN_OP_DATA_V2) { 2249 /* Control packet? */ 2250 OVPN_RUNLOCK(sc); 2251 return (false); 2252 } 2253 2254 m = m_pullup(m, off + sizeof(*uhdr) + ohdrlen); 2255 if (m == NULL) { 2256 OVPN_RUNLOCK(sc); 2257 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2258 return (true); 2259 } 2260 2261 /* 2262 * Simplify things by getting rid of the preceding headers, we don't 2263 * care about them. 2264 */ 2265 m_adj_decap(m, off); 2266 2267 uhdr = mtodo(m, 0); 2268 ohdr = mtodo(m, sizeof(*uhdr)); 2269 2270 key = ovpn_find_key(sc, peer, ohdr); 2271 if (key == NULL || key->decrypt == NULL) { 2272 OVPN_RUNLOCK(sc); 2273 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2274 m_freem(m); 2275 return (true); 2276 } 2277 2278 if (key->decrypt->cipher == OVPN_CIPHER_ALG_NONE) { 2279 /* Now remove the outer headers */ 2280 m_adj_decap(m, sizeof(struct udphdr) + ohdrlen); 2281 2282 ohdr = mtodo(m, sizeof(*uhdr)); 2283 2284 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), 2285 _ovpn_lock_trackerp); 2286 OVPN_UNLOCK_ASSERT(sc); 2287 return (true); 2288 } 2289 2290 ohdrlen += sizeof(ohdr->auth_tag); 2291 2292 m = m_pullup(m, sizeof(*uhdr) + ohdrlen); 2293 if (m == NULL) { 2294 OVPN_RUNLOCK(sc); 2295 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2296 return (true); 2297 } 2298 uhdr = mtodo(m, 0); 2299 ohdr = mtodo(m, sizeof(*uhdr)); 2300 2301 /* Decrypt */ 2302 crp = crypto_getreq(key->decrypt->cryptoid, M_NOWAIT); 2303 if (crp == NULL) { 2304 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2305 OVPN_RUNLOCK(sc); 2306 m_freem(m); 2307 return (true); 2308 } 2309 2310 crp->crp_payload_start = sizeof(struct udphdr) + sizeof(*ohdr); 2311 crp->crp_payload_length = ntohs(uhdr->uh_ulen) - 2312 sizeof(*uhdr) - sizeof(*ohdr); 2313 crp->crp_op = CRYPTO_OP_DECRYPT; 2314 2315 /* AAD validation. */ 2316 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2317 crp->crp_aad = ohdr; 2318 crp->crp_aad_start = 0; 2319 crp->crp_op |= CRYPTO_OP_VERIFY_DIGEST; 2320 crp->crp_digest_start = sizeof(struct udphdr) + 2321 offsetof(struct ovpn_wire_header, auth_tag); 2322 2323 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 2324 memcpy(crp->crp_iv, &ohdr->seq, sizeof(ohdr->seq)); 2325 memcpy(crp->crp_iv + sizeof(ohdr->seq), key->decrypt->nonce, 2326 key->decrypt->noncelen); 2327 2328 crypto_use_mbuf(crp, m); 2329 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 2330 crp->crp_callback = ovpn_decrypt_rx_cb; 2331 crp->crp_opaque = sc; 2332 2333 atomic_add_int(&sc->refcount, 1); 2334 OVPN_RUNLOCK(sc); 2335 if (V_async_crypto) 2336 ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); 2337 else 2338 ret = crypto_dispatch(crp); 2339 if (ret != 0) { 2340 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2341 } 2342 2343 return (true); 2344 } 2345 2346 static void 2347 ovpn_qflush(struct ifnet *ifp __unused) 2348 { 2349 2350 } 2351 2352 static void 2353 ovpn_flush_rxring(struct ovpn_softc *sc) 2354 { 2355 struct ovpn_notification *n; 2356 2357 OVPN_WASSERT(sc); 2358 2359 while (! buf_ring_empty(sc->notifring)) { 2360 n = buf_ring_dequeue_sc(sc->notifring); 2361 free(n, M_OVPN); 2362 } 2363 } 2364 2365 #ifdef VIMAGE 2366 static void 2367 ovpn_reassign(struct ifnet *ifp, struct vnet *new_vnet __unused, 2368 char *unused __unused) 2369 { 2370 struct ovpn_softc *sc = ifp->if_softc; 2371 struct ovpn_kpeer *peer, *tmppeer; 2372 int ret __diagused; 2373 2374 OVPN_WLOCK(sc); 2375 2376 /* Flush keys & configuration. */ 2377 RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) { 2378 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 2379 ret = _ovpn_del_peer(sc, peer); 2380 MPASS(ret == 0); 2381 } 2382 2383 ovpn_flush_rxring(sc); 2384 2385 OVPN_WUNLOCK(sc); 2386 } 2387 #endif 2388 2389 static int 2390 ovpn_clone_match(struct if_clone *ifc, const char *name) 2391 { 2392 /* 2393 * Allow all names that start with 'ovpn', specifically because pfSense 2394 * uses ovpnc1 / ovpns2 2395 */ 2396 return (strncmp(ovpnname, name, strlen(ovpnname)) == 0); 2397 } 2398 2399 static int 2400 ovpn_clone_create(struct if_clone *ifc, char *name, size_t len, 2401 struct ifc_data *ifd, struct ifnet **ifpp) 2402 { 2403 struct ovpn_softc *sc; 2404 struct ifnet *ifp; 2405 char *dp; 2406 int error, unit, wildcard; 2407 2408 /* Try to see if a special unit was requested. */ 2409 error = ifc_name2unit(name, &unit); 2410 if (error != 0) 2411 return (error); 2412 wildcard = (unit < 0); 2413 2414 error = ifc_alloc_unit(ifc, &unit); 2415 if (error != 0) 2416 return (error); 2417 2418 /* 2419 * If no unit had been given, we need to adjust the ifName. 2420 */ 2421 for (dp = name; *dp != '\0'; dp++); 2422 if (wildcard) { 2423 error = snprintf(dp, len - (dp - name), "%d", unit); 2424 if (error > len - (dp - name)) { 2425 /* ifName too long. */ 2426 ifc_free_unit(ifc, unit); 2427 return (ENOSPC); 2428 } 2429 dp += error; 2430 } 2431 2432 /* Make sure it doesn't already exist. */ 2433 if (ifunit(name) != NULL) 2434 return (EEXIST); 2435 2436 sc = malloc(sizeof(struct ovpn_softc), M_OVPN, M_WAITOK | M_ZERO); 2437 sc->ifp = if_alloc(IFT_ENC); 2438 rm_init_flags(&sc->lock, "if_ovpn_lock", RM_RECURSE); 2439 sc->refcount = 0; 2440 2441 sc->notifring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL); 2442 2443 COUNTER_ARRAY_ALLOC(sc->counters, OVPN_COUNTER_SIZE, M_WAITOK); 2444 2445 ifp = sc->ifp; 2446 ifp->if_softc = sc; 2447 strlcpy(ifp->if_xname, name, IFNAMSIZ); 2448 ifp->if_dname = ovpngroupname; 2449 ifp->if_dunit = unit; 2450 2451 ifp->if_addrlen = 0; 2452 ifp->if_mtu = 1428; 2453 ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST; 2454 ifp->if_ioctl = ovpn_ioctl; 2455 ifp->if_transmit = ovpn_transmit; 2456 ifp->if_output = ovpn_output; 2457 ifp->if_qflush = ovpn_qflush; 2458 #ifdef VIMAGE 2459 ifp->if_reassign = ovpn_reassign; 2460 #endif 2461 ifp->if_capabilities |= IFCAP_LINKSTATE; 2462 ifp->if_capenable |= IFCAP_LINKSTATE; 2463 2464 if_attach(ifp); 2465 bpfattach(ifp, DLT_NULL, sizeof(uint32_t)); 2466 *ifpp = ifp; 2467 2468 return (0); 2469 } 2470 2471 static void 2472 ovpn_clone_destroy_cb(struct epoch_context *ctx) 2473 { 2474 struct ovpn_softc *sc; 2475 2476 sc = __containerof(ctx, struct ovpn_softc, epoch_ctx); 2477 2478 MPASS(sc->peercount == 0); 2479 MPASS(RB_EMPTY(&sc->peers)); 2480 2481 COUNTER_ARRAY_FREE(sc->counters, OVPN_COUNTER_SIZE); 2482 2483 if_free(sc->ifp); 2484 free(sc, M_OVPN); 2485 } 2486 2487 static int 2488 ovpn_clone_destroy(struct if_clone *ifc, struct ifnet *ifp, uint32_t flags) 2489 { 2490 struct ovpn_softc *sc; 2491 struct ovpn_kpeer *peer, *tmppeer; 2492 int unit; 2493 int ret __diagused; 2494 2495 sc = ifp->if_softc; 2496 unit = ifp->if_dunit; 2497 2498 OVPN_WLOCK(sc); 2499 2500 if (atomic_load_int(&sc->refcount) > 0) { 2501 OVPN_WUNLOCK(sc); 2502 return (EBUSY); 2503 } 2504 2505 RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) { 2506 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 2507 ret = _ovpn_del_peer(sc, peer); 2508 MPASS(ret == 0); 2509 } 2510 2511 ovpn_flush_rxring(sc); 2512 buf_ring_free(sc->notifring, M_OVPN); 2513 2514 OVPN_WUNLOCK(sc); 2515 2516 bpfdetach(ifp); 2517 if_detach(ifp); 2518 ifp->if_softc = NULL; 2519 2520 NET_EPOCH_CALL(ovpn_clone_destroy_cb, &sc->epoch_ctx); 2521 2522 if (unit != IF_DUNIT_NONE) 2523 ifc_free_unit(ifc, unit); 2524 2525 NET_EPOCH_DRAIN_CALLBACKS(); 2526 2527 return (0); 2528 } 2529 2530 static void 2531 vnet_ovpn_init(const void *unused __unused) 2532 { 2533 struct if_clone_addreq req = { 2534 .match_f = ovpn_clone_match, 2535 .create_f = ovpn_clone_create, 2536 .destroy_f = ovpn_clone_destroy, 2537 }; 2538 V_ovpn_cloner = ifc_attach_cloner(ovpngroupname, &req); 2539 } 2540 VNET_SYSINIT(vnet_ovpn_init, SI_SUB_PSEUDO, SI_ORDER_ANY, 2541 vnet_ovpn_init, NULL); 2542 2543 static void 2544 vnet_ovpn_uninit(const void *unused __unused) 2545 { 2546 if_clone_detach(V_ovpn_cloner); 2547 } 2548 VNET_SYSUNINIT(vnet_ovpn_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY, 2549 vnet_ovpn_uninit, NULL); 2550 2551 static int 2552 ovpnmodevent(module_t mod, int type, void *data) 2553 { 2554 switch (type) { 2555 case MOD_LOAD: 2556 /* Done in vnet_ovpn_init() */ 2557 break; 2558 case MOD_UNLOAD: 2559 /* Done in vnet_ovpn_uninit() */ 2560 break; 2561 default: 2562 return (EOPNOTSUPP); 2563 } 2564 2565 return (0); 2566 } 2567 2568 static moduledata_t ovpn_mod = { 2569 "if_ovpn", 2570 ovpnmodevent, 2571 0 2572 }; 2573 2574 DECLARE_MODULE(if_ovpn, ovpn_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 2575 MODULE_VERSION(if_ovpn, 1); 2576