1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2021-2022 Rubicon Communications, LLC (Netgate) 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 */ 28 #include "opt_inet.h" 29 #include "opt_inet6.h" 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/buf_ring.h> 34 #include <sys/epoch.h> 35 #include <sys/file.h> 36 #include <sys/filedesc.h> 37 #include <sys/kernel.h> 38 #include <sys/malloc.h> 39 #include <sys/mbuf.h> 40 #include <sys/module.h> 41 #include <sys/nv.h> 42 #include <sys/priv.h> 43 #include <sys/protosw.h> 44 #include <sys/rmlock.h> 45 #include <sys/sdt.h> 46 #include <sys/smp.h> 47 #include <sys/socket.h> 48 #include <sys/socketvar.h> 49 #include <sys/sockio.h> 50 #include <sys/sysctl.h> 51 #include <sys/time.h> 52 53 #include <machine/atomic.h> 54 55 #include <net/bpf.h> 56 #include <net/if.h> 57 #include <net/if_clone.h> 58 #include <net/if_types.h> 59 #include <net/if_var.h> 60 #include <net/if_private.h> 61 #include <net/netisr.h> 62 #include <net/route/nhop.h> 63 64 #include <netinet/in.h> 65 #include <netinet/in_fib.h> 66 #include <netinet/ip.h> 67 #include <netinet/ip6.h> 68 #include <netinet/ip_var.h> 69 #include <netinet/udp.h> 70 #include <netinet/udp_var.h> 71 72 #include <netinet6/ip6_var.h> 73 #include <netinet6/in6_fib.h> 74 75 #include <machine/in_cksum.h> 76 77 #include <opencrypto/cryptodev.h> 78 79 #include "if_ovpn.h" 80 81 struct ovpn_kkey_dir { 82 int refcount; 83 uint8_t key[32]; 84 uint8_t keylen; 85 uint8_t nonce[8]; 86 uint8_t noncelen; 87 enum ovpn_key_cipher cipher; 88 crypto_session_t cryptoid; 89 90 struct mtx replay_mtx; 91 /* 92 * Last seen gapless sequence number. New rx seq numbers must be 93 * strictly higher than this. 94 */ 95 uint32_t rx_seq; 96 uint64_t tx_seq; 97 98 /* Seen packets, relative to rx_seq. bit(0) will always be 0. */ 99 uint64_t rx_window; 100 }; 101 102 struct ovpn_kkey { 103 struct ovpn_kkey_dir *encrypt; 104 struct ovpn_kkey_dir *decrypt; 105 uint8_t keyid; 106 uint32_t peerid; 107 }; 108 109 struct ovpn_keepalive { 110 uint32_t interval; 111 uint32_t timeout; 112 }; 113 114 struct ovpn_wire_header { 115 uint32_t opcode; /* opcode, key id, peer id */ 116 uint32_t seq; 117 uint8_t auth_tag[16]; 118 }; 119 120 struct ovpn_peer_counters { 121 uint64_t pkt_in; 122 uint64_t pkt_out; 123 uint64_t bytes_in; 124 uint64_t bytes_out; 125 }; 126 #define OVPN_PEER_COUNTER_SIZE (sizeof(struct ovpn_peer_counters)/sizeof(uint64_t)) 127 128 struct ovpn_notification { 129 enum ovpn_notif_type type; 130 uint32_t peerid; 131 132 /* Delete notification */ 133 enum ovpn_del_reason del_reason; 134 struct ovpn_peer_counters counters; 135 }; 136 137 struct ovpn_softc; 138 139 struct ovpn_kpeer { 140 RB_ENTRY(ovpn_kpeer) tree; 141 int refcount; 142 uint32_t peerid; 143 144 struct ovpn_softc *sc; 145 struct sockaddr_storage local; 146 struct sockaddr_storage remote; 147 148 struct in_addr vpn4; 149 struct in6_addr vpn6; 150 151 struct ovpn_kkey keys[2]; 152 153 enum ovpn_del_reason del_reason; 154 struct ovpn_keepalive keepalive; 155 uint32_t *last_active; 156 struct callout ping_send; 157 struct callout ping_rcv; 158 159 counter_u64_t counters[OVPN_PEER_COUNTER_SIZE]; 160 }; 161 162 struct ovpn_counters { 163 uint64_t lost_ctrl_pkts_in; 164 uint64_t lost_ctrl_pkts_out; 165 uint64_t lost_data_pkts_in; 166 uint64_t lost_data_pkts_out; 167 uint64_t nomem_data_pkts_in; 168 uint64_t nomem_data_pkts_out; 169 uint64_t received_ctrl_pkts; 170 uint64_t received_data_pkts; 171 uint64_t sent_ctrl_pkts; 172 uint64_t sent_data_pkts; 173 174 uint64_t transport_bytes_sent; 175 uint64_t transport_bytes_received; 176 uint64_t tunnel_bytes_sent; 177 uint64_t tunnel_bytes_received; 178 }; 179 #define OVPN_COUNTER_SIZE (sizeof(struct ovpn_counters)/sizeof(uint64_t)) 180 181 RB_HEAD(ovpn_kpeers, ovpn_kpeer); 182 183 struct ovpn_softc { 184 int refcount; 185 struct rmlock lock; 186 struct ifnet *ifp; 187 struct socket *so; 188 int peercount; 189 struct ovpn_kpeers peers; 190 191 /* Pending notification */ 192 struct buf_ring *notifring; 193 194 counter_u64_t counters[OVPN_COUNTER_SIZE]; 195 196 struct epoch_context epoch_ctx; 197 }; 198 199 static struct ovpn_kpeer *ovpn_find_peer(struct ovpn_softc *, uint32_t); 200 static bool ovpn_udp_input(struct mbuf *, int, struct inpcb *, 201 const struct sockaddr *, void *); 202 static int ovpn_transmit_to_peer(struct ifnet *, struct mbuf *, 203 struct ovpn_kpeer *, struct rm_priotracker *); 204 static int ovpn_encap(struct ovpn_softc *, uint32_t, struct mbuf *); 205 static int ovpn_get_af(struct mbuf *); 206 static void ovpn_free_kkey_dir(struct ovpn_kkey_dir *); 207 static bool ovpn_check_replay(struct ovpn_kkey_dir *, uint32_t); 208 static int ovpn_peer_compare(struct ovpn_kpeer *, struct ovpn_kpeer *); 209 210 static RB_PROTOTYPE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare); 211 static RB_GENERATE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare); 212 213 #define OVPN_MTU_MIN 576 214 #define OVPN_MTU_MAX (IP_MAXPACKET - sizeof(struct ip) - \ 215 sizeof(struct udphdr) - sizeof(struct ovpn_wire_header)) 216 217 #define OVPN_OP_DATA_V2 0x09 218 #define OVPN_OP_SHIFT 3 219 #define OVPN_SEQ_ROTATE 0x80000000 220 221 VNET_DEFINE_STATIC(struct if_clone *, ovpn_cloner); 222 #define V_ovpn_cloner VNET(ovpn_cloner) 223 224 #define OVPN_RLOCK_TRACKER struct rm_priotracker _ovpn_lock_tracker; \ 225 struct rm_priotracker *_ovpn_lock_trackerp = &_ovpn_lock_tracker 226 #define OVPN_RLOCK(sc) rm_rlock(&(sc)->lock, _ovpn_lock_trackerp) 227 #define OVPN_RUNLOCK(sc) rm_runlock(&(sc)->lock, _ovpn_lock_trackerp) 228 #define OVPN_WLOCK(sc) rm_wlock(&(sc)->lock) 229 #define OVPN_WUNLOCK(sc) rm_wunlock(&(sc)->lock) 230 #define OVPN_ASSERT(sc) rm_assert(&(sc)->lock, RA_LOCKED) 231 #define OVPN_RASSERT(sc) rm_assert(&(sc)->lock, RA_RLOCKED) 232 #define OVPN_WASSERT(sc) rm_assert(&(sc)->lock, RA_WLOCKED) 233 #define OVPN_UNLOCK_ASSERT(sc) rm_assert(&(sc)->lock, RA_UNLOCKED) 234 235 #define OVPN_COUNTER(sc, name) \ 236 ((sc)->counters[offsetof(struct ovpn_counters, name)/sizeof(uint64_t)]) 237 #define OVPN_PEER_COUNTER(peer, name) \ 238 ((peer)->counters[offsetof(struct ovpn_peer_counters, name) / \ 239 sizeof(uint64_t)]) 240 241 #define OVPN_COUNTER_ADD(sc, name, val) \ 242 counter_u64_add(OVPN_COUNTER(sc, name), val) 243 #define OVPN_PEER_COUNTER_ADD(p, name, val) \ 244 counter_u64_add(OVPN_PEER_COUNTER(p, name), val) 245 246 #define TO_IN(x) ((struct sockaddr_in *)(x)) 247 #define TO_IN6(x) ((struct sockaddr_in6 *)(x)) 248 249 SDT_PROVIDER_DEFINE(if_ovpn); 250 SDT_PROBE_DEFINE1(if_ovpn, tx, transmit, start, "struct mbuf *"); 251 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip4, "struct in_addr *", "struct ovpn_kpeer *"); 252 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip6, "struct in6_addr *", "struct ovpn_kpeer *"); 253 254 static const char ovpnname[] = "ovpn"; 255 static const char ovpngroupname[] = "openvpn"; 256 257 static MALLOC_DEFINE(M_OVPN, ovpnname, "OpenVPN DCO Interface"); 258 259 SYSCTL_DECL(_net_link); 260 static SYSCTL_NODE(_net_link, IFT_OTHER, openvpn, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 261 "OpenVPN DCO Interface"); 262 VNET_DEFINE_STATIC(int, replay_protection) = 0; 263 #define V_replay_protection VNET(replay_protection) 264 SYSCTL_INT(_net_link_openvpn, OID_AUTO, replay_protection, CTLFLAG_VNET | CTLFLAG_RW, 265 &VNET_NAME(replay_protection), 0, "Validate sequence numbers"); 266 267 VNET_DEFINE_STATIC(int, async_crypto); 268 #define V_async_crypto VNET(async_crypto) 269 SYSCTL_INT(_net_link_openvpn, OID_AUTO, async_crypto, 270 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_crypto), 0, 271 "Use asynchronous mode to parallelize crypto jobs."); 272 273 VNET_DEFINE_STATIC(int, async_netisr_queue); 274 #define V_async_netisr_queue VNET(async_netisr_queue) 275 SYSCTL_INT(_net_link_openvpn, OID_AUTO, netisr_queue, 276 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_netisr_queue), 0, 277 "Use netisr_queue() rather than netisr_dispatch()."); 278 279 static int 280 ovpn_peer_compare(struct ovpn_kpeer *a, struct ovpn_kpeer *b) 281 { 282 return (a->peerid - b->peerid); 283 } 284 285 static struct ovpn_kpeer * 286 ovpn_find_peer(struct ovpn_softc *sc, uint32_t peerid) 287 { 288 struct ovpn_kpeer p; 289 290 OVPN_ASSERT(sc); 291 292 p.peerid = peerid; 293 294 return (RB_FIND(ovpn_kpeers, &sc->peers, &p)); 295 } 296 297 static struct ovpn_kpeer * 298 ovpn_find_only_peer(struct ovpn_softc *sc) 299 { 300 OVPN_ASSERT(sc); 301 302 return (RB_ROOT(&sc->peers)); 303 } 304 305 static uint16_t 306 ovpn_get_port(struct sockaddr_storage *s) 307 { 308 switch (s->ss_family) { 309 case AF_INET: { 310 struct sockaddr_in *in = (struct sockaddr_in *)s; 311 return (in->sin_port); 312 } 313 case AF_INET6: { 314 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)s; 315 return (in6->sin6_port); 316 } 317 default: 318 panic("Unsupported address family %d", s->ss_family); 319 } 320 } 321 322 static int 323 ovpn_nvlist_to_sockaddr(const nvlist_t *nvl, struct sockaddr_storage *sa) 324 { 325 int af; 326 327 if (! nvlist_exists_number(nvl, "af")) 328 return (EINVAL); 329 if (! nvlist_exists_binary(nvl, "address")) 330 return (EINVAL); 331 if (! nvlist_exists_number(nvl, "port")) 332 return (EINVAL); 333 334 af = nvlist_get_number(nvl, "af"); 335 336 switch (af) { 337 #ifdef INET 338 case AF_INET: { 339 struct sockaddr_in *in = (struct sockaddr_in *)sa; 340 size_t len; 341 const void *addr = nvlist_get_binary(nvl, "address", &len); 342 in->sin_family = af; 343 if (len != sizeof(in->sin_addr)) 344 return (EINVAL); 345 346 memcpy(&in->sin_addr, addr, sizeof(in->sin_addr)); 347 in->sin_port = nvlist_get_number(nvl, "port"); 348 break; 349 } 350 #endif 351 #ifdef INET6 352 case AF_INET6: { 353 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)sa; 354 size_t len; 355 const void *addr = nvlist_get_binary(nvl, "address", &len); 356 in6->sin6_family = af; 357 if (len != sizeof(in6->sin6_addr)) 358 return (EINVAL); 359 360 memcpy(&in6->sin6_addr, addr, sizeof(in6->sin6_addr)); 361 in6->sin6_port = nvlist_get_number(nvl, "port"); 362 break; 363 } 364 #endif 365 default: 366 return (EINVAL); 367 } 368 369 return (0); 370 } 371 372 static bool 373 ovpn_has_peers(struct ovpn_softc *sc) 374 { 375 OVPN_ASSERT(sc); 376 377 return (sc->peercount > 0); 378 } 379 380 static void 381 ovpn_rele_so(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 382 { 383 bool has_peers; 384 385 OVPN_WASSERT(sc); 386 387 if (sc->so == NULL) 388 return; 389 390 has_peers = ovpn_has_peers(sc); 391 392 /* Only remove the tunnel function if we're releasing the socket for 393 * the last peer. */ 394 if (! has_peers) 395 (void)udp_set_kernel_tunneling(sc->so, NULL, NULL, NULL); 396 397 sorele(sc->so); 398 399 if (! has_peers) 400 sc->so = NULL; 401 } 402 403 static void 404 ovpn_notify_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 405 { 406 struct ovpn_notification *n; 407 408 OVPN_WASSERT(sc); 409 410 n = malloc(sizeof(*n), M_OVPN, M_NOWAIT); 411 if (n == NULL) 412 return; 413 414 n->peerid = peer->peerid; 415 n->type = OVPN_NOTIF_DEL_PEER; 416 n->del_reason = peer->del_reason; 417 418 n->counters.pkt_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_in)); 419 n->counters.pkt_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_out)); 420 n->counters.bytes_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_in)); 421 n->counters.bytes_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_out)); 422 423 if (buf_ring_enqueue(sc->notifring, n) != 0) { 424 free(n, M_OVPN); 425 } else if (sc->so != NULL) { 426 /* Wake up userspace */ 427 sc->so->so_error = EAGAIN; 428 sorwakeup(sc->so); 429 sowwakeup(sc->so); 430 } 431 } 432 433 static void 434 ovpn_notify_key_rotation(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 435 { 436 struct ovpn_notification *n; 437 438 n = malloc(sizeof(*n), M_OVPN, M_NOWAIT | M_ZERO); 439 if (n == NULL) 440 return; 441 442 n->peerid = peer->peerid; 443 n->type = OVPN_NOTIF_ROTATE_KEY; 444 445 if (buf_ring_enqueue(sc->notifring, n) != 0) { 446 free(n, M_OVPN); 447 } else if (sc->so != NULL) { 448 /* Wake up userspace */ 449 sc->so->so_error = EAGAIN; 450 sorwakeup(sc->so); 451 sowwakeup(sc->so); 452 } 453 } 454 455 static void 456 ovpn_peer_release_ref(struct ovpn_kpeer *peer, bool locked) 457 { 458 struct ovpn_softc *sc; 459 460 CURVNET_ASSERT_SET(); 461 462 atomic_add_int(&peer->refcount, -1); 463 464 if (atomic_load_int(&peer->refcount) > 0) 465 return; 466 467 sc = peer->sc; 468 469 if (! locked) { 470 OVPN_WLOCK(sc); 471 472 /* Might have changed before we acquired the lock. */ 473 if (atomic_load_int(&peer->refcount) > 0) { 474 OVPN_WUNLOCK(sc); 475 return; 476 } 477 } 478 479 OVPN_ASSERT(sc); 480 481 /* The peer should have been removed from the list already. */ 482 MPASS(ovpn_find_peer(sc, peer->peerid) == NULL); 483 484 ovpn_notify_del_peer(sc, peer); 485 486 for (int i = 0; i < 2; i++) { 487 ovpn_free_kkey_dir(peer->keys[i].encrypt); 488 ovpn_free_kkey_dir(peer->keys[i].decrypt); 489 } 490 491 ovpn_rele_so(sc, peer); 492 493 callout_stop(&peer->ping_send); 494 callout_stop(&peer->ping_rcv); 495 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 496 free(peer, M_OVPN); 497 498 if (! locked) 499 OVPN_WUNLOCK(sc); 500 } 501 502 static int 503 ovpn_new_peer(struct ifnet *ifp, const nvlist_t *nvl) 504 { 505 #ifdef INET6 506 struct epoch_tracker et; 507 #endif 508 struct sockaddr_storage remote; 509 struct ovpn_kpeer *peer = NULL; 510 struct file *fp = NULL; 511 struct sockaddr *name = NULL; 512 struct ovpn_softc *sc = ifp->if_softc; 513 struct thread *td = curthread; 514 struct socket *so = NULL; 515 int fd; 516 uint32_t peerid; 517 int ret = 0; 518 519 if (nvl == NULL) 520 return (EINVAL); 521 522 if (! nvlist_exists_number(nvl, "peerid")) 523 return (EINVAL); 524 525 if (! nvlist_exists_number(nvl, "fd")) 526 return (EINVAL); 527 528 if (! nvlist_exists_nvlist(nvl, "remote")) 529 return (EINVAL); 530 531 peerid = nvlist_get_number(nvl, "peerid"); 532 533 ret = ovpn_nvlist_to_sockaddr(nvlist_get_nvlist(nvl, "remote"), 534 &remote); 535 if (ret != 0) 536 return (ret); 537 538 fd = nvlist_get_number(nvl, "fd"); 539 540 /* Look up the userspace process and use the fd to find the socket. */ 541 ret = getsock(td, fd, &cap_connect_rights, &fp); 542 if (ret != 0) 543 return (ret); 544 545 so = fp->f_data; 546 547 peer = malloc(sizeof(*peer), M_OVPN, M_WAITOK | M_ZERO); 548 peer->peerid = peerid; 549 peer->sc = sc; 550 peer->refcount = 1; 551 peer->last_active = uma_zalloc_pcpu(pcpu_zone_4, M_WAITOK | M_ZERO); 552 COUNTER_ARRAY_ALLOC(peer->counters, OVPN_PEER_COUNTER_SIZE, M_WAITOK); 553 554 if (nvlist_exists_binary(nvl, "vpn_ipv4")) { 555 size_t len; 556 const void *addr = nvlist_get_binary(nvl, "vpn_ipv4", &len); 557 if (len != sizeof(peer->vpn4)) { 558 ret = EINVAL; 559 goto error; 560 } 561 memcpy(&peer->vpn4, addr, len); 562 } 563 564 if (nvlist_exists_binary(nvl, "vpn_ipv6")) { 565 size_t len; 566 const void *addr = nvlist_get_binary(nvl, "vpn_ipv6", &len); 567 if (len != sizeof(peer->vpn6)) { 568 ret = EINVAL; 569 goto error; 570 } 571 memcpy(&peer->vpn6, addr, len); 572 } 573 574 callout_init_rm(&peer->ping_send, &sc->lock, CALLOUT_SHAREDLOCK); 575 callout_init_rm(&peer->ping_rcv, &sc->lock, 0); 576 577 ret = so->so_proto->pr_sockaddr(so, &name); 578 if (ret) 579 goto error; 580 581 if (ovpn_get_port((struct sockaddr_storage *)name) == 0) { 582 ret = EINVAL; 583 goto error; 584 } 585 if (name->sa_family != remote.ss_family) { 586 ret = EINVAL; 587 goto error; 588 } 589 590 memcpy(&peer->local, name, name->sa_len); 591 memcpy(&peer->remote, &remote, sizeof(remote)); 592 free(name, M_SONAME); 593 name = NULL; 594 595 if (peer->local.ss_family == AF_INET6 && 596 IN6_IS_ADDR_V4MAPPED(&TO_IN6(&peer->remote)->sin6_addr)) { 597 /* V4 mapped address, so treat this as v4, not v6. */ 598 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->local); 599 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->remote); 600 } 601 602 #ifdef INET6 603 if (peer->local.ss_family == AF_INET6 && 604 IN6_IS_ADDR_UNSPECIFIED(&TO_IN6(&peer->local)->sin6_addr)) { 605 NET_EPOCH_ENTER(et); 606 ret = in6_selectsrc_addr(curthread->td_proc->p_fibnum, 607 &TO_IN6(&peer->remote)->sin6_addr, 608 0, NULL, &TO_IN6(&peer->local)->sin6_addr, NULL); 609 NET_EPOCH_EXIT(et); 610 if (ret != 0) { 611 goto error; 612 } 613 } 614 #endif 615 OVPN_WLOCK(sc); 616 617 /* Disallow peer id re-use. */ 618 if (ovpn_find_peer(sc, peerid) != NULL) { 619 ret = EEXIST; 620 goto error_locked; 621 } 622 623 /* Make sure this is really a UDP socket. */ 624 if (so->so_type != SOCK_DGRAM || so->so_proto->pr_type != SOCK_DGRAM) { 625 ret = EPROTOTYPE; 626 goto error_locked; 627 } 628 629 /* Must be the same socket as for other peers on this interface. */ 630 if (sc->so != NULL && so != sc->so) 631 goto error_locked; 632 633 if (sc->so == NULL) 634 sc->so = so; 635 636 /* Insert the peer into the list. */ 637 RB_INSERT(ovpn_kpeers, &sc->peers, peer); 638 sc->peercount++; 639 soref(sc->so); 640 641 ret = udp_set_kernel_tunneling(sc->so, ovpn_udp_input, NULL, sc); 642 if (ret == EBUSY) { 643 /* Fine, another peer already set the input function. */ 644 ret = 0; 645 } 646 if (ret != 0) { 647 RB_REMOVE(ovpn_kpeers, &sc->peers, peer); 648 sc->peercount--; 649 goto error_locked; 650 } 651 652 OVPN_WUNLOCK(sc); 653 654 goto done; 655 656 error_locked: 657 OVPN_WUNLOCK(sc); 658 error: 659 free(name, M_SONAME); 660 COUNTER_ARRAY_FREE(peer->counters, OVPN_PEER_COUNTER_SIZE); 661 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 662 free(peer, M_OVPN); 663 done: 664 if (fp != NULL) 665 fdrop(fp, td); 666 667 return (ret); 668 } 669 670 static int 671 _ovpn_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 672 { 673 struct ovpn_kpeer *tmp __diagused; 674 675 OVPN_WASSERT(sc); 676 CURVNET_ASSERT_SET(); 677 678 MPASS(RB_FIND(ovpn_kpeers, &sc->peers, peer) == peer); 679 680 tmp = RB_REMOVE(ovpn_kpeers, &sc->peers, peer); 681 MPASS(tmp != NULL); 682 683 sc->peercount--; 684 685 ovpn_peer_release_ref(peer, true); 686 687 return (0); 688 } 689 690 static int 691 ovpn_del_peer(struct ifnet *ifp, nvlist_t *nvl) 692 { 693 struct ovpn_softc *sc = ifp->if_softc; 694 struct ovpn_kpeer *peer; 695 uint32_t peerid; 696 int ret; 697 698 OVPN_WASSERT(sc); 699 700 if (nvl == NULL) 701 return (EINVAL); 702 703 if (! nvlist_exists_number(nvl, "peerid")) 704 return (EINVAL); 705 706 peerid = nvlist_get_number(nvl, "peerid"); 707 708 peer = ovpn_find_peer(sc, peerid); 709 if (peer == NULL) 710 return (ENOENT); 711 712 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 713 ret = _ovpn_del_peer(sc, peer); 714 715 return (ret); 716 } 717 718 static int 719 ovpn_create_kkey_dir(struct ovpn_kkey_dir **kdirp, 720 const nvlist_t *nvl) 721 { 722 struct crypto_session_params csp; 723 struct ovpn_kkey_dir *kdir; 724 const char *ciphername; 725 enum ovpn_key_cipher cipher; 726 const void *key, *iv; 727 size_t keylen = 0, ivlen = 0; 728 int error; 729 730 if (! nvlist_exists_string(nvl, "cipher")) 731 return (EINVAL); 732 ciphername = nvlist_get_string(nvl, "cipher"); 733 734 if (strcmp(ciphername, "none") == 0) 735 cipher = OVPN_CIPHER_ALG_NONE; 736 else if (strcmp(ciphername, "AES-256-GCM") == 0 || 737 strcmp(ciphername, "AES-192-GCM") == 0 || 738 strcmp(ciphername, "AES-128-GCM") == 0) 739 cipher = OVPN_CIPHER_ALG_AES_GCM; 740 else if (strcmp(ciphername, "CHACHA20-POLY1305") == 0) 741 cipher = OVPN_CIPHER_ALG_CHACHA20_POLY1305; 742 else 743 return (EINVAL); 744 745 if (cipher != OVPN_CIPHER_ALG_NONE) { 746 if (! nvlist_exists_binary(nvl, "key")) 747 return (EINVAL); 748 key = nvlist_get_binary(nvl, "key", &keylen); 749 if (keylen > sizeof(kdir->key)) 750 return (E2BIG); 751 752 if (! nvlist_exists_binary(nvl, "iv")) 753 return (EINVAL); 754 iv = nvlist_get_binary(nvl, "iv", &ivlen); 755 if (ivlen != 8) 756 return (E2BIG); 757 } 758 759 kdir = malloc(sizeof(struct ovpn_kkey_dir), M_OVPN, 760 M_WAITOK | M_ZERO); 761 762 kdir->cipher = cipher; 763 kdir->keylen = keylen; 764 kdir->tx_seq = 1; 765 memcpy(kdir->key, key, keylen); 766 kdir->noncelen = ivlen; 767 memcpy(kdir->nonce, iv, ivlen); 768 769 if (kdir->cipher != OVPN_CIPHER_ALG_NONE) { 770 /* Crypto init */ 771 bzero(&csp, sizeof(csp)); 772 csp.csp_mode = CSP_MODE_AEAD; 773 774 if (kdir->cipher == OVPN_CIPHER_ALG_CHACHA20_POLY1305) 775 csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305; 776 else 777 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 778 779 csp.csp_flags |= CSP_F_SEPARATE_AAD; 780 781 csp.csp_cipher_klen = kdir->keylen; 782 csp.csp_cipher_key = kdir->key; 783 csp.csp_ivlen = 96 / 8; 784 785 error = crypto_newsession(&kdir->cryptoid, &csp, 786 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE); 787 if (error) { 788 free(kdir, M_OVPN); 789 return (error); 790 } 791 } 792 793 mtx_init(&kdir->replay_mtx, "if_ovpn rx replay", NULL, MTX_DEF); 794 *kdirp = kdir; 795 796 return (0); 797 } 798 799 static void 800 ovpn_free_kkey_dir(struct ovpn_kkey_dir *kdir) 801 { 802 if (kdir == NULL) 803 return; 804 805 mtx_destroy(&kdir->replay_mtx); 806 807 crypto_freesession(kdir->cryptoid); 808 free(kdir, M_OVPN); 809 } 810 811 static int 812 ovpn_set_key(struct ifnet *ifp, const nvlist_t *nvl) 813 { 814 struct ovpn_softc *sc = ifp->if_softc; 815 struct ovpn_kkey_dir *enc, *dec; 816 struct ovpn_kpeer *peer; 817 int slot, keyid, peerid; 818 int error; 819 820 if (nvl == NULL) 821 return (EINVAL); 822 823 if (! nvlist_exists_number(nvl, "slot")) 824 return (EINVAL); 825 slot = nvlist_get_number(nvl, "slot"); 826 827 if (! nvlist_exists_number(nvl, "keyid")) 828 return (EINVAL); 829 keyid = nvlist_get_number(nvl, "keyid"); 830 831 if (! nvlist_exists_number(nvl, "peerid")) 832 return (EINVAL); 833 peerid = nvlist_get_number(nvl, "peerid"); 834 835 if (slot != OVPN_KEY_SLOT_PRIMARY && 836 slot != OVPN_KEY_SLOT_SECONDARY) 837 return (EINVAL); 838 839 if (! nvlist_exists_nvlist(nvl, "encrypt") || 840 ! nvlist_exists_nvlist(nvl, "decrypt")) 841 return (EINVAL); 842 843 error = ovpn_create_kkey_dir(&enc, nvlist_get_nvlist(nvl, "encrypt")); 844 if (error) 845 return (error); 846 847 error = ovpn_create_kkey_dir(&dec, nvlist_get_nvlist(nvl, "decrypt")); 848 if (error) { 849 ovpn_free_kkey_dir(enc); 850 return (error); 851 } 852 853 OVPN_WLOCK(sc); 854 855 peer = ovpn_find_peer(sc, peerid); 856 if (peer == NULL) { 857 ovpn_free_kkey_dir(dec); 858 ovpn_free_kkey_dir(enc); 859 OVPN_WUNLOCK(sc); 860 return (ENOENT); 861 } 862 863 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 864 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 865 866 peer->keys[slot].encrypt = enc; 867 peer->keys[slot].decrypt = dec; 868 869 peer->keys[slot].keyid = keyid; 870 peer->keys[slot].peerid = peerid; 871 872 OVPN_WUNLOCK(sc); 873 874 return (0); 875 } 876 877 static int 878 ovpn_check_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, enum ovpn_key_slot slot) 879 { 880 OVPN_ASSERT(sc); 881 882 if (peer->keys[slot].encrypt == NULL) 883 return (ENOLINK); 884 885 if (peer->keys[slot].decrypt == NULL) 886 return (ENOLINK); 887 888 return (0); 889 } 890 891 static int 892 ovpn_start(struct ifnet *ifp) 893 { 894 struct ovpn_softc *sc = ifp->if_softc; 895 896 OVPN_WLOCK(sc); 897 898 ifp->if_flags |= IFF_UP; 899 ifp->if_drv_flags |= IFF_DRV_RUNNING; 900 if_link_state_change(ifp, LINK_STATE_UP); 901 902 OVPN_WUNLOCK(sc); 903 904 return (0); 905 } 906 907 static int 908 ovpn_swap_keys(struct ifnet *ifp, nvlist_t *nvl) 909 { 910 struct ovpn_softc *sc = ifp->if_softc; 911 struct ovpn_kpeer *peer; 912 struct ovpn_kkey tmpkey; 913 int error; 914 915 if (nvl == NULL) 916 return (EINVAL); 917 918 if (! nvlist_exists_number(nvl, "peerid")) 919 return (EINVAL); 920 921 OVPN_WLOCK(sc); 922 923 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 924 if (peer == NULL) { 925 OVPN_WUNLOCK(sc); 926 return (ENOENT); 927 } 928 929 /* Check that we have a second key to swap to. */ 930 error = ovpn_check_key(sc, peer, OVPN_KEY_SLOT_SECONDARY); 931 if (error) { 932 OVPN_WUNLOCK(sc); 933 return (error); 934 } 935 936 tmpkey = peer->keys[0]; 937 peer->keys[0] = peer->keys[1]; 938 peer->keys[1] = tmpkey; 939 940 OVPN_WUNLOCK(sc); 941 942 return (0); 943 } 944 945 static int 946 ovpn_del_key(struct ifnet *ifp, const nvlist_t *nvl) 947 { 948 enum ovpn_key_slot slot; 949 struct ovpn_kpeer *peer; 950 struct ovpn_softc *sc = ifp->if_softc; 951 952 if (nvl == NULL) 953 return (EINVAL); 954 955 if (! nvlist_exists_number(nvl, "peerid")) 956 return (EINVAL); 957 958 if (! nvlist_exists_number(nvl, "slot")) 959 return (EINVAL); 960 slot = nvlist_get_number(nvl, "slot"); 961 962 if (slot != OVPN_KEY_SLOT_PRIMARY && 963 slot != OVPN_KEY_SLOT_SECONDARY) 964 return (EINVAL); 965 966 OVPN_WLOCK(sc); 967 968 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 969 if (peer == NULL) { 970 OVPN_WUNLOCK(sc); 971 return (ENOENT); 972 } 973 974 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 975 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 976 977 peer->keys[slot].encrypt = NULL; 978 peer->keys[slot].decrypt = NULL; 979 980 peer->keys[slot].keyid = 0; 981 peer->keys[slot].peerid = 0; 982 983 OVPN_WUNLOCK(sc); 984 985 return (0); 986 } 987 988 static void 989 ovpn_send_ping(void *arg) 990 { 991 static const uint8_t ping_str[] = { 992 0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb, 993 0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48 994 }; 995 996 struct epoch_tracker et; 997 struct ovpn_kpeer *peer = arg; 998 struct ovpn_softc *sc = peer->sc; 999 struct mbuf *m; 1000 1001 OVPN_RASSERT(sc); 1002 1003 /* Ensure we repeat! */ 1004 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 1005 ovpn_send_ping, peer); 1006 1007 m = m_get2(sizeof(ping_str), M_NOWAIT, MT_DATA, M_PKTHDR); 1008 if (m == NULL) 1009 return; 1010 1011 m_copyback(m, 0, sizeof(ping_str), ping_str); 1012 m->m_len = m->m_pkthdr.len = sizeof(ping_str); 1013 1014 CURVNET_SET(sc->ifp->if_vnet); 1015 NET_EPOCH_ENTER(et); 1016 (void)ovpn_transmit_to_peer(sc->ifp, m, peer, NULL); 1017 NET_EPOCH_EXIT(et); 1018 CURVNET_RESTORE(); 1019 } 1020 1021 static void 1022 ovpn_timeout(void *arg) 1023 { 1024 struct ovpn_kpeer *peer = arg; 1025 struct ovpn_softc *sc = peer->sc; 1026 uint32_t last, _last_active; 1027 int ret __diagused; 1028 int cpu; 1029 1030 OVPN_WASSERT(sc); 1031 1032 last = 0; 1033 CPU_FOREACH(cpu) { 1034 _last_active = *zpcpu_get_cpu(peer->last_active, cpu); 1035 if (_last_active > last) 1036 last = _last_active; 1037 } 1038 1039 if (last + peer->keepalive.timeout > time_uptime) { 1040 callout_reset(&peer->ping_rcv, 1041 (peer->keepalive.timeout - (time_uptime - last)) * hz, 1042 ovpn_timeout, peer); 1043 return; 1044 } 1045 1046 CURVNET_SET(sc->ifp->if_vnet); 1047 peer->del_reason = OVPN_DEL_REASON_TIMEOUT; 1048 ret = _ovpn_del_peer(sc, peer); 1049 MPASS(ret == 0); 1050 CURVNET_RESTORE(); 1051 } 1052 1053 static int 1054 ovpn_set_peer(struct ifnet *ifp, const nvlist_t *nvl) 1055 { 1056 struct ovpn_softc *sc = ifp->if_softc; 1057 struct ovpn_kpeer *peer; 1058 1059 if (nvl == NULL) 1060 return (EINVAL); 1061 1062 if (! nvlist_exists_number(nvl, "interval") || 1063 ! nvlist_exists_number(nvl, "timeout") || 1064 ! nvlist_exists_number(nvl, "peerid")) 1065 return (EINVAL); 1066 1067 OVPN_WLOCK(sc); 1068 1069 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 1070 if (peer == NULL) { 1071 OVPN_WUNLOCK(sc); 1072 return (ENOENT); 1073 } 1074 1075 peer->keepalive.interval = nvlist_get_number(nvl, "interval"); 1076 peer->keepalive.timeout = nvlist_get_number(nvl, "timeout"); 1077 1078 if (peer->keepalive.interval > 0) 1079 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 1080 ovpn_send_ping, peer); 1081 if (peer->keepalive.timeout > 0) 1082 callout_reset(&peer->ping_rcv, peer->keepalive.timeout * hz, 1083 ovpn_timeout, peer); 1084 1085 OVPN_WUNLOCK(sc); 1086 1087 return (0); 1088 } 1089 1090 static int 1091 ovpn_set_ifmode(struct ifnet *ifp, const nvlist_t *nvl) 1092 { 1093 struct ovpn_softc *sc = ifp->if_softc; 1094 int ifmode; 1095 1096 if (nvl == NULL) 1097 return (EINVAL); 1098 1099 if (! nvlist_exists_number(nvl, "ifmode") ) 1100 return (EINVAL); 1101 1102 ifmode = nvlist_get_number(nvl, "ifmode"); 1103 1104 OVPN_WLOCK(sc); 1105 1106 /* deny this if UP */ 1107 if (ifp->if_flags & IFF_UP) { 1108 OVPN_WUNLOCK(sc); 1109 return (EBUSY); 1110 } 1111 1112 switch (ifmode & ~IFF_MULTICAST) { 1113 case IFF_POINTOPOINT: 1114 case IFF_BROADCAST: 1115 ifp->if_flags &= 1116 ~(IFF_BROADCAST|IFF_POINTOPOINT|IFF_MULTICAST); 1117 ifp->if_flags |= ifmode; 1118 break; 1119 default: 1120 OVPN_WUNLOCK(sc); 1121 return (EINVAL); 1122 } 1123 1124 OVPN_WUNLOCK(sc); 1125 1126 return (0); 1127 } 1128 1129 static int 1130 ovpn_ioctl_set(struct ifnet *ifp, struct ifdrv *ifd) 1131 { 1132 struct ovpn_softc *sc = ifp->if_softc; 1133 uint8_t *buf = NULL; 1134 nvlist_t *nvl = NULL; 1135 int ret; 1136 1137 if (ifd->ifd_len != 0) { 1138 if (ifd->ifd_len > OVPN_MAX_REQUEST_SIZE) 1139 return (E2BIG); 1140 1141 buf = malloc(ifd->ifd_len, M_OVPN, M_WAITOK); 1142 1143 ret = copyin(ifd->ifd_data, buf, ifd->ifd_len); 1144 if (ret != 0) { 1145 free(buf, M_OVPN); 1146 return (ret); 1147 } 1148 1149 nvl = nvlist_unpack(buf, ifd->ifd_len, 0); 1150 free(buf, M_OVPN); 1151 if (nvl == NULL) { 1152 return (EINVAL); 1153 } 1154 } 1155 1156 switch (ifd->ifd_cmd) { 1157 case OVPN_NEW_PEER: 1158 ret = ovpn_new_peer(ifp, nvl); 1159 break; 1160 case OVPN_DEL_PEER: 1161 OVPN_WLOCK(sc); 1162 ret = ovpn_del_peer(ifp, nvl); 1163 OVPN_WUNLOCK(sc); 1164 break; 1165 case OVPN_NEW_KEY: 1166 ret = ovpn_set_key(ifp, nvl); 1167 break; 1168 case OVPN_START_VPN: 1169 ret = ovpn_start(ifp); 1170 break; 1171 case OVPN_SWAP_KEYS: 1172 ret = ovpn_swap_keys(ifp, nvl); 1173 break; 1174 case OVPN_DEL_KEY: 1175 ret = ovpn_del_key(ifp, nvl); 1176 break; 1177 case OVPN_SET_PEER: 1178 ret = ovpn_set_peer(ifp, nvl); 1179 break; 1180 case OVPN_SET_IFMODE: 1181 ret = ovpn_set_ifmode(ifp, nvl); 1182 break; 1183 default: 1184 ret = ENOTSUP; 1185 } 1186 1187 nvlist_destroy(nvl); 1188 return (ret); 1189 } 1190 1191 static int 1192 ovpn_add_counters(nvlist_t *parent, const char *name, counter_u64_t in, 1193 counter_u64_t out) 1194 { 1195 nvlist_t *nvl; 1196 1197 nvl = nvlist_create(0); 1198 if (nvl == NULL) 1199 return (ENOMEM); 1200 1201 nvlist_add_number(nvl, "in", counter_u64_fetch(in)); 1202 nvlist_add_number(nvl, "out", counter_u64_fetch(out)); 1203 1204 nvlist_add_nvlist(parent, name, nvl); 1205 1206 nvlist_destroy(nvl); 1207 1208 return (0); 1209 } 1210 1211 static int 1212 ovpn_get_stats(struct ovpn_softc *sc, nvlist_t **onvl) 1213 { 1214 nvlist_t *nvl; 1215 int ret; 1216 1217 nvl = nvlist_create(0); 1218 if (nvl == NULL) 1219 return (ENOMEM); 1220 1221 #define OVPN_COUNTER_OUT(name, in, out) \ 1222 do { \ 1223 ret = ovpn_add_counters(nvl, name, OVPN_COUNTER(sc, in), \ 1224 OVPN_COUNTER(sc, out)); \ 1225 if (ret != 0) \ 1226 goto error; \ 1227 } while(0) 1228 1229 OVPN_COUNTER_OUT("lost_ctrl", lost_ctrl_pkts_in, lost_ctrl_pkts_out); 1230 OVPN_COUNTER_OUT("lost_data", lost_data_pkts_in, lost_data_pkts_out); 1231 OVPN_COUNTER_OUT("nomem_data", nomem_data_pkts_in, 1232 nomem_data_pkts_out); 1233 OVPN_COUNTER_OUT("data", received_data_pkts, sent_data_pkts); 1234 OVPN_COUNTER_OUT("ctrl", received_ctrl_pkts, sent_ctrl_pkts); 1235 OVPN_COUNTER_OUT("tunnel", tunnel_bytes_received, 1236 tunnel_bytes_received); 1237 OVPN_COUNTER_OUT("transport", transport_bytes_received, 1238 transport_bytes_received); 1239 #undef OVPN_COUNTER_OUT 1240 1241 *onvl = nvl; 1242 1243 return (0); 1244 1245 error: 1246 nvlist_destroy(nvl); 1247 return (ret); 1248 } 1249 1250 static int 1251 ovpn_get_peer_stats(struct ovpn_softc *sc, nvlist_t **nvl) 1252 { 1253 struct ovpn_kpeer *peer; 1254 nvlist_t *nvpeer = NULL; 1255 int ret; 1256 1257 OVPN_RLOCK_TRACKER; 1258 1259 *nvl = nvlist_create(0); 1260 if (*nvl == NULL) 1261 return (ENOMEM); 1262 1263 #define OVPN_PEER_COUNTER_OUT(name, in, out) \ 1264 do { \ 1265 ret = ovpn_add_counters(nvpeer, name, \ 1266 OVPN_PEER_COUNTER(peer, in), OVPN_PEER_COUNTER(peer, out)); \ 1267 if (ret != 0) \ 1268 goto error; \ 1269 } while(0) 1270 1271 OVPN_RLOCK(sc); 1272 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1273 nvpeer = nvlist_create(0); 1274 if (nvpeer == NULL) { 1275 OVPN_RUNLOCK(sc); 1276 nvlist_destroy(*nvl); 1277 *nvl = NULL; 1278 return (ENOMEM); 1279 } 1280 1281 nvlist_add_number(nvpeer, "peerid", peer->peerid); 1282 1283 OVPN_PEER_COUNTER_OUT("packets", pkt_in, pkt_out); 1284 OVPN_PEER_COUNTER_OUT("bytes", bytes_in, bytes_out); 1285 1286 nvlist_append_nvlist_array(*nvl, "peers", nvpeer); 1287 nvlist_destroy(nvpeer); 1288 } 1289 #undef OVPN_PEER_COUNTER_OUT 1290 OVPN_RUNLOCK(sc); 1291 1292 return (0); 1293 1294 error: 1295 nvlist_destroy(nvpeer); 1296 nvlist_destroy(*nvl); 1297 *nvl = NULL; 1298 return (ret); 1299 } 1300 1301 static int 1302 ovpn_poll_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1303 { 1304 nvlist_t *nvl; 1305 1306 nvl = nvlist_create(0); 1307 if (nvl == NULL) 1308 return (ENOMEM); 1309 1310 nvlist_add_number(nvl, "pending", buf_ring_count(sc->notifring)); 1311 1312 *onvl = nvl; 1313 1314 return (0); 1315 } 1316 1317 static void 1318 ovpn_notif_add_counters(nvlist_t *parent, struct ovpn_notification *n) 1319 { 1320 nvlist_t *nvl; 1321 1322 nvl = nvlist_create(0); 1323 if (nvl == NULL) 1324 return; 1325 1326 nvlist_add_number(nvl, "in", n->counters.pkt_in); 1327 nvlist_add_number(nvl, "out", n->counters.pkt_out); 1328 1329 nvlist_add_nvlist(parent, "packets", nvl); 1330 nvlist_destroy(nvl); 1331 1332 nvl = nvlist_create(0); 1333 if (nvl == NULL) 1334 return; 1335 1336 nvlist_add_number(nvl, "in", n->counters.bytes_in); 1337 nvlist_add_number(nvl, "out", n->counters.bytes_out); 1338 1339 nvlist_add_nvlist(parent, "bytes", nvl); 1340 nvlist_destroy(nvl); 1341 } 1342 1343 static int 1344 opvn_get_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1345 { 1346 struct ovpn_notification *n; 1347 nvlist_t *nvl; 1348 1349 /* Check if we have notifications pending. */ 1350 n = buf_ring_dequeue_mc(sc->notifring); 1351 if (n == NULL) 1352 return (ENOENT); 1353 1354 nvl = nvlist_create(0); 1355 if (nvl == NULL) { 1356 free(n, M_OVPN); 1357 return (ENOMEM); 1358 } 1359 nvlist_add_number(nvl, "peerid", n->peerid); 1360 nvlist_add_number(nvl, "notification", n->type); 1361 if (n->type == OVPN_NOTIF_DEL_PEER) { 1362 nvlist_add_number(nvl, "del_reason", n->del_reason); 1363 1364 /* No error handling, because we want to send the notification 1365 * even if we can't attach the counters. */ 1366 ovpn_notif_add_counters(nvl, n); 1367 } 1368 free(n, M_OVPN); 1369 1370 *onvl = nvl; 1371 1372 return (0); 1373 } 1374 1375 static int 1376 ovpn_ioctl_get(struct ifnet *ifp, struct ifdrv *ifd) 1377 { 1378 struct ovpn_softc *sc = ifp->if_softc; 1379 nvlist_t *nvl = NULL; 1380 int error; 1381 1382 switch (ifd->ifd_cmd) { 1383 case OVPN_GET_STATS: 1384 error = ovpn_get_stats(sc, &nvl); 1385 break; 1386 case OVPN_GET_PEER_STATS: 1387 error = ovpn_get_peer_stats(sc, &nvl); 1388 break; 1389 case OVPN_POLL_PKT: 1390 error = ovpn_poll_pkt(sc, &nvl); 1391 break; 1392 case OVPN_GET_PKT: 1393 error = opvn_get_pkt(sc, &nvl); 1394 break; 1395 default: 1396 error = ENOTSUP; 1397 break; 1398 } 1399 1400 if (error == 0) { 1401 void *packed = NULL; 1402 size_t len; 1403 1404 MPASS(nvl != NULL); 1405 1406 packed = nvlist_pack(nvl, &len); 1407 if (! packed) { 1408 nvlist_destroy(nvl); 1409 return (ENOMEM); 1410 } 1411 1412 if (len > ifd->ifd_len) { 1413 free(packed, M_NVLIST); 1414 nvlist_destroy(nvl); 1415 return (ENOSPC); 1416 } 1417 1418 error = copyout(packed, ifd->ifd_data, len); 1419 ifd->ifd_len = len; 1420 1421 free(packed, M_NVLIST); 1422 nvlist_destroy(nvl); 1423 } 1424 1425 return (error); 1426 } 1427 1428 static int 1429 ovpn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 1430 { 1431 struct ifdrv *ifd; 1432 int error; 1433 1434 CURVNET_ASSERT_SET(); 1435 1436 switch (cmd) { 1437 case SIOCSDRVSPEC: 1438 case SIOCGDRVSPEC: 1439 error = priv_check(curthread, PRIV_NET_OVPN); 1440 if (error) 1441 return (error); 1442 break; 1443 } 1444 1445 switch (cmd) { 1446 case SIOCSDRVSPEC: 1447 ifd = (struct ifdrv *)data; 1448 error = ovpn_ioctl_set(ifp, ifd); 1449 break; 1450 case SIOCGDRVSPEC: 1451 ifd = (struct ifdrv *)data; 1452 error = ovpn_ioctl_get(ifp, ifd); 1453 break; 1454 case SIOCSIFMTU: { 1455 struct ifreq *ifr = (struct ifreq *)data; 1456 if (ifr->ifr_mtu < OVPN_MTU_MIN || ifr->ifr_mtu > OVPN_MTU_MAX) 1457 return (EINVAL); 1458 1459 ifp->if_mtu = ifr->ifr_mtu; 1460 return (0); 1461 } 1462 case SIOCSIFADDR: 1463 case SIOCADDMULTI: 1464 case SIOCDELMULTI: 1465 case SIOCGIFMTU: 1466 case SIOCSIFFLAGS: 1467 return (0); 1468 default: 1469 error = EINVAL; 1470 } 1471 1472 return (error); 1473 } 1474 1475 static int 1476 ovpn_encrypt_tx_cb(struct cryptop *crp) 1477 { 1478 struct epoch_tracker et; 1479 struct ovpn_kpeer *peer = crp->crp_opaque; 1480 struct ovpn_softc *sc = peer->sc; 1481 struct mbuf *m = crp->crp_buf.cb_mbuf; 1482 int tunnel_len; 1483 int ret; 1484 1485 CURVNET_SET(sc->ifp->if_vnet); 1486 NET_EPOCH_ENTER(et); 1487 1488 if (crp->crp_etype != 0) { 1489 crypto_freereq(crp); 1490 ovpn_peer_release_ref(peer, false); 1491 NET_EPOCH_EXIT(et); 1492 CURVNET_RESTORE(); 1493 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1494 m_freem(m); 1495 return (0); 1496 } 1497 1498 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1499 1500 tunnel_len = m->m_pkthdr.len - sizeof(struct ovpn_wire_header); 1501 ret = ovpn_encap(sc, peer->peerid, m); 1502 if (ret == 0) { 1503 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1504 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 1505 } 1506 1507 crypto_freereq(crp); 1508 ovpn_peer_release_ref(peer, false); 1509 1510 NET_EPOCH_EXIT(et); 1511 CURVNET_RESTORE(); 1512 1513 return (0); 1514 } 1515 1516 static void 1517 ovpn_finish_rx(struct ovpn_softc *sc, struct mbuf *m, 1518 struct ovpn_kpeer *peer, struct ovpn_kkey *key, uint32_t seq, 1519 struct rm_priotracker *_ovpn_lock_trackerp) 1520 { 1521 uint32_t af; 1522 1523 OVPN_RASSERT(sc); 1524 NET_EPOCH_ASSERT(); 1525 1526 /* Replay protection. */ 1527 if (V_replay_protection && ! ovpn_check_replay(key->decrypt, seq)) { 1528 OVPN_RUNLOCK(sc); 1529 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1530 m_freem(m); 1531 return; 1532 } 1533 1534 critical_enter(); 1535 *zpcpu_get(peer->last_active) = time_uptime; 1536 critical_exit(); 1537 1538 OVPN_RUNLOCK(sc); 1539 1540 OVPN_COUNTER_ADD(sc, received_data_pkts, 1); 1541 OVPN_COUNTER_ADD(sc, tunnel_bytes_received, m->m_pkthdr.len); 1542 OVPN_PEER_COUNTER_ADD(peer, pkt_in, 1); 1543 OVPN_PEER_COUNTER_ADD(peer, bytes_in, m->m_pkthdr.len); 1544 1545 /* Receive the packet on our interface. */ 1546 m->m_pkthdr.rcvif = sc->ifp; 1547 1548 /* Clear checksum flags in case the real hardware set them. */ 1549 m->m_pkthdr.csum_flags = 0; 1550 1551 /* Clear mbuf tags & flags */ 1552 m_tag_delete_nonpersistent(m); 1553 m_clrprotoflags(m); 1554 1555 /* Ensure we can read the first byte. */ 1556 m = m_pullup(m, 1); 1557 if (m == NULL) { 1558 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 1559 return; 1560 } 1561 1562 /* 1563 * Check for address family, and disregard any control packets (e.g. 1564 * keepalive). 1565 */ 1566 af = ovpn_get_af(m); 1567 if (af != 0) { 1568 BPF_MTAP2(sc->ifp, &af, sizeof(af), m); 1569 if (V_async_netisr_queue) 1570 netisr_queue(af == AF_INET ? NETISR_IP : NETISR_IPV6, m); 1571 else 1572 netisr_dispatch(af == AF_INET ? NETISR_IP : NETISR_IPV6, m); 1573 } else { 1574 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1575 m_freem(m); 1576 } 1577 } 1578 1579 static struct ovpn_kkey * 1580 ovpn_find_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, 1581 const struct ovpn_wire_header *ohdr) 1582 { 1583 struct ovpn_kkey *key = NULL; 1584 uint8_t keyid; 1585 1586 OVPN_RASSERT(sc); 1587 1588 keyid = (ntohl(ohdr->opcode) >> 24) & 0x07; 1589 1590 if (peer->keys[0].keyid == keyid) 1591 key = &peer->keys[0]; 1592 else if (peer->keys[1].keyid == keyid) 1593 key = &peer->keys[1]; 1594 1595 return (key); 1596 } 1597 1598 static int 1599 ovpn_decrypt_rx_cb(struct cryptop *crp) 1600 { 1601 struct epoch_tracker et; 1602 struct ovpn_softc *sc = crp->crp_opaque; 1603 struct mbuf *m = crp->crp_buf.cb_mbuf; 1604 struct ovpn_kkey *key; 1605 struct ovpn_kpeer *peer; 1606 struct ovpn_wire_header *ohdr; 1607 uint32_t peerid; 1608 1609 OVPN_RLOCK_TRACKER; 1610 1611 OVPN_RLOCK(sc); 1612 1613 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1614 1615 if (crp->crp_etype != 0) { 1616 crypto_freereq(crp); 1617 atomic_add_int(&sc->refcount, -1); 1618 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1619 OVPN_RUNLOCK(sc); 1620 m_freem(m); 1621 return (0); 1622 } 1623 1624 CURVNET_SET(sc->ifp->if_vnet); 1625 1626 ohdr = mtodo(m, sizeof(struct udphdr)); 1627 1628 peerid = ntohl(ohdr->opcode) & 0x00ffffff; 1629 peer = ovpn_find_peer(sc, peerid); 1630 if (peer == NULL) { 1631 /* No such peer. Drop packet. */ 1632 crypto_freereq(crp); 1633 atomic_add_int(&sc->refcount, -1); 1634 OVPN_RUNLOCK(sc); 1635 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1636 m_freem(m); 1637 CURVNET_RESTORE(); 1638 return (0); 1639 } 1640 1641 key = ovpn_find_key(sc, peer, ohdr); 1642 if (key == NULL) { 1643 crypto_freereq(crp); 1644 atomic_add_int(&sc->refcount, -1); 1645 /* 1646 * Has this key been removed between us starting the decrypt 1647 * and finishing it? 1648 */ 1649 OVPN_RUNLOCK(sc); 1650 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1651 m_freem(m); 1652 CURVNET_RESTORE(); 1653 return (0); 1654 } 1655 1656 /* Now remove the outer headers */ 1657 m_adj_decap(m, sizeof(struct udphdr) + 1658 sizeof(struct ovpn_wire_header)); 1659 1660 NET_EPOCH_ENTER(et); 1661 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), _ovpn_lock_trackerp); 1662 NET_EPOCH_EXIT(et); 1663 OVPN_UNLOCK_ASSERT(sc); 1664 1665 CURVNET_RESTORE(); 1666 1667 crypto_freereq(crp); 1668 atomic_add_int(&sc->refcount, -1); 1669 1670 return (0); 1671 } 1672 1673 static int 1674 ovpn_get_af(struct mbuf *m) 1675 { 1676 struct ip *ip; 1677 struct ip6_hdr *ip6; 1678 1679 /* 1680 * We should pullup, but we're only interested in the first byte, so 1681 * that'll always be contiguous. 1682 */ 1683 ip = mtod(m, struct ip *); 1684 if (ip->ip_v == IPVERSION) 1685 return (AF_INET); 1686 1687 ip6 = mtod(m, struct ip6_hdr *); 1688 if ((ip6->ip6_vfc & IPV6_VERSION_MASK) == IPV6_VERSION) 1689 return (AF_INET6); 1690 1691 return (0); 1692 } 1693 1694 #ifdef INET 1695 static struct ovpn_kpeer * 1696 ovpn_find_peer_by_ip(struct ovpn_softc *sc, const struct in_addr addr) 1697 { 1698 struct ovpn_kpeer *peer = NULL; 1699 1700 OVPN_ASSERT(sc); 1701 1702 /* TODO: Add a second RB so we can look up by IP. */ 1703 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1704 if (addr.s_addr == peer->vpn4.s_addr) 1705 return (peer); 1706 } 1707 1708 return (peer); 1709 } 1710 #endif 1711 1712 #ifdef INET6 1713 static struct ovpn_kpeer * 1714 ovpn_find_peer_by_ip6(struct ovpn_softc *sc, const struct in6_addr *addr) 1715 { 1716 struct ovpn_kpeer *peer = NULL; 1717 1718 OVPN_ASSERT(sc); 1719 1720 /* TODO: Add a third RB so we can look up by IPv6 address. */ 1721 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1722 if (memcmp(addr, &peer->vpn6, sizeof(*addr)) == 0) 1723 return (peer); 1724 } 1725 1726 return (peer); 1727 } 1728 #endif 1729 1730 static struct ovpn_kpeer * 1731 ovpn_route_peer(struct ovpn_softc *sc, struct mbuf **m0, 1732 const struct sockaddr *dst) 1733 { 1734 struct ovpn_kpeer *peer = NULL; 1735 int af; 1736 1737 NET_EPOCH_ASSERT(); 1738 OVPN_ASSERT(sc); 1739 1740 /* Shortcut if we're a client (or are a server and have only one client). */ 1741 if (sc->peercount == 1) 1742 return (ovpn_find_only_peer(sc)); 1743 1744 if (dst != NULL) 1745 af = dst->sa_family; 1746 else 1747 af = ovpn_get_af(*m0); 1748 1749 switch (af) { 1750 #ifdef INET 1751 case AF_INET: { 1752 const struct sockaddr_in *sa = (const struct sockaddr_in *)dst; 1753 struct nhop_object *nh; 1754 const struct in_addr *ip_dst; 1755 1756 if (sa != NULL) { 1757 ip_dst = &sa->sin_addr; 1758 } else { 1759 struct ip *ip; 1760 1761 *m0 = m_pullup(*m0, sizeof(struct ip)); 1762 if (*m0 == NULL) 1763 return (NULL); 1764 ip = mtod(*m0, struct ip *); 1765 ip_dst = &ip->ip_dst; 1766 } 1767 1768 peer = ovpn_find_peer_by_ip(sc, *ip_dst); 1769 SDT_PROBE2(if_ovpn, tx, route, ip4, ip_dst, peer); 1770 if (peer == NULL) { 1771 nh = fib4_lookup(M_GETFIB(*m0), *ip_dst, 0, 1772 NHR_NONE, 0); 1773 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1774 peer = ovpn_find_peer_by_ip(sc, 1775 nh->gw4_sa.sin_addr); 1776 SDT_PROBE2(if_ovpn, tx, route, ip4, 1777 &nh->gw4_sa.sin_addr, peer); 1778 } 1779 } 1780 break; 1781 } 1782 #endif 1783 #ifdef INET6 1784 case AF_INET6: { 1785 const struct sockaddr_in6 *sa6 = 1786 (const struct sockaddr_in6 *)dst; 1787 struct nhop_object *nh; 1788 const struct in6_addr *ip6_dst; 1789 1790 if (sa6 != NULL) { 1791 ip6_dst = &sa6->sin6_addr; 1792 } else { 1793 struct ip6_hdr *ip6; 1794 1795 *m0 = m_pullup(*m0, sizeof(struct ip6_hdr)); 1796 if (*m0 == NULL) 1797 return (NULL); 1798 ip6 = mtod(*m0, struct ip6_hdr *); 1799 ip6_dst = &ip6->ip6_dst; 1800 } 1801 1802 peer = ovpn_find_peer_by_ip6(sc, ip6_dst); 1803 SDT_PROBE2(if_ovpn, tx, route, ip6, ip6_dst, peer); 1804 if (peer == NULL) { 1805 nh = fib6_lookup(M_GETFIB(*m0), ip6_dst, 0, 1806 NHR_NONE, 0); 1807 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1808 peer = ovpn_find_peer_by_ip6(sc, 1809 &nh->gw6_sa.sin6_addr); 1810 SDT_PROBE2(if_ovpn, tx, route, ip6, 1811 &nh->gw6_sa.sin6_addr, peer); 1812 } 1813 } 1814 break; 1815 } 1816 #endif 1817 } 1818 1819 return (peer); 1820 } 1821 1822 static int 1823 ovpn_transmit(struct ifnet *ifp, struct mbuf *m) 1824 { 1825 return (ifp->if_output(ifp, m, NULL, NULL)); 1826 } 1827 1828 static int 1829 ovpn_transmit_to_peer(struct ifnet *ifp, struct mbuf *m, 1830 struct ovpn_kpeer *peer, struct rm_priotracker *_ovpn_lock_trackerp) 1831 { 1832 struct ovpn_wire_header *ohdr; 1833 struct ovpn_kkey *key; 1834 struct ovpn_softc *sc; 1835 struct cryptop *crp; 1836 uint32_t af, seq; 1837 uint64_t seq64; 1838 size_t len, ovpn_hdr_len; 1839 int tunnel_len; 1840 int ret; 1841 1842 sc = ifp->if_softc; 1843 1844 OVPN_RASSERT(sc); 1845 1846 tunnel_len = m->m_pkthdr.len; 1847 1848 key = &peer->keys[OVPN_KEY_SLOT_PRIMARY]; 1849 if (key->encrypt == NULL) { 1850 if (_ovpn_lock_trackerp != NULL) 1851 OVPN_RUNLOCK(sc); 1852 m_freem(m); 1853 return (ENOLINK); 1854 } 1855 1856 af = ovpn_get_af(m); 1857 /* Don't capture control packets. */ 1858 if (af != 0) 1859 BPF_MTAP2(ifp, &af, sizeof(af), m); 1860 1861 len = m->m_pkthdr.len; 1862 MPASS(len <= ifp->if_mtu); 1863 1864 ovpn_hdr_len = sizeof(struct ovpn_wire_header); 1865 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) 1866 ovpn_hdr_len -= 16; /* No auth tag. */ 1867 1868 M_PREPEND(m, ovpn_hdr_len, M_NOWAIT); 1869 if (m == NULL) { 1870 if (_ovpn_lock_trackerp != NULL) 1871 OVPN_RUNLOCK(sc); 1872 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1873 return (ENOBUFS); 1874 } 1875 ohdr = mtod(m, struct ovpn_wire_header *); 1876 ohdr->opcode = (OVPN_OP_DATA_V2 << OVPN_OP_SHIFT) | key->keyid; 1877 ohdr->opcode <<= 24; 1878 ohdr->opcode |= key->peerid; 1879 ohdr->opcode = htonl(ohdr->opcode); 1880 1881 seq64 = atomic_fetchadd_64(&peer->keys[OVPN_KEY_SLOT_PRIMARY].encrypt->tx_seq, 1); 1882 if (seq64 == OVPN_SEQ_ROTATE) { 1883 ovpn_notify_key_rotation(sc, peer); 1884 } else if (seq64 > UINT32_MAX) { 1885 /* We've wrapped, give up on this packet. */ 1886 if (_ovpn_lock_trackerp != NULL) 1887 OVPN_RUNLOCK(sc); 1888 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1889 1890 /* Let's avoid (very unlikely, but still) wraparounds of the 1891 * 64-bit counter taking us back to 0. */ 1892 atomic_store_64(&peer->keys[OVPN_KEY_SLOT_PRIMARY].encrypt->tx_seq, 1893 UINT32_MAX); 1894 1895 return (ENOBUFS); 1896 } 1897 1898 seq = htonl(seq64 & UINT32_MAX); 1899 ohdr->seq = seq; 1900 1901 OVPN_PEER_COUNTER_ADD(peer, pkt_out, 1); 1902 OVPN_PEER_COUNTER_ADD(peer, bytes_out, len); 1903 1904 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) { 1905 ret = ovpn_encap(sc, peer->peerid, m); 1906 if (_ovpn_lock_trackerp != NULL) 1907 OVPN_RUNLOCK(sc); 1908 if (ret == 0) { 1909 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1910 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 1911 } 1912 return (ret); 1913 } 1914 1915 crp = crypto_getreq(key->encrypt->cryptoid, M_NOWAIT); 1916 if (crp == NULL) { 1917 if (_ovpn_lock_trackerp != NULL) 1918 OVPN_RUNLOCK(sc); 1919 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1920 m_freem(m); 1921 return (ENOBUFS); 1922 } 1923 1924 /* Encryption covers only the payload, not the header. */ 1925 crp->crp_payload_start = sizeof(*ohdr); 1926 crp->crp_payload_length = len; 1927 crp->crp_op = CRYPTO_OP_ENCRYPT; 1928 1929 /* 1930 * AAD data covers the ovpn_wire_header minus the auth 1931 * tag. 1932 */ 1933 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 1934 crp->crp_aad = ohdr; 1935 crp->crp_aad_start = 0; 1936 crp->crp_op |= CRYPTO_OP_COMPUTE_DIGEST; 1937 crp->crp_digest_start = offsetof(struct ovpn_wire_header, auth_tag); 1938 1939 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 1940 memcpy(crp->crp_iv, &seq, sizeof(seq)); 1941 memcpy(crp->crp_iv + sizeof(seq), key->encrypt->nonce, 1942 key->encrypt->noncelen); 1943 1944 crypto_use_mbuf(crp, m); 1945 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 1946 crp->crp_callback = ovpn_encrypt_tx_cb; 1947 crp->crp_opaque = peer; 1948 1949 atomic_add_int(&peer->refcount, 1); 1950 if (_ovpn_lock_trackerp != NULL) 1951 OVPN_RUNLOCK(sc); 1952 if (V_async_crypto) 1953 ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); 1954 else 1955 ret = crypto_dispatch(crp); 1956 if (ret) { 1957 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1958 } 1959 1960 return (ret); 1961 } 1962 1963 /* 1964 * Note: Expects to hold the read lock on entry, and will release it itself. 1965 */ 1966 static int 1967 ovpn_encap(struct ovpn_softc *sc, uint32_t peerid, struct mbuf *m) 1968 { 1969 struct udphdr *udp; 1970 struct ovpn_kpeer *peer; 1971 int len; 1972 1973 OVPN_RLOCK_TRACKER; 1974 1975 OVPN_RLOCK(sc); 1976 NET_EPOCH_ASSERT(); 1977 1978 peer = ovpn_find_peer(sc, peerid); 1979 if (peer == NULL || sc->ifp->if_link_state != LINK_STATE_UP) { 1980 OVPN_RUNLOCK(sc); 1981 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1982 m_freem(m); 1983 return (ENETDOWN); 1984 } 1985 1986 len = m->m_pkthdr.len; 1987 1988 M_PREPEND(m, sizeof(struct udphdr), M_NOWAIT); 1989 if (m == NULL) { 1990 OVPN_RUNLOCK(sc); 1991 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1992 m_freem(m); 1993 return (ENOBUFS); 1994 } 1995 udp = mtod(m, struct udphdr *); 1996 1997 MPASS(peer->local.ss_family == peer->remote.ss_family); 1998 1999 udp->uh_sport = ovpn_get_port(&peer->local); 2000 udp->uh_dport = ovpn_get_port(&peer->remote); 2001 udp->uh_ulen = htons(sizeof(struct udphdr) + len); 2002 2003 switch (peer->remote.ss_family) { 2004 #ifdef INET 2005 case AF_INET: { 2006 struct sockaddr_in *in_local = TO_IN(&peer->local); 2007 struct sockaddr_in *in_remote = TO_IN(&peer->remote); 2008 struct ip *ip; 2009 2010 /* 2011 * This requires knowing the source IP, which we don't. Happily 2012 * we're allowed to keep this at 0, and the checksum won't do 2013 * anything the crypto won't already do. 2014 */ 2015 udp->uh_sum = 0; 2016 2017 /* Set the checksum flags so we recalculate checksums. */ 2018 m->m_pkthdr.csum_flags |= CSUM_IP; 2019 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 2020 2021 M_PREPEND(m, sizeof(struct ip), M_NOWAIT); 2022 if (m == NULL) { 2023 OVPN_RUNLOCK(sc); 2024 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2025 return (ENOBUFS); 2026 } 2027 ip = mtod(m, struct ip *); 2028 2029 ip->ip_tos = 0; 2030 ip->ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) + 2031 len); 2032 ip->ip_off = 0; 2033 ip->ip_ttl = V_ip_defttl; 2034 ip->ip_p = IPPROTO_UDP; 2035 ip->ip_sum = 0; 2036 if (in_local->sin_port != 0) 2037 ip->ip_src = in_local->sin_addr; 2038 else 2039 ip->ip_src.s_addr = INADDR_ANY; 2040 ip->ip_dst = in_remote->sin_addr; 2041 2042 OVPN_RUNLOCK(sc); 2043 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 2044 2045 return (ip_output(m, NULL, NULL, 0, NULL, NULL)); 2046 } 2047 #endif 2048 #ifdef INET6 2049 case AF_INET6: { 2050 struct sockaddr_in6 *in6_local = TO_IN6(&peer->local); 2051 struct sockaddr_in6 *in6_remote = TO_IN6(&peer->remote); 2052 struct ip6_hdr *ip6; 2053 2054 M_PREPEND(m, sizeof(struct ip6_hdr), M_NOWAIT); 2055 if (m == NULL) { 2056 OVPN_RUNLOCK(sc); 2057 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2058 return (ENOBUFS); 2059 } 2060 m = m_pullup(m, sizeof(*ip6) + sizeof(*udp)); 2061 if (m == NULL) { 2062 OVPN_RUNLOCK(sc); 2063 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2064 return (ENOBUFS); 2065 } 2066 2067 ip6 = mtod(m, struct ip6_hdr *); 2068 2069 ip6->ip6_vfc = IPV6_VERSION; 2070 ip6->ip6_flow &= ~IPV6_FLOWINFO_MASK; 2071 ip6->ip6_plen = htons(sizeof(*ip6) + sizeof(struct udphdr) + 2072 len); 2073 ip6->ip6_nxt = IPPROTO_UDP; 2074 ip6->ip6_hlim = V_ip6_defhlim; 2075 2076 memcpy(&ip6->ip6_src, &in6_local->sin6_addr, 2077 sizeof(ip6->ip6_src)); 2078 memcpy(&ip6->ip6_dst, &in6_remote->sin6_addr, 2079 sizeof(ip6->ip6_dst)); 2080 2081 udp = mtodo(m, sizeof(*ip6)); 2082 udp->uh_sum = in6_cksum_pseudo(ip6, 2083 m->m_pkthdr.len - sizeof(struct ip6_hdr), 2084 IPPROTO_UDP, 0); 2085 2086 m->m_pkthdr.csum_flags |= CSUM_UDP_IPV6; 2087 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 2088 2089 OVPN_RUNLOCK(sc); 2090 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 2091 2092 return (ip6_output(m, NULL, NULL, IPV6_UNSPECSRC, NULL, NULL, 2093 NULL)); 2094 } 2095 #endif 2096 default: 2097 panic("Unsupported address family %d", 2098 peer->remote.ss_family); 2099 } 2100 } 2101 2102 static int 2103 ovpn_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 2104 struct route *ro) 2105 { 2106 struct ovpn_softc *sc; 2107 struct ovpn_kpeer *peer; 2108 2109 OVPN_RLOCK_TRACKER; 2110 2111 sc = ifp->if_softc; 2112 2113 OVPN_RLOCK(sc); 2114 2115 SDT_PROBE1(if_ovpn, tx, transmit, start, m); 2116 2117 if (__predict_false(ifp->if_link_state != LINK_STATE_UP)) { 2118 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2119 OVPN_RUNLOCK(sc); 2120 m_freem(m); 2121 return (ENETDOWN); 2122 } 2123 2124 /** 2125 * Only obey 'dst' (i.e. the gateway) if no route is supplied. 2126 * That's our indication that we're being called through pf's route-to, 2127 * and we should route according to 'dst' instead. We can't do so 2128 * consistently, because the usual openvpn configuration sets the first 2129 * non-server IP in the subnet as the gateway. If we always use that 2130 * one we'd end up routing all traffic to the first client. 2131 * tl;dr: 'ro == NULL' tells us pf is doing a route-to, and then but 2132 * only then, we should treat 'dst' as the destination. */ 2133 peer = ovpn_route_peer(sc, &m, ro == NULL ? dst : NULL); 2134 if (peer == NULL) { 2135 /* No destination. */ 2136 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2137 OVPN_RUNLOCK(sc); 2138 m_freem(m); 2139 return (ENETDOWN); 2140 } 2141 2142 return (ovpn_transmit_to_peer(ifp, m, peer, _ovpn_lock_trackerp)); 2143 } 2144 2145 static bool 2146 ovpn_check_replay(struct ovpn_kkey_dir *key, uint32_t seq) 2147 { 2148 uint32_t d; 2149 2150 mtx_lock(&key->replay_mtx); 2151 2152 /* Sequence number must be strictly greater than rx_seq */ 2153 if (seq <= key->rx_seq) { 2154 mtx_unlock(&key->replay_mtx); 2155 return (false); 2156 } 2157 2158 /* Large jump. The packet authenticated okay, so just accept that. */ 2159 if (seq > (key->rx_seq + (sizeof(key->rx_window) * 8))) { 2160 key->rx_seq = seq; 2161 key->rx_window = 0; 2162 mtx_unlock(&key->replay_mtx); 2163 return (true); 2164 } 2165 2166 /* Happy case. */ 2167 if ((seq == key->rx_seq + 1) && key->rx_window == 0) { 2168 key->rx_seq++; 2169 mtx_unlock(&key->replay_mtx); 2170 return (true); 2171 } 2172 2173 d = seq - key->rx_seq - 1; 2174 2175 if (key->rx_window & ((uint64_t)1 << d)) { 2176 /* Dupe! */ 2177 mtx_unlock(&key->replay_mtx); 2178 return (false); 2179 } 2180 2181 key->rx_window |= (uint64_t)1 << d; 2182 2183 while (key->rx_window & 1) { 2184 key->rx_seq++; 2185 key->rx_window >>= 1; 2186 } 2187 2188 mtx_unlock(&key->replay_mtx); 2189 2190 return (true); 2191 } 2192 2193 static struct ovpn_kpeer * 2194 ovpn_peer_from_mbuf(struct ovpn_softc *sc, struct mbuf *m, int off) 2195 { 2196 struct ovpn_wire_header ohdr; 2197 uint32_t peerid; 2198 const size_t hdrlen = sizeof(ohdr) - sizeof(ohdr.auth_tag); 2199 2200 OVPN_RASSERT(sc); 2201 2202 if (m_length(m, NULL) < (off + sizeof(struct udphdr) + hdrlen)) 2203 return (NULL); 2204 2205 m_copydata(m, off + sizeof(struct udphdr), hdrlen, (caddr_t)&ohdr); 2206 2207 peerid = ntohl(ohdr.opcode) & 0x00ffffff; 2208 2209 return (ovpn_find_peer(sc, peerid)); 2210 } 2211 2212 static bool 2213 ovpn_udp_input(struct mbuf *m, int off, struct inpcb *inp, 2214 const struct sockaddr *sa, void *ctx) 2215 { 2216 struct ovpn_softc *sc = ctx; 2217 struct ovpn_wire_header tmphdr; 2218 struct ovpn_wire_header *ohdr; 2219 struct udphdr *uhdr; 2220 struct ovpn_kkey *key; 2221 struct cryptop *crp; 2222 struct ovpn_kpeer *peer; 2223 size_t ohdrlen; 2224 int ret; 2225 uint8_t op; 2226 2227 OVPN_RLOCK_TRACKER; 2228 2229 M_ASSERTPKTHDR(m); 2230 2231 OVPN_COUNTER_ADD(sc, transport_bytes_received, m->m_pkthdr.len - off); 2232 2233 ohdrlen = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2234 2235 OVPN_RLOCK(sc); 2236 2237 peer = ovpn_peer_from_mbuf(sc, m, off); 2238 if (peer == NULL) { 2239 OVPN_RUNLOCK(sc); 2240 return (false); 2241 } 2242 2243 if (m_length(m, NULL) < (off + sizeof(*uhdr) + ohdrlen)) { 2244 /* Short packet. */ 2245 OVPN_RUNLOCK(sc); 2246 return (false); 2247 } 2248 2249 m_copydata(m, off + sizeof(*uhdr), ohdrlen, (caddr_t)&tmphdr); 2250 2251 op = ntohl(tmphdr.opcode) >> 24 >> OVPN_OP_SHIFT; 2252 if (op != OVPN_OP_DATA_V2) { 2253 /* Control packet? */ 2254 OVPN_RUNLOCK(sc); 2255 return (false); 2256 } 2257 2258 m = m_pullup(m, off + sizeof(*uhdr) + ohdrlen); 2259 if (m == NULL) { 2260 OVPN_RUNLOCK(sc); 2261 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2262 return (true); 2263 } 2264 2265 /* 2266 * Simplify things by getting rid of the preceding headers, we don't 2267 * care about them. 2268 */ 2269 m_adj_decap(m, off); 2270 2271 uhdr = mtodo(m, 0); 2272 ohdr = mtodo(m, sizeof(*uhdr)); 2273 2274 key = ovpn_find_key(sc, peer, ohdr); 2275 if (key == NULL || key->decrypt == NULL) { 2276 OVPN_RUNLOCK(sc); 2277 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2278 m_freem(m); 2279 return (true); 2280 } 2281 2282 if (key->decrypt->cipher == OVPN_CIPHER_ALG_NONE) { 2283 /* Now remove the outer headers */ 2284 m_adj_decap(m, sizeof(struct udphdr) + ohdrlen); 2285 2286 ohdr = mtodo(m, sizeof(*uhdr)); 2287 2288 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), 2289 _ovpn_lock_trackerp); 2290 OVPN_UNLOCK_ASSERT(sc); 2291 return (true); 2292 } 2293 2294 ohdrlen += sizeof(ohdr->auth_tag); 2295 2296 m = m_pullup(m, sizeof(*uhdr) + ohdrlen); 2297 if (m == NULL) { 2298 OVPN_RUNLOCK(sc); 2299 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2300 return (true); 2301 } 2302 uhdr = mtodo(m, 0); 2303 ohdr = mtodo(m, sizeof(*uhdr)); 2304 2305 /* Decrypt */ 2306 crp = crypto_getreq(key->decrypt->cryptoid, M_NOWAIT); 2307 if (crp == NULL) { 2308 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2309 OVPN_RUNLOCK(sc); 2310 m_freem(m); 2311 return (true); 2312 } 2313 2314 crp->crp_payload_start = sizeof(struct udphdr) + sizeof(*ohdr); 2315 crp->crp_payload_length = ntohs(uhdr->uh_ulen) - 2316 sizeof(*uhdr) - sizeof(*ohdr); 2317 crp->crp_op = CRYPTO_OP_DECRYPT; 2318 2319 /* AAD validation. */ 2320 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2321 crp->crp_aad = ohdr; 2322 crp->crp_aad_start = 0; 2323 crp->crp_op |= CRYPTO_OP_VERIFY_DIGEST; 2324 crp->crp_digest_start = sizeof(struct udphdr) + 2325 offsetof(struct ovpn_wire_header, auth_tag); 2326 2327 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 2328 memcpy(crp->crp_iv, &ohdr->seq, sizeof(ohdr->seq)); 2329 memcpy(crp->crp_iv + sizeof(ohdr->seq), key->decrypt->nonce, 2330 key->decrypt->noncelen); 2331 2332 crypto_use_mbuf(crp, m); 2333 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 2334 crp->crp_callback = ovpn_decrypt_rx_cb; 2335 crp->crp_opaque = sc; 2336 2337 atomic_add_int(&sc->refcount, 1); 2338 OVPN_RUNLOCK(sc); 2339 if (V_async_crypto) 2340 ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); 2341 else 2342 ret = crypto_dispatch(crp); 2343 if (ret != 0) { 2344 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2345 } 2346 2347 return (true); 2348 } 2349 2350 static void 2351 ovpn_qflush(struct ifnet *ifp __unused) 2352 { 2353 2354 } 2355 2356 static void 2357 ovpn_flush_rxring(struct ovpn_softc *sc) 2358 { 2359 struct ovpn_notification *n; 2360 2361 OVPN_WASSERT(sc); 2362 2363 while (! buf_ring_empty(sc->notifring)) { 2364 n = buf_ring_dequeue_sc(sc->notifring); 2365 free(n, M_OVPN); 2366 } 2367 } 2368 2369 #ifdef VIMAGE 2370 static void 2371 ovpn_reassign(struct ifnet *ifp, struct vnet *new_vnet __unused, 2372 char *unused __unused) 2373 { 2374 struct ovpn_softc *sc = ifp->if_softc; 2375 struct ovpn_kpeer *peer, *tmppeer; 2376 int ret __diagused; 2377 2378 OVPN_WLOCK(sc); 2379 2380 /* Flush keys & configuration. */ 2381 RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) { 2382 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 2383 ret = _ovpn_del_peer(sc, peer); 2384 MPASS(ret == 0); 2385 } 2386 2387 ovpn_flush_rxring(sc); 2388 2389 OVPN_WUNLOCK(sc); 2390 } 2391 #endif 2392 2393 static int 2394 ovpn_clone_match(struct if_clone *ifc, const char *name) 2395 { 2396 /* 2397 * Allow all names that start with 'ovpn', specifically because pfSense 2398 * uses ovpnc1 / ovpns2 2399 */ 2400 return (strncmp(ovpnname, name, strlen(ovpnname)) == 0); 2401 } 2402 2403 static int 2404 ovpn_clone_create(struct if_clone *ifc, char *name, size_t len, 2405 struct ifc_data *ifd, struct ifnet **ifpp) 2406 { 2407 struct ovpn_softc *sc; 2408 struct ifnet *ifp; 2409 char *dp; 2410 int error, unit, wildcard; 2411 2412 /* Try to see if a special unit was requested. */ 2413 error = ifc_name2unit(name, &unit); 2414 if (error != 0) 2415 return (error); 2416 wildcard = (unit < 0); 2417 2418 error = ifc_alloc_unit(ifc, &unit); 2419 if (error != 0) 2420 return (error); 2421 2422 /* 2423 * If no unit had been given, we need to adjust the ifName. 2424 */ 2425 for (dp = name; *dp != '\0'; dp++); 2426 if (wildcard) { 2427 error = snprintf(dp, len - (dp - name), "%d", unit); 2428 if (error > len - (dp - name)) { 2429 /* ifName too long. */ 2430 ifc_free_unit(ifc, unit); 2431 return (ENOSPC); 2432 } 2433 dp += error; 2434 } 2435 2436 /* Make sure it doesn't already exist. */ 2437 if (ifunit(name) != NULL) 2438 return (EEXIST); 2439 2440 sc = malloc(sizeof(struct ovpn_softc), M_OVPN, M_WAITOK | M_ZERO); 2441 sc->ifp = if_alloc(IFT_ENC); 2442 rm_init_flags(&sc->lock, "if_ovpn_lock", RM_RECURSE); 2443 sc->refcount = 0; 2444 2445 sc->notifring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL); 2446 2447 COUNTER_ARRAY_ALLOC(sc->counters, OVPN_COUNTER_SIZE, M_WAITOK); 2448 2449 ifp = sc->ifp; 2450 ifp->if_softc = sc; 2451 strlcpy(ifp->if_xname, name, IFNAMSIZ); 2452 ifp->if_dname = ovpngroupname; 2453 ifp->if_dunit = unit; 2454 2455 ifp->if_addrlen = 0; 2456 ifp->if_mtu = 1428; 2457 ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST; 2458 ifp->if_ioctl = ovpn_ioctl; 2459 ifp->if_transmit = ovpn_transmit; 2460 ifp->if_output = ovpn_output; 2461 ifp->if_qflush = ovpn_qflush; 2462 #ifdef VIMAGE 2463 ifp->if_reassign = ovpn_reassign; 2464 #endif 2465 ifp->if_capabilities |= IFCAP_LINKSTATE; 2466 ifp->if_capenable |= IFCAP_LINKSTATE; 2467 2468 if_attach(ifp); 2469 bpfattach(ifp, DLT_NULL, sizeof(uint32_t)); 2470 *ifpp = ifp; 2471 2472 return (0); 2473 } 2474 2475 static void 2476 ovpn_clone_destroy_cb(struct epoch_context *ctx) 2477 { 2478 struct ovpn_softc *sc; 2479 2480 sc = __containerof(ctx, struct ovpn_softc, epoch_ctx); 2481 2482 MPASS(sc->peercount == 0); 2483 MPASS(RB_EMPTY(&sc->peers)); 2484 2485 COUNTER_ARRAY_FREE(sc->counters, OVPN_COUNTER_SIZE); 2486 2487 if_free(sc->ifp); 2488 free(sc, M_OVPN); 2489 } 2490 2491 static int 2492 ovpn_clone_destroy(struct if_clone *ifc, struct ifnet *ifp, uint32_t flags) 2493 { 2494 struct ovpn_softc *sc; 2495 struct ovpn_kpeer *peer, *tmppeer; 2496 int unit; 2497 int ret __diagused; 2498 2499 sc = ifp->if_softc; 2500 unit = ifp->if_dunit; 2501 2502 OVPN_WLOCK(sc); 2503 2504 if (atomic_load_int(&sc->refcount) > 0) { 2505 OVPN_WUNLOCK(sc); 2506 return (EBUSY); 2507 } 2508 2509 RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) { 2510 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 2511 ret = _ovpn_del_peer(sc, peer); 2512 MPASS(ret == 0); 2513 } 2514 2515 ovpn_flush_rxring(sc); 2516 buf_ring_free(sc->notifring, M_OVPN); 2517 2518 OVPN_WUNLOCK(sc); 2519 2520 bpfdetach(ifp); 2521 if_detach(ifp); 2522 ifp->if_softc = NULL; 2523 2524 NET_EPOCH_CALL(ovpn_clone_destroy_cb, &sc->epoch_ctx); 2525 2526 if (unit != IF_DUNIT_NONE) 2527 ifc_free_unit(ifc, unit); 2528 2529 NET_EPOCH_DRAIN_CALLBACKS(); 2530 2531 return (0); 2532 } 2533 2534 static void 2535 vnet_ovpn_init(const void *unused __unused) 2536 { 2537 struct if_clone_addreq req = { 2538 .match_f = ovpn_clone_match, 2539 .create_f = ovpn_clone_create, 2540 .destroy_f = ovpn_clone_destroy, 2541 }; 2542 V_ovpn_cloner = ifc_attach_cloner(ovpngroupname, &req); 2543 } 2544 VNET_SYSINIT(vnet_ovpn_init, SI_SUB_PSEUDO, SI_ORDER_ANY, 2545 vnet_ovpn_init, NULL); 2546 2547 static void 2548 vnet_ovpn_uninit(const void *unused __unused) 2549 { 2550 if_clone_detach(V_ovpn_cloner); 2551 } 2552 VNET_SYSUNINIT(vnet_ovpn_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY, 2553 vnet_ovpn_uninit, NULL); 2554 2555 static int 2556 ovpnmodevent(module_t mod, int type, void *data) 2557 { 2558 switch (type) { 2559 case MOD_LOAD: 2560 /* Done in vnet_ovpn_init() */ 2561 break; 2562 case MOD_UNLOAD: 2563 /* Done in vnet_ovpn_uninit() */ 2564 break; 2565 default: 2566 return (EOPNOTSUPP); 2567 } 2568 2569 return (0); 2570 } 2571 2572 static moduledata_t ovpn_mod = { 2573 "if_ovpn", 2574 ovpnmodevent, 2575 0 2576 }; 2577 2578 DECLARE_MODULE(if_ovpn, ovpn_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 2579 MODULE_VERSION(if_ovpn, 1); 2580