1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2021-2022 Rubicon Communications, LLC (Netgate) 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 */ 28 #include "opt_inet.h" 29 #include "opt_inet6.h" 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/buf_ring.h> 34 #include <sys/epoch.h> 35 #include <sys/file.h> 36 #include <sys/filedesc.h> 37 #include <sys/kernel.h> 38 #include <sys/malloc.h> 39 #include <sys/mbuf.h> 40 #include <sys/module.h> 41 #include <sys/nv.h> 42 #include <sys/priv.h> 43 #include <sys/protosw.h> 44 #include <sys/rmlock.h> 45 #include <sys/sdt.h> 46 #include <sys/smp.h> 47 #include <sys/socket.h> 48 #include <sys/socketvar.h> 49 #include <sys/sockio.h> 50 #include <sys/sysctl.h> 51 #include <sys/time.h> 52 53 #include <machine/atomic.h> 54 55 #include <net/bpf.h> 56 #include <net/if.h> 57 #include <net/if_clone.h> 58 #include <net/if_types.h> 59 #include <net/if_var.h> 60 #include <net/netisr.h> 61 #include <net/route/nhop.h> 62 63 #include <netinet/in.h> 64 #include <netinet/in_fib.h> 65 #include <netinet/ip.h> 66 #include <netinet/ip6.h> 67 #include <netinet/ip_var.h> 68 #include <netinet/udp.h> 69 #include <netinet/udp_var.h> 70 71 #include <netinet6/ip6_var.h> 72 #include <netinet6/in6_fib.h> 73 74 #include <machine/in_cksum.h> 75 76 #include <opencrypto/cryptodev.h> 77 78 #include "if_ovpn.h" 79 80 struct ovpn_kkey_dir { 81 int refcount; 82 uint8_t key[32]; 83 uint8_t keylen; 84 uint8_t nonce[8]; 85 uint8_t noncelen; 86 enum ovpn_key_cipher cipher; 87 crypto_session_t cryptoid; 88 89 struct mtx replay_mtx; 90 /* 91 * Last seen gapless sequence number. New rx seq numbers must be 92 * strictly higher than this. 93 */ 94 uint32_t rx_seq; 95 /* Seen packets, relative to rx_seq. bit(0) will always be 0. */ 96 uint64_t rx_window; 97 }; 98 99 struct ovpn_kkey { 100 struct ovpn_kkey_dir *encrypt; 101 struct ovpn_kkey_dir *decrypt; 102 uint8_t keyid; 103 uint32_t peerid; 104 }; 105 106 struct ovpn_keepalive { 107 uint32_t interval; 108 uint32_t timeout; 109 }; 110 111 struct ovpn_wire_header { 112 uint32_t opcode; /* opcode, key id, peer id */ 113 uint32_t seq; 114 uint8_t auth_tag[16]; 115 }; 116 117 struct ovpn_notification { 118 enum ovpn_notif_type type; 119 uint32_t peerid; 120 }; 121 122 struct ovpn_softc; 123 124 struct ovpn_kpeer { 125 int refcount; 126 uint32_t peerid; 127 128 struct ovpn_softc *sc; 129 struct sockaddr_storage local; 130 struct sockaddr_storage remote; 131 132 struct in_addr vpn4; 133 struct in6_addr vpn6; 134 135 struct ovpn_kkey keys[2]; 136 uint32_t tx_seq; 137 138 struct ovpn_keepalive keepalive; 139 uint32_t *last_active; 140 struct callout ping_send; 141 struct callout ping_rcv; 142 }; 143 144 #define OVPN_MAX_PEERS 128 145 146 struct ovpn_counters { 147 uint64_t lost_ctrl_pkts_in; 148 uint64_t lost_ctrl_pkts_out; 149 uint64_t lost_data_pkts_in; 150 uint64_t lost_data_pkts_out; 151 uint64_t nomem_data_pkts_in; 152 uint64_t nomem_data_pkts_out; 153 uint64_t received_ctrl_pkts; 154 uint64_t received_data_pkts; 155 uint64_t sent_ctrl_pkts; 156 uint64_t sent_data_pkts; 157 158 uint64_t transport_bytes_sent; 159 uint64_t transport_bytes_received; 160 uint64_t tunnel_bytes_sent; 161 uint64_t tunnel_bytes_received; 162 }; 163 #define OVPN_COUNTER_SIZE (sizeof(struct ovpn_counters)/sizeof(uint64_t)) 164 165 struct ovpn_softc { 166 int refcount; 167 struct rmlock lock; 168 struct ifnet *ifp; 169 struct socket *so; 170 int peercount; 171 struct ovpn_kpeer *peers[OVPN_MAX_PEERS]; /* XXX Hard limit for now? */ 172 173 /* Pending packets */ 174 struct buf_ring *rxring; 175 struct buf_ring *notifring; 176 177 counter_u64_t counters[OVPN_COUNTER_SIZE]; 178 179 struct epoch_context epoch_ctx; 180 }; 181 182 static struct ovpn_kpeer *ovpn_find_peer(struct ovpn_softc *, uint32_t); 183 static bool ovpn_udp_input(struct mbuf *, int, struct inpcb *, 184 const struct sockaddr *, void *); 185 static int ovpn_transmit_to_peer(struct ifnet *, struct mbuf *, 186 struct ovpn_kpeer *, struct rm_priotracker *); 187 static int ovpn_encap(struct ovpn_softc *, uint32_t, struct mbuf *); 188 static int ovpn_get_af(struct mbuf *); 189 static void ovpn_free_kkey_dir(struct ovpn_kkey_dir *); 190 static bool ovpn_check_replay(struct ovpn_kkey_dir *, uint32_t); 191 192 #define OVPN_MTU_MIN 576 193 #define OVPN_MTU_MAX (IP_MAXPACKET - sizeof(struct ip) - \ 194 sizeof(struct udphdr) - sizeof(struct ovpn_wire_header)) 195 196 #define OVPN_OP_DATA_V2 0x09 197 #define OVPN_OP_SHIFT 3 198 199 VNET_DEFINE_STATIC(struct if_clone *, ovpn_cloner); 200 #define V_ovpn_cloner VNET(ovpn_cloner) 201 202 #define OVPN_RLOCK_TRACKER struct rm_priotracker _ovpn_lock_tracker; \ 203 struct rm_priotracker *_ovpn_lock_trackerp = &_ovpn_lock_tracker 204 #define OVPN_RLOCK(sc) rm_rlock(&(sc)->lock, _ovpn_lock_trackerp) 205 #define OVPN_RUNLOCK(sc) rm_runlock(&(sc)->lock, _ovpn_lock_trackerp) 206 #define OVPN_WLOCK(sc) rm_wlock(&(sc)->lock) 207 #define OVPN_WUNLOCK(sc) rm_wunlock(&(sc)->lock) 208 #define OVPN_ASSERT(sc) rm_assert(&(sc)->lock, RA_LOCKED) 209 #define OVPN_RASSERT(sc) rm_assert(&(sc)->lock, RA_RLOCKED) 210 #define OVPN_WASSERT(sc) rm_assert(&(sc)->lock, RA_WLOCKED) 211 #define OVPN_UNLOCK_ASSERT(sc) rm_assert(&(sc)->lock, RA_UNLOCKED) 212 213 #define OVPN_COUNTER_ADD(sc, name, val) \ 214 counter_u64_add(sc->counters[offsetof(struct ovpn_counters, name) / \ 215 sizeof(uint64_t)], val) 216 217 #define TO_IN(x) ((struct sockaddr_in *)(x)) 218 #define TO_IN6(x) ((struct sockaddr_in6 *)(x)) 219 220 SDT_PROVIDER_DEFINE(if_ovpn); 221 SDT_PROBE_DEFINE1(if_ovpn, tx, transmit, start, "struct mbuf *"); 222 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip4, "struct in_addr *", "struct ovpn_kpeer *"); 223 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip6, "struct in6_addr *", "struct ovpn_kpeer *"); 224 225 static const char ovpnname[] = "ovpn"; 226 static const char ovpngroupname[] = "openvpn"; 227 228 static MALLOC_DEFINE(M_OVPN, ovpnname, "OpenVPN DCO Interface"); 229 230 SYSCTL_DECL(_net_link); 231 static SYSCTL_NODE(_net_link, IFT_OTHER, openvpn, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 232 "OpenVPN DCO Interface"); 233 VNET_DEFINE_STATIC(int, replay_protection) = 0; 234 #define V_replay_protection VNET(replay_protection) 235 SYSCTL_INT(_net_link_openvpn, OID_AUTO, replay_protection, CTLFLAG_VNET | CTLFLAG_RW, 236 &VNET_NAME(replay_protection), 0, "Validate sequence numbers"); 237 238 static struct ovpn_kpeer * 239 ovpn_find_peer(struct ovpn_softc *sc, uint32_t peerid) 240 { 241 struct ovpn_kpeer *p = NULL; 242 243 OVPN_ASSERT(sc); 244 245 for (int i = 0; i < OVPN_MAX_PEERS; i++) { 246 p = sc->peers[i]; 247 if (p == NULL) 248 continue; 249 250 if (p->peerid == peerid) { 251 MPASS(p->sc == sc); 252 break; 253 } 254 } 255 256 return (p); 257 } 258 259 static struct ovpn_kpeer * 260 ovpn_find_only_peer(struct ovpn_softc *sc) 261 { 262 OVPN_ASSERT(sc); 263 264 for (int i = 0; i < OVPN_MAX_PEERS; i++) { 265 if (sc->peers[i] == NULL) 266 continue; 267 return (sc->peers[i]); 268 } 269 270 MPASS(false); 271 272 return (NULL); 273 } 274 275 static uint16_t 276 ovpn_get_port(struct sockaddr_storage *s) 277 { 278 switch (s->ss_family) { 279 case AF_INET: { 280 struct sockaddr_in *in = (struct sockaddr_in *)s; 281 return (in->sin_port); 282 } 283 case AF_INET6: { 284 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)s; 285 return (in6->sin6_port); 286 } 287 default: 288 panic("Unsupported address family %d", s->ss_family); 289 } 290 } 291 292 static int 293 ovpn_nvlist_to_sockaddr(const nvlist_t *nvl, struct sockaddr_storage *sa) 294 { 295 int af; 296 297 if (! nvlist_exists_number(nvl, "af")) 298 return (EINVAL); 299 if (! nvlist_exists_binary(nvl, "address")) 300 return (EINVAL); 301 if (! nvlist_exists_number(nvl, "port")) 302 return (EINVAL); 303 304 af = nvlist_get_number(nvl, "af"); 305 306 switch (af) { 307 #ifdef INET 308 case AF_INET: { 309 struct sockaddr_in *in = (struct sockaddr_in *)sa; 310 size_t len; 311 const void *addr = nvlist_get_binary(nvl, "address", &len); 312 in->sin_family = af; 313 if (len != sizeof(in->sin_addr)) 314 return (EINVAL); 315 316 memcpy(&in->sin_addr, addr, sizeof(in->sin_addr)); 317 in->sin_port = nvlist_get_number(nvl, "port"); 318 break; 319 } 320 #endif 321 #ifdef INET6 322 case AF_INET6: { 323 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)sa; 324 size_t len; 325 const void *addr = nvlist_get_binary(nvl, "address", &len); 326 in6->sin6_family = af; 327 if (len != sizeof(in6->sin6_addr)) 328 return (EINVAL); 329 330 memcpy(&in6->sin6_addr, addr, sizeof(in6->sin6_addr)); 331 in6->sin6_port = nvlist_get_number(nvl, "port"); 332 break; 333 } 334 #endif 335 default: 336 return (EINVAL); 337 } 338 339 return (0); 340 } 341 342 static bool 343 ovpn_has_peers(struct ovpn_softc *sc) 344 { 345 OVPN_ASSERT(sc); 346 347 return (sc->peercount > 0); 348 } 349 350 static void 351 ovpn_rele_so(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 352 { 353 bool has_peers; 354 355 OVPN_WASSERT(sc); 356 357 if (sc->so == NULL) 358 return; 359 360 has_peers = ovpn_has_peers(sc); 361 362 /* Only remove the tunnel function if we're releasing the socket for 363 * the last peer. */ 364 if (! has_peers) 365 (void)udp_set_kernel_tunneling(sc->so, NULL, NULL, NULL); 366 367 sorele(sc->so); 368 369 if (! has_peers) 370 sc->so = NULL; 371 } 372 373 static void 374 ovpn_notify_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 375 { 376 struct ovpn_notification *n; 377 378 OVPN_WASSERT(sc); 379 380 n = malloc(sizeof(*n), M_OVPN, M_NOWAIT); 381 if (n == NULL) 382 return; 383 384 n->peerid = peer->peerid; 385 n->type = OVPN_NOTIF_DEL_PEER; 386 if (buf_ring_enqueue(sc->notifring, n) != 0) { 387 free(n, M_OVPN); 388 } else if (sc->so != NULL) { 389 /* Wake up userspace */ 390 sc->so->so_error = EAGAIN; 391 sorwakeup(sc->so); 392 sowwakeup(sc->so); 393 } 394 } 395 396 static void 397 ovpn_peer_release_ref(struct ovpn_kpeer *peer, bool locked) 398 { 399 struct ovpn_softc *sc; 400 401 atomic_add_int(&peer->refcount, -1); 402 403 if (atomic_load_int(&peer->refcount) > 0) 404 return; 405 406 sc = peer->sc; 407 408 if (! locked) { 409 OVPN_WLOCK(sc); 410 411 /* Might have changed before we acquired the lock. */ 412 if (atomic_load_int(&peer->refcount) > 0) { 413 OVPN_WUNLOCK(sc); 414 return; 415 } 416 } 417 418 /* The peer should have been removed from the list already. */ 419 MPASS(ovpn_find_peer(sc, peer->peerid) == NULL); 420 421 ovpn_notify_del_peer(sc, peer); 422 423 for (int i = 0; i < 2; i++) { 424 ovpn_free_kkey_dir(peer->keys[i].encrypt); 425 ovpn_free_kkey_dir(peer->keys[i].decrypt); 426 } 427 428 ovpn_rele_so(sc, peer); 429 430 callout_stop(&peer->ping_send); 431 callout_stop(&peer->ping_rcv); 432 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 433 free(peer, M_OVPN); 434 435 if (! locked) 436 OVPN_WUNLOCK(sc); 437 } 438 439 static int 440 ovpn_new_peer(struct ifnet *ifp, const nvlist_t *nvl) 441 { 442 #ifdef INET6 443 struct epoch_tracker et; 444 #endif 445 struct sockaddr_storage remote; 446 struct ovpn_kpeer *peer = NULL; 447 struct file *fp = NULL; 448 struct sockaddr *name = NULL; 449 struct ovpn_softc *sc = ifp->if_softc; 450 struct thread *td = curthread; 451 struct socket *so = NULL; 452 int fd; 453 uint32_t peerid; 454 int ret = 0, i; 455 456 if (nvl == NULL) 457 return (EINVAL); 458 459 if (! nvlist_exists_number(nvl, "peerid")) 460 return (EINVAL); 461 462 if (! nvlist_exists_number(nvl, "fd")) 463 return (EINVAL); 464 465 if (! nvlist_exists_nvlist(nvl, "remote")) 466 return (EINVAL); 467 468 peerid = nvlist_get_number(nvl, "peerid"); 469 470 ret = ovpn_nvlist_to_sockaddr(nvlist_get_nvlist(nvl, "remote"), 471 &remote); 472 if (ret != 0) 473 return (ret); 474 475 fd = nvlist_get_number(nvl, "fd"); 476 477 /* Look up the userspace process and use the fd to find the socket. */ 478 ret = getsock(td, fd, &cap_connect_rights, &fp); 479 if (ret != 0) 480 return (ret); 481 482 so = fp->f_data; 483 484 peer = malloc(sizeof(*peer), M_OVPN, M_WAITOK | M_ZERO); 485 peer->peerid = peerid; 486 peer->sc = sc; 487 peer->tx_seq = 1; 488 peer->refcount = 1; 489 peer->last_active = uma_zalloc_pcpu(pcpu_zone_4, M_WAITOK | M_ZERO); 490 491 if (nvlist_exists_binary(nvl, "vpn_ipv4")) { 492 size_t len; 493 const void *addr = nvlist_get_binary(nvl, "vpn_ipv4", &len); 494 if (len != sizeof(peer->vpn4)) { 495 ret = EINVAL; 496 goto error; 497 } 498 memcpy(&peer->vpn4, addr, len); 499 } 500 501 if (nvlist_exists_binary(nvl, "vpn_ipv6")) { 502 size_t len; 503 const void *addr = nvlist_get_binary(nvl, "vpn_ipv6", &len); 504 if (len != sizeof(peer->vpn6)) { 505 ret = EINVAL; 506 goto error; 507 } 508 memcpy(&peer->vpn6, addr, len); 509 } 510 511 callout_init_rm(&peer->ping_send, &sc->lock, CALLOUT_SHAREDLOCK); 512 callout_init_rm(&peer->ping_rcv, &sc->lock, 0); 513 514 ret = so->so_proto->pr_sockaddr(so, &name); 515 if (ret) 516 goto error; 517 518 if (ovpn_get_port((struct sockaddr_storage *)name) == 0) { 519 ret = EINVAL; 520 goto error; 521 } 522 if (name->sa_family != remote.ss_family) { 523 ret = EINVAL; 524 goto error; 525 } 526 527 memcpy(&peer->local, name, name->sa_len); 528 memcpy(&peer->remote, &remote, sizeof(remote)); 529 free(name, M_SONAME); 530 name = NULL; 531 532 if (peer->local.ss_family == AF_INET6 && 533 IN6_IS_ADDR_V4MAPPED(&TO_IN6(&peer->remote)->sin6_addr)) { 534 /* V4 mapped address, so treat this as v4, not v6. */ 535 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->local); 536 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->remote); 537 } 538 539 #ifdef INET6 540 if (peer->local.ss_family == AF_INET6 && 541 IN6_IS_ADDR_UNSPECIFIED(&TO_IN6(&peer->local)->sin6_addr)) { 542 NET_EPOCH_ENTER(et); 543 ret = in6_selectsrc_addr(curthread->td_proc->p_fibnum, 544 &TO_IN6(&peer->remote)->sin6_addr, 545 0, NULL, &TO_IN6(&peer->local)->sin6_addr, NULL); 546 NET_EPOCH_EXIT(et); 547 if (ret != 0) { 548 goto error; 549 } 550 } 551 #endif 552 OVPN_WLOCK(sc); 553 554 /* Disallow peer id re-use. */ 555 if (ovpn_find_peer(sc, peerid) != NULL) { 556 ret = EEXIST; 557 goto error_locked; 558 } 559 560 /* Make sure this is really a UDP socket. */ 561 if (so->so_type != SOCK_DGRAM || so->so_proto->pr_type != SOCK_DGRAM) { 562 ret = EPROTOTYPE; 563 goto error_locked; 564 } 565 566 /* Must be the same socket as for other peers on this interface. */ 567 if (sc->so != NULL && so != sc->so) 568 goto error_locked; 569 570 if (sc->so == NULL) 571 sc->so = so; 572 573 /* Insert the peer into the list. */ 574 for (i = 0; i < OVPN_MAX_PEERS; i++) { 575 if (sc->peers[i] != NULL) 576 continue; 577 578 MPASS(sc->peers[i] == NULL); 579 sc->peers[i] = peer; 580 sc->peercount++; 581 soref(sc->so); 582 break; 583 } 584 if (i == OVPN_MAX_PEERS) { 585 ret = ENOSPC; 586 goto error_locked; 587 } 588 589 ret = udp_set_kernel_tunneling(sc->so, ovpn_udp_input, NULL, sc); 590 if (ret == EBUSY) { 591 /* Fine, another peer already set the input function. */ 592 ret = 0; 593 } 594 if (ret != 0) { 595 sc->peers[i] = NULL; 596 sc->peercount--; 597 goto error_locked; 598 } 599 600 OVPN_WUNLOCK(sc); 601 602 goto done; 603 604 error_locked: 605 OVPN_WUNLOCK(sc); 606 error: 607 free(name, M_SONAME); 608 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 609 free(peer, M_OVPN); 610 done: 611 if (fp != NULL) 612 fdrop(fp, td); 613 614 return (ret); 615 } 616 617 static int 618 _ovpn_del_peer(struct ovpn_softc *sc, uint32_t peerid) 619 { 620 struct ovpn_kpeer *peer; 621 int i; 622 623 OVPN_WASSERT(sc); 624 625 for (i = 0; i < OVPN_MAX_PEERS; i++) { 626 if (sc->peers[i] == NULL) 627 continue; 628 if (sc->peers[i]->peerid != peerid) 629 continue; 630 631 peer = sc->peers[i]; 632 break; 633 } 634 if (i == OVPN_MAX_PEERS) 635 return (ENOENT); 636 637 sc->peers[i] = NULL; 638 sc->peercount--; 639 640 ovpn_peer_release_ref(peer, true); 641 642 return (0); 643 } 644 645 static int 646 ovpn_del_peer(struct ifnet *ifp, nvlist_t *nvl) 647 { 648 struct ovpn_softc *sc = ifp->if_softc; 649 uint32_t peerid; 650 int ret; 651 652 OVPN_WASSERT(sc); 653 654 if (nvl == NULL) 655 return (EINVAL); 656 657 if (! nvlist_exists_number(nvl, "peerid")) 658 return (EINVAL); 659 660 peerid = nvlist_get_number(nvl, "peerid"); 661 662 ret = _ovpn_del_peer(sc, peerid); 663 664 return (ret); 665 } 666 667 static int 668 ovpn_create_kkey_dir(struct ovpn_kkey_dir **kdirp, 669 const nvlist_t *nvl) 670 { 671 struct crypto_session_params csp; 672 struct ovpn_kkey_dir *kdir; 673 const char *ciphername; 674 enum ovpn_key_cipher cipher; 675 const void *key, *iv; 676 size_t keylen = 0, ivlen = 0; 677 int error; 678 679 if (! nvlist_exists_string(nvl, "cipher")) 680 return (EINVAL); 681 ciphername = nvlist_get_string(nvl, "cipher"); 682 683 if (strcmp(ciphername, "none") == 0) 684 cipher = OVPN_CIPHER_ALG_NONE; 685 else if (strcmp(ciphername, "AES-256-GCM") == 0) 686 cipher = OVPN_CIPHER_ALG_AES_GCM; 687 else if (strcmp(ciphername, "CHACHA20-POLY1305") == 0) 688 cipher = OVPN_CIPHER_ALG_CHACHA20_POLY1305; 689 else 690 return (EINVAL); 691 692 if (cipher != OVPN_CIPHER_ALG_NONE) { 693 if (! nvlist_exists_binary(nvl, "key")) 694 return (EINVAL); 695 key = nvlist_get_binary(nvl, "key", &keylen); 696 if (keylen > sizeof(kdir->key)) 697 return (E2BIG); 698 699 if (! nvlist_exists_binary(nvl, "iv")) 700 return (EINVAL); 701 iv = nvlist_get_binary(nvl, "iv", &ivlen); 702 if (ivlen != 8) 703 return (E2BIG); 704 } 705 706 kdir = malloc(sizeof(struct ovpn_kkey_dir), M_OVPN, 707 M_WAITOK | M_ZERO); 708 709 kdir->cipher = cipher; 710 kdir->keylen = keylen; 711 memcpy(kdir->key, key, keylen); 712 kdir->noncelen = ivlen; 713 memcpy(kdir->nonce, iv, ivlen); 714 715 if (kdir->cipher != OVPN_CIPHER_ALG_NONE) { 716 /* Crypto init */ 717 bzero(&csp, sizeof(csp)); 718 csp.csp_mode = CSP_MODE_AEAD; 719 720 if (kdir->cipher == OVPN_CIPHER_ALG_CHACHA20_POLY1305) 721 csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305; 722 else 723 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 724 725 csp.csp_flags |= CSP_F_SEPARATE_AAD; 726 727 csp.csp_cipher_klen = kdir->keylen; 728 csp.csp_cipher_key = kdir->key; 729 csp.csp_ivlen = 96 / 8; 730 731 error = crypto_newsession(&kdir->cryptoid, &csp, 732 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE); 733 if (error) { 734 free(kdir, M_OVPN); 735 return (error); 736 } 737 } 738 739 mtx_init(&kdir->replay_mtx, "if_ovpn rx replay", NULL, MTX_DEF); 740 *kdirp = kdir; 741 742 return (0); 743 } 744 745 static void 746 ovpn_free_kkey_dir(struct ovpn_kkey_dir *kdir) 747 { 748 if (kdir == NULL) 749 return; 750 751 mtx_destroy(&kdir->replay_mtx); 752 753 crypto_freesession(kdir->cryptoid); 754 free(kdir, M_OVPN); 755 } 756 757 static int 758 ovpn_set_key(struct ifnet *ifp, const nvlist_t *nvl) 759 { 760 struct ovpn_softc *sc = ifp->if_softc; 761 struct ovpn_kkey_dir *enc, *dec; 762 struct ovpn_kpeer *peer; 763 int slot, keyid, peerid; 764 int error; 765 766 if (nvl == NULL) 767 return (EINVAL); 768 769 if (! nvlist_exists_number(nvl, "slot")) 770 return (EINVAL); 771 slot = nvlist_get_number(nvl, "slot"); 772 773 if (! nvlist_exists_number(nvl, "keyid")) 774 return (EINVAL); 775 keyid = nvlist_get_number(nvl, "keyid"); 776 777 if (! nvlist_exists_number(nvl, "peerid")) 778 return (EINVAL); 779 peerid = nvlist_get_number(nvl, "peerid"); 780 781 if (slot != OVPN_KEY_SLOT_PRIMARY && 782 slot != OVPN_KEY_SLOT_SECONDARY) 783 return (EINVAL); 784 785 if (! nvlist_exists_nvlist(nvl, "encrypt") || 786 ! nvlist_exists_nvlist(nvl, "decrypt")) 787 return (EINVAL); 788 789 error = ovpn_create_kkey_dir(&enc, nvlist_get_nvlist(nvl, "encrypt")); 790 if (error) 791 return (error); 792 793 error = ovpn_create_kkey_dir(&dec, nvlist_get_nvlist(nvl, "decrypt")); 794 if (error) { 795 ovpn_free_kkey_dir(enc); 796 return (error); 797 } 798 799 OVPN_WLOCK(sc); 800 801 peer = ovpn_find_peer(sc, peerid); 802 if (peer == NULL) { 803 ovpn_free_kkey_dir(dec); 804 ovpn_free_kkey_dir(enc); 805 OVPN_WUNLOCK(sc); 806 return (ENOENT); 807 } 808 809 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 810 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 811 812 peer->keys[slot].encrypt = enc; 813 peer->keys[slot].decrypt = dec; 814 815 peer->keys[slot].keyid = keyid; 816 peer->keys[slot].peerid = peerid; 817 818 OVPN_WUNLOCK(sc); 819 820 return (0); 821 } 822 823 static int 824 ovpn_check_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, enum ovpn_key_slot slot) 825 { 826 OVPN_ASSERT(sc); 827 828 if (peer->keys[slot].encrypt == NULL) 829 return (ENOLINK); 830 831 if (peer->keys[slot].decrypt == NULL) 832 return (ENOLINK); 833 834 return (0); 835 } 836 837 static int 838 ovpn_start(struct ifnet *ifp) 839 { 840 struct ovpn_softc *sc = ifp->if_softc; 841 842 OVPN_WLOCK(sc); 843 844 ifp->if_flags |= IFF_UP; 845 ifp->if_drv_flags |= IFF_DRV_RUNNING; 846 if_link_state_change(ifp, LINK_STATE_UP); 847 848 OVPN_WUNLOCK(sc); 849 850 return (0); 851 } 852 853 static int 854 ovpn_swap_keys(struct ifnet *ifp, nvlist_t *nvl) 855 { 856 struct ovpn_softc *sc = ifp->if_softc; 857 struct ovpn_kpeer *peer; 858 struct ovpn_kkey tmpkey; 859 int error; 860 861 if (nvl == NULL) 862 return (EINVAL); 863 864 if (! nvlist_exists_number(nvl, "peerid")) 865 return (EINVAL); 866 867 OVPN_WLOCK(sc); 868 869 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 870 if (peer == NULL) { 871 OVPN_WUNLOCK(sc); 872 return (ENOENT); 873 } 874 875 /* Check that we have a second key to swap to. */ 876 error = ovpn_check_key(sc, peer, OVPN_KEY_SLOT_SECONDARY); 877 if (error) { 878 OVPN_WUNLOCK(sc); 879 return (error); 880 } 881 882 tmpkey = peer->keys[0]; 883 peer->keys[0] = peer->keys[1]; 884 peer->keys[1] = tmpkey; 885 886 OVPN_WUNLOCK(sc); 887 888 return (0); 889 } 890 891 static int 892 ovpn_del_key(struct ifnet *ifp, const nvlist_t *nvl) 893 { 894 enum ovpn_key_slot slot; 895 struct ovpn_kpeer *peer; 896 struct ovpn_softc *sc = ifp->if_softc; 897 898 if (nvl == NULL) 899 return (EINVAL); 900 901 if (! nvlist_exists_number(nvl, "peerid")) 902 return (EINVAL); 903 904 if (! nvlist_exists_number(nvl, "slot")) 905 return (EINVAL); 906 slot = nvlist_get_number(nvl, "slot"); 907 908 if (slot != OVPN_KEY_SLOT_PRIMARY && 909 slot != OVPN_KEY_SLOT_SECONDARY) 910 return (EINVAL); 911 912 OVPN_WLOCK(sc); 913 914 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 915 if (peer == NULL) { 916 OVPN_WUNLOCK(sc); 917 return (ENOENT); 918 } 919 920 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 921 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 922 923 peer->keys[slot].encrypt = NULL; 924 peer->keys[slot].decrypt = NULL; 925 926 peer->keys[slot].keyid = 0; 927 peer->keys[slot].peerid = 0; 928 929 OVPN_WUNLOCK(sc); 930 931 return (0); 932 } 933 934 static int 935 ovpn_send_pkt(struct ifnet *ifp, const nvlist_t *nvl) 936 { 937 struct epoch_tracker et; 938 struct ovpn_softc *sc = ifp->if_softc; 939 struct mbuf *m; 940 const uint8_t *pkt; 941 size_t pktlen; 942 uint32_t peerid; 943 int ret; 944 945 if (nvl == NULL) 946 return (EINVAL); 947 948 if (! nvlist_exists_binary(nvl, "packet")) 949 return (EINVAL); 950 pkt = nvlist_get_binary(nvl, "packet", &pktlen); 951 952 if (! nvlist_exists_number(nvl, "peerid")) 953 return (EINVAL); 954 955 peerid = nvlist_get_number(nvl, "peerid"); 956 957 /* 958 * Check that userspace isn't giving us a data packet. That might lead 959 * to IV re-use, which would be bad. 960 */ 961 if ((pkt[0] >> OVPN_OP_SHIFT) == OVPN_OP_DATA_V2) 962 return (EINVAL); 963 964 m = m_get2(pktlen, M_WAITOK, MT_DATA, M_PKTHDR); 965 if (m == NULL) 966 return (ENOMEM); 967 968 m->m_len = m->m_pkthdr.len = pktlen; 969 m_copyback(m, 0, m->m_len, pkt); 970 971 /* Now prepend IP/UDP headers and transmit the mbuf. */ 972 NET_EPOCH_ENTER(et); 973 ret = ovpn_encap(sc, peerid, m); 974 NET_EPOCH_EXIT(et); 975 if (ret == 0) 976 OVPN_COUNTER_ADD(sc, sent_ctrl_pkts, 1); 977 else 978 OVPN_COUNTER_ADD(sc, lost_ctrl_pkts_out, 1); 979 980 return (ret); 981 } 982 983 static void 984 ovpn_send_ping(void *arg) 985 { 986 static const uint8_t ping_str[] = { 987 0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb, 988 0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48 989 }; 990 991 struct epoch_tracker et; 992 struct ovpn_kpeer *peer = arg; 993 struct ovpn_softc *sc = peer->sc; 994 struct mbuf *m; 995 996 OVPN_RASSERT(sc); 997 998 /* Ensure we repeat! */ 999 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 1000 ovpn_send_ping, peer); 1001 1002 m = m_get2(sizeof(ping_str), M_NOWAIT, MT_DATA, M_PKTHDR); 1003 if (m == NULL) 1004 return; 1005 1006 m_copyback(m, 0, sizeof(ping_str), ping_str); 1007 m->m_len = m->m_pkthdr.len = sizeof(ping_str); 1008 1009 CURVNET_SET(sc->ifp->if_vnet); 1010 NET_EPOCH_ENTER(et); 1011 (void)ovpn_transmit_to_peer(sc->ifp, m, peer, NULL); 1012 NET_EPOCH_EXIT(et); 1013 CURVNET_RESTORE(); 1014 } 1015 1016 static void 1017 ovpn_timeout(void *arg) 1018 { 1019 struct ovpn_kpeer *peer = arg; 1020 struct ovpn_softc *sc = peer->sc; 1021 uint32_t last, _last_active; 1022 int ret __diagused; 1023 int cpu; 1024 1025 OVPN_WASSERT(sc); 1026 1027 last = 0; 1028 CPU_FOREACH(cpu) { 1029 _last_active = *zpcpu_get_cpu(peer->last_active, cpu); 1030 if (_last_active > last) 1031 last = _last_active; 1032 } 1033 1034 if (last + peer->keepalive.timeout > time_uptime) { 1035 callout_reset(&peer->ping_rcv, 1036 (peer->keepalive.timeout - (time_uptime - last)) * hz, 1037 ovpn_timeout, peer); 1038 return; 1039 } 1040 1041 CURVNET_SET(sc->ifp->if_vnet); 1042 ret = _ovpn_del_peer(sc, peer->peerid); 1043 MPASS(ret == 0); 1044 CURVNET_RESTORE(); 1045 } 1046 1047 static int 1048 ovpn_set_peer(struct ifnet *ifp, const nvlist_t *nvl) 1049 { 1050 struct ovpn_softc *sc = ifp->if_softc; 1051 struct ovpn_kpeer *peer; 1052 1053 if (nvl == NULL) 1054 return (EINVAL); 1055 1056 if (! nvlist_exists_number(nvl, "interval") || 1057 ! nvlist_exists_number(nvl, "timeout") || 1058 ! nvlist_exists_number(nvl, "peerid")) 1059 return (EINVAL); 1060 1061 OVPN_WLOCK(sc); 1062 1063 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 1064 if (peer == NULL) { 1065 OVPN_WUNLOCK(sc); 1066 return (ENOENT); 1067 } 1068 1069 peer->keepalive.interval = nvlist_get_number(nvl, "interval"); 1070 peer->keepalive.timeout = nvlist_get_number(nvl, "timeout"); 1071 1072 if (peer->keepalive.interval > 0) 1073 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 1074 ovpn_send_ping, peer); 1075 if (peer->keepalive.timeout > 0) 1076 callout_reset(&peer->ping_rcv, peer->keepalive.timeout * hz, 1077 ovpn_timeout, peer); 1078 1079 OVPN_WUNLOCK(sc); 1080 1081 return (0); 1082 } 1083 1084 static int 1085 ovpn_set_ifmode(struct ifnet *ifp, const nvlist_t *nvl) 1086 { 1087 struct ovpn_softc *sc = ifp->if_softc; 1088 int ifmode; 1089 1090 if (nvl == NULL) 1091 return (EINVAL); 1092 1093 if (! nvlist_exists_number(nvl, "ifmode") ) 1094 return (EINVAL); 1095 1096 ifmode = nvlist_get_number(nvl, "ifmode"); 1097 1098 OVPN_WLOCK(sc); 1099 1100 /* deny this if UP */ 1101 if (ifp->if_flags & IFF_UP) { 1102 OVPN_WUNLOCK(sc); 1103 return (EBUSY); 1104 } 1105 1106 switch (ifmode & ~IFF_MULTICAST) { 1107 case IFF_POINTOPOINT: 1108 case IFF_BROADCAST: 1109 ifp->if_flags &= 1110 ~(IFF_BROADCAST|IFF_POINTOPOINT|IFF_MULTICAST); 1111 ifp->if_flags |= ifmode; 1112 break; 1113 default: 1114 OVPN_WUNLOCK(sc); 1115 return (EINVAL); 1116 } 1117 1118 OVPN_WUNLOCK(sc); 1119 1120 return (0); 1121 } 1122 1123 static int 1124 ovpn_ioctl_set(struct ifnet *ifp, struct ifdrv *ifd) 1125 { 1126 struct ovpn_softc *sc = ifp->if_softc; 1127 uint8_t *buf = NULL; 1128 nvlist_t *nvl = NULL; 1129 int ret; 1130 1131 if (ifd->ifd_len != 0) { 1132 if (ifd->ifd_len > OVPN_MAX_REQUEST_SIZE) 1133 return (E2BIG); 1134 1135 buf = malloc(ifd->ifd_len, M_OVPN, M_WAITOK); 1136 1137 ret = copyin(ifd->ifd_data, buf, ifd->ifd_len); 1138 if (ret != 0) { 1139 free(buf, M_OVPN); 1140 return (ret); 1141 } 1142 1143 nvl = nvlist_unpack(buf, ifd->ifd_len, 0); 1144 free(buf, M_OVPN); 1145 if (nvl == NULL) { 1146 return (EINVAL); 1147 } 1148 } 1149 1150 switch (ifd->ifd_cmd) { 1151 case OVPN_NEW_PEER: 1152 ret = ovpn_new_peer(ifp, nvl); 1153 break; 1154 case OVPN_DEL_PEER: 1155 OVPN_WLOCK(sc); 1156 ret = ovpn_del_peer(ifp, nvl); 1157 OVPN_WUNLOCK(sc); 1158 break; 1159 case OVPN_NEW_KEY: 1160 ret = ovpn_set_key(ifp, nvl); 1161 break; 1162 case OVPN_START_VPN: 1163 ret = ovpn_start(ifp); 1164 break; 1165 case OVPN_SWAP_KEYS: 1166 ret = ovpn_swap_keys(ifp, nvl); 1167 break; 1168 case OVPN_DEL_KEY: 1169 ret = ovpn_del_key(ifp, nvl); 1170 break; 1171 case OVPN_SEND_PKT: 1172 ret = ovpn_send_pkt(ifp, nvl); 1173 break; 1174 case OVPN_SET_PEER: 1175 ret = ovpn_set_peer(ifp, nvl); 1176 break; 1177 case OVPN_SET_IFMODE: 1178 ret = ovpn_set_ifmode(ifp, nvl); 1179 break; 1180 default: 1181 ret = ENOTSUP; 1182 } 1183 1184 nvlist_destroy(nvl); 1185 return (ret); 1186 } 1187 1188 static int 1189 ovpn_add_counters(nvlist_t *parent, const char *name, counter_u64_t in, 1190 counter_u64_t out) 1191 { 1192 nvlist_t *nvl; 1193 1194 nvl = nvlist_create(0); 1195 if (nvl == NULL) 1196 return (ENOMEM); 1197 1198 nvlist_add_number(nvl, "in", counter_u64_fetch(in)); 1199 nvlist_add_number(nvl, "out", counter_u64_fetch(out)); 1200 1201 nvlist_add_nvlist(parent, name, nvl); 1202 1203 nvlist_destroy(nvl); 1204 1205 return (0); 1206 } 1207 1208 static int 1209 ovpn_get_stats(struct ovpn_softc *sc, nvlist_t **onvl) 1210 { 1211 nvlist_t *nvl; 1212 int ret; 1213 1214 nvl = nvlist_create(0); 1215 if (nvl == NULL) 1216 return (ENOMEM); 1217 1218 #define OVPN_COUNTER_OUT(name, in, out) \ 1219 do { \ 1220 ret = ovpn_add_counters(nvl, name, \ 1221 sc->counters[offsetof(struct ovpn_counters, in) / \ 1222 sizeof(uint64_t)], \ 1223 sc->counters[offsetof(struct ovpn_counters, out) / \ 1224 sizeof(uint64_t)]); \ 1225 if (ret != 0) \ 1226 goto error; \ 1227 } while(0) 1228 1229 OVPN_COUNTER_OUT("lost_ctrl", lost_ctrl_pkts_in, lost_ctrl_pkts_out); 1230 OVPN_COUNTER_OUT("lost_data", lost_data_pkts_in, lost_data_pkts_out); 1231 OVPN_COUNTER_OUT("nomem_data", nomem_data_pkts_in, 1232 nomem_data_pkts_out); 1233 OVPN_COUNTER_OUT("data", received_data_pkts, sent_data_pkts); 1234 OVPN_COUNTER_OUT("ctrl", received_ctrl_pkts, sent_ctrl_pkts); 1235 OVPN_COUNTER_OUT("tunnel", tunnel_bytes_received, 1236 tunnel_bytes_received); 1237 OVPN_COUNTER_OUT("transport", transport_bytes_received, 1238 transport_bytes_received); 1239 #undef OVPN_COUNTER_OUT 1240 1241 *onvl = nvl; 1242 1243 return (0); 1244 1245 error: 1246 nvlist_destroy(nvl); 1247 return (ret); 1248 } 1249 1250 static int 1251 ovpn_poll_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1252 { 1253 nvlist_t *nvl; 1254 1255 nvl = nvlist_create(0); 1256 if (nvl == NULL) 1257 return (ENOMEM); 1258 1259 nvlist_add_number(nvl, "pending", 1260 buf_ring_count(sc->rxring) + buf_ring_count(sc->notifring)); 1261 1262 *onvl = nvl; 1263 1264 return (0); 1265 } 1266 1267 static int 1268 opvn_get_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1269 { 1270 struct ovpn_notification *n; 1271 struct ovpn_wire_header *ohdr; 1272 struct mbuf *m; 1273 uint8_t *buf; 1274 nvlist_t *nvl; 1275 uint32_t peerid; 1276 u_int mlength; 1277 1278 /* Check if we have notifications pending. */ 1279 n = buf_ring_dequeue_mc(sc->notifring); 1280 if (n != NULL) { 1281 nvl = nvlist_create(0); 1282 if (nvl == NULL) { 1283 free(n, M_OVPN); 1284 return (ENOMEM); 1285 } 1286 nvlist_add_number(nvl, "peerid", n->peerid); 1287 nvlist_add_number(nvl, "notification", n->type); 1288 free(n, M_OVPN); 1289 1290 *onvl = nvl; 1291 1292 return (0); 1293 } 1294 1295 /* Queued packet. */ 1296 m = buf_ring_dequeue_mc(sc->rxring); 1297 if (m == NULL) 1298 return (ENOENT); 1299 1300 mlength = m_length(m, NULL); 1301 buf = malloc(mlength, M_NVLIST, M_WAITOK); 1302 m_copydata(m, 0, mlength, buf); 1303 ohdr = (struct ovpn_wire_header *)buf; 1304 peerid = ntohl(ohdr->opcode) & 0x00ffffff; 1305 1306 nvl = nvlist_create(0); 1307 if (nvl == NULL) { 1308 OVPN_COUNTER_ADD(sc, lost_ctrl_pkts_in, 1); 1309 m_freem(m); 1310 free(buf, M_NVLIST); 1311 return (ENOMEM); 1312 } 1313 1314 nvlist_move_binary(nvl, "packet", buf, mlength); 1315 buf = NULL; 1316 nvlist_add_number(nvl, "peerid", peerid); 1317 1318 *onvl = nvl; 1319 1320 m_freem(m); 1321 1322 return (0); 1323 } 1324 1325 static int 1326 ovpn_ioctl_get(struct ifnet *ifp, struct ifdrv *ifd) 1327 { 1328 struct ovpn_softc *sc = ifp->if_softc; 1329 nvlist_t *nvl = NULL; 1330 int error; 1331 1332 switch (ifd->ifd_cmd) { 1333 case OVPN_GET_STATS: 1334 error = ovpn_get_stats(sc, &nvl); 1335 break; 1336 case OVPN_POLL_PKT: 1337 error = ovpn_poll_pkt(sc, &nvl); 1338 break; 1339 case OVPN_GET_PKT: 1340 error = opvn_get_pkt(sc, &nvl); 1341 break; 1342 default: 1343 error = ENOTSUP; 1344 break; 1345 } 1346 1347 if (error == 0) { 1348 void *packed = NULL; 1349 size_t len; 1350 1351 MPASS(nvl != NULL); 1352 1353 packed = nvlist_pack(nvl, &len); 1354 if (! packed) { 1355 nvlist_destroy(nvl); 1356 return (ENOMEM); 1357 } 1358 1359 if (len > ifd->ifd_len) { 1360 free(packed, M_NVLIST); 1361 nvlist_destroy(nvl); 1362 return (ENOSPC); 1363 } 1364 1365 error = copyout(packed, ifd->ifd_data, len); 1366 ifd->ifd_len = len; 1367 1368 free(packed, M_NVLIST); 1369 nvlist_destroy(nvl); 1370 } 1371 1372 return (error); 1373 } 1374 1375 static int 1376 ovpn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 1377 { 1378 struct ifdrv *ifd; 1379 int error; 1380 1381 switch (cmd) { 1382 case SIOCSDRVSPEC: 1383 case SIOCGDRVSPEC: 1384 error = priv_check(curthread, PRIV_NET_OVPN); 1385 if (error) 1386 return (error); 1387 break; 1388 } 1389 1390 switch (cmd) { 1391 case SIOCSDRVSPEC: 1392 ifd = (struct ifdrv *)data; 1393 error = ovpn_ioctl_set(ifp, ifd); 1394 break; 1395 case SIOCGDRVSPEC: 1396 ifd = (struct ifdrv *)data; 1397 error = ovpn_ioctl_get(ifp, ifd); 1398 break; 1399 case SIOCSIFMTU: { 1400 struct ifreq *ifr = (struct ifreq *)data; 1401 if (ifr->ifr_mtu < OVPN_MTU_MIN || ifr->ifr_mtu > OVPN_MTU_MAX) 1402 return (EINVAL); 1403 1404 ifp->if_mtu = ifr->ifr_mtu; 1405 return (0); 1406 } 1407 case SIOCSIFADDR: 1408 case SIOCADDMULTI: 1409 case SIOCDELMULTI: 1410 case SIOCGIFMTU: 1411 case SIOCSIFFLAGS: 1412 return (0); 1413 default: 1414 error = EINVAL; 1415 } 1416 1417 return (error); 1418 } 1419 1420 static int 1421 ovpn_encrypt_tx_cb(struct cryptop *crp) 1422 { 1423 struct epoch_tracker et; 1424 struct ovpn_kpeer *peer = crp->crp_opaque; 1425 struct ovpn_softc *sc = peer->sc; 1426 struct mbuf *m = crp->crp_buf.cb_mbuf; 1427 int tunnel_len; 1428 int ret; 1429 1430 if (crp->crp_etype != 0) { 1431 crypto_freereq(crp); 1432 ovpn_peer_release_ref(peer, false); 1433 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1434 m_freem(m); 1435 return (0); 1436 } 1437 1438 NET_EPOCH_ENTER(et); 1439 CURVNET_SET(sc->ifp->if_vnet); 1440 1441 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1442 1443 tunnel_len = m->m_pkthdr.len - sizeof(struct ovpn_wire_header); 1444 ret = ovpn_encap(sc, peer->peerid, m); 1445 if (ret == 0) { 1446 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1447 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 1448 } 1449 1450 CURVNET_RESTORE(); 1451 NET_EPOCH_EXIT(et); 1452 1453 crypto_freereq(crp); 1454 ovpn_peer_release_ref(peer, false); 1455 1456 return (0); 1457 } 1458 1459 static void 1460 ovpn_finish_rx(struct ovpn_softc *sc, struct mbuf *m, 1461 struct ovpn_kpeer *peer, struct ovpn_kkey *key, uint32_t seq, 1462 struct rm_priotracker *_ovpn_lock_trackerp) 1463 { 1464 uint32_t af; 1465 1466 OVPN_RASSERT(sc); 1467 NET_EPOCH_ASSERT(); 1468 1469 /* Replay protection. */ 1470 if (V_replay_protection && ! ovpn_check_replay(key->decrypt, seq)) { 1471 OVPN_RUNLOCK(sc); 1472 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1473 m_freem(m); 1474 return; 1475 } 1476 1477 critical_enter(); 1478 *zpcpu_get(peer->last_active) = time_uptime; 1479 critical_exit(); 1480 1481 OVPN_RUNLOCK(sc); 1482 1483 OVPN_COUNTER_ADD(sc, received_data_pkts, 1); 1484 OVPN_COUNTER_ADD(sc, tunnel_bytes_received, m->m_pkthdr.len); 1485 1486 /* Receive the packet on our interface. */ 1487 m->m_pkthdr.rcvif = sc->ifp; 1488 1489 /* Clear checksum flags in case the real hardware set them. */ 1490 m->m_pkthdr.csum_flags = 0; 1491 1492 /* Ensure we can read the first byte. */ 1493 m = m_pullup(m, 1); 1494 if (m == NULL) { 1495 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 1496 return; 1497 } 1498 1499 /* 1500 * Check for address family, and disregard any control packets (e.g. 1501 * keepalive). 1502 */ 1503 af = ovpn_get_af(m); 1504 if (af != 0) { 1505 BPF_MTAP2(sc->ifp, &af, sizeof(af), m); 1506 netisr_dispatch(af == AF_INET ? NETISR_IP : NETISR_IPV6, m); 1507 } else { 1508 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1509 m_freem(m); 1510 } 1511 } 1512 1513 static struct ovpn_kkey * 1514 ovpn_find_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, 1515 const struct ovpn_wire_header *ohdr) 1516 { 1517 struct ovpn_kkey *key = NULL; 1518 uint8_t keyid; 1519 1520 OVPN_RASSERT(sc); 1521 1522 keyid = (ntohl(ohdr->opcode) >> 24) & 0x07; 1523 1524 if (peer->keys[0].keyid == keyid) 1525 key = &peer->keys[0]; 1526 else if (peer->keys[1].keyid == keyid) 1527 key = &peer->keys[1]; 1528 1529 return (key); 1530 } 1531 1532 static int 1533 ovpn_decrypt_rx_cb(struct cryptop *crp) 1534 { 1535 struct epoch_tracker et; 1536 struct ovpn_softc *sc = crp->crp_opaque; 1537 struct mbuf *m = crp->crp_buf.cb_mbuf; 1538 struct ovpn_kkey *key; 1539 struct ovpn_kpeer *peer; 1540 struct ovpn_wire_header *ohdr; 1541 uint32_t peerid; 1542 1543 OVPN_RLOCK_TRACKER; 1544 1545 OVPN_RLOCK(sc); 1546 1547 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1548 1549 if (crp->crp_etype != 0) { 1550 crypto_freereq(crp); 1551 atomic_add_int(&sc->refcount, -1); 1552 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1553 OVPN_RUNLOCK(sc); 1554 m_freem(m); 1555 return (0); 1556 } 1557 1558 CURVNET_SET(sc->ifp->if_vnet); 1559 1560 ohdr = mtodo(m, sizeof(struct udphdr)); 1561 1562 peerid = ntohl(ohdr->opcode) & 0x00ffffff; 1563 peer = ovpn_find_peer(sc, peerid); 1564 if (peer == NULL) { 1565 /* No such peer. Drop packet. */ 1566 crypto_freereq(crp); 1567 atomic_add_int(&sc->refcount, -1); 1568 OVPN_RUNLOCK(sc); 1569 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1570 m_freem(m); 1571 CURVNET_RESTORE(); 1572 return (0); 1573 } 1574 1575 key = ovpn_find_key(sc, peer, ohdr); 1576 if (key == NULL) { 1577 crypto_freereq(crp); 1578 atomic_add_int(&sc->refcount, -1); 1579 /* 1580 * Has this key been removed between us starting the decrypt 1581 * and finishing it? 1582 */ 1583 OVPN_RUNLOCK(sc); 1584 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1585 m_freem(m); 1586 CURVNET_RESTORE(); 1587 return (0); 1588 } 1589 1590 /* Now remove the outer headers */ 1591 m_adj_decap(m, sizeof(struct udphdr) + 1592 sizeof(struct ovpn_wire_header)); 1593 1594 NET_EPOCH_ENTER(et); 1595 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), _ovpn_lock_trackerp); 1596 NET_EPOCH_EXIT(et); 1597 OVPN_UNLOCK_ASSERT(sc); 1598 1599 CURVNET_RESTORE(); 1600 1601 crypto_freereq(crp); 1602 atomic_add_int(&sc->refcount, -1); 1603 1604 return (0); 1605 } 1606 1607 static int 1608 ovpn_get_af(struct mbuf *m) 1609 { 1610 struct ip *ip; 1611 struct ip6_hdr *ip6; 1612 1613 /* 1614 * We should pullup, but we're only interested in the first byte, so 1615 * that'll always be contiguous. 1616 */ 1617 ip = mtod(m, struct ip *); 1618 if (ip->ip_v == IPVERSION) 1619 return (AF_INET); 1620 1621 ip6 = mtod(m, struct ip6_hdr *); 1622 if ((ip6->ip6_vfc & IPV6_VERSION_MASK) == IPV6_VERSION) 1623 return (AF_INET6); 1624 1625 return (0); 1626 } 1627 1628 #ifdef INET 1629 static struct ovpn_kpeer * 1630 ovpn_find_peer_by_ip(struct ovpn_softc *sc, const struct in_addr addr) 1631 { 1632 struct ovpn_kpeer *peer = NULL; 1633 1634 OVPN_ASSERT(sc); 1635 1636 for (int i = 0; i < OVPN_MAX_PEERS; i++) { 1637 if (sc->peers[i] == NULL) 1638 continue; 1639 if (addr.s_addr == sc->peers[i]->vpn4.s_addr) { 1640 peer = sc->peers[i]; 1641 break; 1642 } 1643 } 1644 1645 return (peer); 1646 } 1647 #endif 1648 1649 #ifdef INET6 1650 static struct ovpn_kpeer * 1651 ovpn_find_peer_by_ip6(struct ovpn_softc *sc, const struct in6_addr *addr) 1652 { 1653 struct ovpn_kpeer *peer = NULL; 1654 1655 OVPN_ASSERT(sc); 1656 1657 for (int i = 0; i < OVPN_MAX_PEERS; i++) { 1658 if (sc->peers[i] == NULL) 1659 continue; 1660 if (memcmp(addr, &sc->peers[i]->vpn6, sizeof(*addr)) == 0) { 1661 peer = sc->peers[i]; 1662 break; 1663 } 1664 } 1665 1666 return (peer); 1667 } 1668 #endif 1669 1670 static struct ovpn_kpeer * 1671 ovpn_route_peer(struct ovpn_softc *sc, struct mbuf **m0, 1672 const struct sockaddr *dst) 1673 { 1674 struct ovpn_kpeer *peer = NULL; 1675 int af; 1676 1677 NET_EPOCH_ASSERT(); 1678 OVPN_ASSERT(sc); 1679 1680 /* Shortcut if we're a client (or are a server and have only one client). */ 1681 if (sc->peercount == 1) 1682 return (ovpn_find_only_peer(sc)); 1683 1684 if (dst != NULL) 1685 af = dst->sa_family; 1686 else 1687 af = ovpn_get_af(*m0); 1688 1689 switch (af) { 1690 #ifdef INET 1691 case AF_INET: { 1692 const struct sockaddr_in *sa = (const struct sockaddr_in *)dst; 1693 struct nhop_object *nh; 1694 const struct in_addr *ip_dst; 1695 1696 if (sa != NULL) { 1697 ip_dst = &sa->sin_addr; 1698 } else { 1699 struct ip *ip; 1700 1701 *m0 = m_pullup(*m0, sizeof(struct ip)); 1702 if (*m0 == NULL) 1703 return (NULL); 1704 ip = mtod(*m0, struct ip *); 1705 ip_dst = &ip->ip_dst; 1706 } 1707 1708 peer = ovpn_find_peer_by_ip(sc, *ip_dst); 1709 SDT_PROBE2(if_ovpn, tx, route, ip4, ip_dst, peer); 1710 if (peer == NULL) { 1711 nh = fib4_lookup(M_GETFIB(*m0), *ip_dst, 0, 1712 NHR_NONE, 0); 1713 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1714 peer = ovpn_find_peer_by_ip(sc, 1715 nh->gw4_sa.sin_addr); 1716 SDT_PROBE2(if_ovpn, tx, route, ip4, 1717 &nh->gw4_sa.sin_addr, peer); 1718 } 1719 } 1720 break; 1721 } 1722 #endif 1723 #ifdef INET6 1724 case AF_INET6: { 1725 const struct sockaddr_in6 *sa6 = 1726 (const struct sockaddr_in6 *)dst; 1727 struct nhop_object *nh; 1728 const struct in6_addr *ip6_dst; 1729 1730 if (sa6 != NULL) { 1731 ip6_dst = &sa6->sin6_addr; 1732 } else { 1733 struct ip6_hdr *ip6; 1734 1735 *m0 = m_pullup(*m0, sizeof(struct ip6_hdr)); 1736 if (*m0 == NULL) 1737 return (NULL); 1738 ip6 = mtod(*m0, struct ip6_hdr *); 1739 ip6_dst = &ip6->ip6_dst; 1740 } 1741 1742 peer = ovpn_find_peer_by_ip6(sc, ip6_dst); 1743 SDT_PROBE2(if_ovpn, tx, route, ip6, ip6_dst, peer); 1744 if (peer == NULL) { 1745 nh = fib6_lookup(M_GETFIB(*m0), ip6_dst, 0, 1746 NHR_NONE, 0); 1747 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1748 peer = ovpn_find_peer_by_ip6(sc, 1749 &nh->gw6_sa.sin6_addr); 1750 SDT_PROBE2(if_ovpn, tx, route, ip6, 1751 &nh->gw6_sa.sin6_addr, peer); 1752 } 1753 } 1754 break; 1755 } 1756 #endif 1757 } 1758 1759 return (peer); 1760 } 1761 1762 static int 1763 ovpn_transmit(struct ifnet *ifp, struct mbuf *m) 1764 { 1765 return (ifp->if_output(ifp, m, NULL, NULL)); 1766 } 1767 1768 static int 1769 ovpn_transmit_to_peer(struct ifnet *ifp, struct mbuf *m, 1770 struct ovpn_kpeer *peer, struct rm_priotracker *_ovpn_lock_trackerp) 1771 { 1772 struct ovpn_wire_header *ohdr; 1773 struct ovpn_kkey *key; 1774 struct ovpn_softc *sc; 1775 struct cryptop *crp; 1776 uint32_t af, seq; 1777 size_t len, ovpn_hdr_len; 1778 int tunnel_len; 1779 int ret; 1780 1781 sc = ifp->if_softc; 1782 1783 OVPN_RASSERT(sc); 1784 1785 tunnel_len = m->m_pkthdr.len; 1786 1787 key = &peer->keys[OVPN_KEY_SLOT_PRIMARY]; 1788 if (key->encrypt == NULL) { 1789 if (_ovpn_lock_trackerp != NULL) 1790 OVPN_RUNLOCK(sc); 1791 m_freem(m); 1792 return (ENOLINK); 1793 } 1794 1795 af = ovpn_get_af(m); 1796 /* Don't capture control packets. */ 1797 if (af != 0) 1798 BPF_MTAP2(ifp, &af, sizeof(af), m); 1799 1800 len = m->m_pkthdr.len; 1801 MPASS(len <= ifp->if_mtu); 1802 1803 ovpn_hdr_len = sizeof(struct ovpn_wire_header); 1804 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) 1805 ovpn_hdr_len -= 16; /* No auth tag. */ 1806 1807 M_PREPEND(m, ovpn_hdr_len, M_NOWAIT); 1808 if (m == NULL) { 1809 if (_ovpn_lock_trackerp != NULL) 1810 OVPN_RUNLOCK(sc); 1811 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1812 return (ENOBUFS); 1813 } 1814 ohdr = mtod(m, struct ovpn_wire_header *); 1815 ohdr->opcode = (OVPN_OP_DATA_V2 << OVPN_OP_SHIFT) | key->keyid; 1816 ohdr->opcode <<= 24; 1817 ohdr->opcode |= key->peerid; 1818 ohdr->opcode = htonl(ohdr->opcode); 1819 1820 seq = atomic_fetchadd_32(&peer->tx_seq, 1); 1821 seq = htonl(seq); 1822 ohdr->seq = seq; 1823 1824 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) { 1825 ret = ovpn_encap(sc, peer->peerid, m); 1826 if (_ovpn_lock_trackerp != NULL) 1827 OVPN_RUNLOCK(sc); 1828 if (ret == 0) { 1829 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1830 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 1831 } 1832 return (ret); 1833 } 1834 1835 crp = crypto_getreq(key->encrypt->cryptoid, M_NOWAIT); 1836 if (crp == NULL) { 1837 if (_ovpn_lock_trackerp != NULL) 1838 OVPN_RUNLOCK(sc); 1839 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1840 m_freem(m); 1841 return (ENOBUFS); 1842 } 1843 1844 /* Encryption covers only the payload, not the header. */ 1845 crp->crp_payload_start = sizeof(*ohdr); 1846 crp->crp_payload_length = len; 1847 crp->crp_op = CRYPTO_OP_ENCRYPT; 1848 1849 /* 1850 * AAD data covers the ovpn_wire_header minus the auth 1851 * tag. 1852 */ 1853 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 1854 crp->crp_aad = ohdr; 1855 crp->crp_aad_start = 0; 1856 crp->crp_op |= CRYPTO_OP_COMPUTE_DIGEST; 1857 crp->crp_digest_start = offsetof(struct ovpn_wire_header, auth_tag); 1858 1859 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 1860 memcpy(crp->crp_iv, &seq, sizeof(seq)); 1861 memcpy(crp->crp_iv + sizeof(seq), key->encrypt->nonce, 1862 key->encrypt->noncelen); 1863 1864 crypto_use_mbuf(crp, m); 1865 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 1866 crp->crp_callback = ovpn_encrypt_tx_cb; 1867 crp->crp_opaque = peer; 1868 1869 atomic_add_int(&peer->refcount, 1); 1870 if (_ovpn_lock_trackerp != NULL) 1871 OVPN_RUNLOCK(sc); 1872 ret = crypto_dispatch(crp); 1873 if (ret) { 1874 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1875 } 1876 1877 return (ret); 1878 } 1879 1880 /* 1881 * Note: Expects to hold the read lock on entry, and will release it itself. 1882 */ 1883 static int 1884 ovpn_encap(struct ovpn_softc *sc, uint32_t peerid, struct mbuf *m) 1885 { 1886 struct udphdr *udp; 1887 struct ovpn_kpeer *peer; 1888 int len; 1889 1890 OVPN_RLOCK_TRACKER; 1891 1892 OVPN_RLOCK(sc); 1893 NET_EPOCH_ASSERT(); 1894 1895 peer = ovpn_find_peer(sc, peerid); 1896 if (peer == NULL || sc->ifp->if_link_state != LINK_STATE_UP) { 1897 OVPN_RUNLOCK(sc); 1898 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1899 m_freem(m); 1900 return (ENETDOWN); 1901 } 1902 1903 len = m->m_pkthdr.len; 1904 1905 M_PREPEND(m, sizeof(struct udphdr), M_NOWAIT); 1906 if (m == NULL) { 1907 OVPN_RUNLOCK(sc); 1908 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1909 m_freem(m); 1910 return (ENOBUFS); 1911 } 1912 udp = mtod(m, struct udphdr *); 1913 1914 MPASS(peer->local.ss_family == peer->remote.ss_family); 1915 1916 udp->uh_sport = ovpn_get_port(&peer->local); 1917 udp->uh_dport = ovpn_get_port(&peer->remote); 1918 udp->uh_ulen = htons(sizeof(struct udphdr) + len); 1919 1920 switch (peer->remote.ss_family) { 1921 #ifdef INET 1922 case AF_INET: { 1923 struct sockaddr_in *in_local = TO_IN(&peer->local); 1924 struct sockaddr_in *in_remote = TO_IN(&peer->remote); 1925 struct ip *ip; 1926 1927 /* 1928 * This requires knowing the source IP, which we don't. Happily 1929 * we're allowed to keep this at 0, and the checksum won't do 1930 * anything the crypto won't already do. 1931 */ 1932 udp->uh_sum = 0; 1933 1934 /* Set the checksum flags so we recalculate checksums. */ 1935 m->m_pkthdr.csum_flags |= CSUM_IP; 1936 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 1937 1938 M_PREPEND(m, sizeof(struct ip), M_NOWAIT); 1939 if (m == NULL) { 1940 OVPN_RUNLOCK(sc); 1941 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1942 return (ENOBUFS); 1943 } 1944 ip = mtod(m, struct ip *); 1945 1946 ip->ip_tos = 0; 1947 ip->ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) + 1948 len); 1949 ip->ip_off = 0; 1950 ip->ip_ttl = V_ip_defttl; 1951 ip->ip_p = IPPROTO_UDP; 1952 ip->ip_sum = 0; 1953 if (in_local->sin_port != 0) 1954 ip->ip_src = in_local->sin_addr; 1955 else 1956 ip->ip_src.s_addr = INADDR_ANY; 1957 ip->ip_dst = in_remote->sin_addr; 1958 1959 OVPN_RUNLOCK(sc); 1960 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 1961 1962 return (ip_output(m, NULL, NULL, 0, NULL, NULL)); 1963 } 1964 #endif 1965 #ifdef INET6 1966 case AF_INET6: { 1967 struct sockaddr_in6 *in6_local = TO_IN6(&peer->local); 1968 struct sockaddr_in6 *in6_remote = TO_IN6(&peer->remote); 1969 struct ip6_hdr *ip6; 1970 1971 M_PREPEND(m, sizeof(struct ip6_hdr), M_NOWAIT); 1972 if (m == NULL) { 1973 OVPN_RUNLOCK(sc); 1974 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1975 return (ENOBUFS); 1976 } 1977 m = m_pullup(m, sizeof(*ip6) + sizeof(*udp)); 1978 if (m == NULL) { 1979 OVPN_RUNLOCK(sc); 1980 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1981 return (ENOBUFS); 1982 } 1983 1984 ip6 = mtod(m, struct ip6_hdr *); 1985 1986 ip6->ip6_vfc = IPV6_VERSION; 1987 ip6->ip6_flow &= ~IPV6_FLOWINFO_MASK; 1988 ip6->ip6_plen = htons(sizeof(*ip6) + sizeof(struct udphdr) + 1989 len); 1990 ip6->ip6_nxt = IPPROTO_UDP; 1991 ip6->ip6_hlim = V_ip6_defhlim; 1992 1993 memcpy(&ip6->ip6_src, &in6_local->sin6_addr, 1994 sizeof(ip6->ip6_src)); 1995 memcpy(&ip6->ip6_dst, &in6_remote->sin6_addr, 1996 sizeof(ip6->ip6_dst)); 1997 1998 udp = mtodo(m, sizeof(*ip6)); 1999 udp->uh_sum = in6_cksum_pseudo(ip6, 2000 m->m_pkthdr.len - sizeof(struct ip6_hdr), 2001 IPPROTO_UDP, 0); 2002 2003 m->m_pkthdr.csum_flags |= CSUM_UDP_IPV6; 2004 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 2005 2006 OVPN_RUNLOCK(sc); 2007 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 2008 2009 return (ip6_output(m, NULL, NULL, IPV6_UNSPECSRC, NULL, NULL, 2010 NULL)); 2011 } 2012 #endif 2013 default: 2014 panic("Unsupported address family %d", 2015 peer->remote.ss_family); 2016 } 2017 } 2018 2019 static int 2020 ovpn_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 2021 struct route *ro) 2022 { 2023 struct ovpn_softc *sc; 2024 struct ovpn_kpeer *peer; 2025 2026 OVPN_RLOCK_TRACKER; 2027 2028 sc = ifp->if_softc; 2029 2030 OVPN_RLOCK(sc); 2031 2032 SDT_PROBE1(if_ovpn, tx, transmit, start, m); 2033 2034 if (__predict_false(ifp->if_link_state != LINK_STATE_UP)) { 2035 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2036 OVPN_RUNLOCK(sc); 2037 m_freem(m); 2038 return (ENETDOWN); 2039 } 2040 2041 /** 2042 * Only obey 'dst' (i.e. the gateway) if no route is supplied. 2043 * That's our indication that we're being called through pf's route-to, 2044 * and we should route according to 'dst' instead. We can't do so 2045 * consistently, because the usual openvpn configuration sets the first 2046 * non-server IP in the subnet as the gateway. If we always use that 2047 * one we'd end up routing all traffic to the first client. 2048 * tl;dr: 'ro == NULL' tells us pf is doing a route-to, and then but 2049 * only then, we should treat 'dst' as the destination. */ 2050 peer = ovpn_route_peer(sc, &m, ro == NULL ? dst : NULL); 2051 if (peer == NULL) { 2052 /* No destination. */ 2053 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2054 OVPN_RUNLOCK(sc); 2055 m_freem(m); 2056 return (ENETDOWN); 2057 } 2058 2059 return (ovpn_transmit_to_peer(ifp, m, peer, _ovpn_lock_trackerp)); 2060 } 2061 2062 static void 2063 ovpn_rcv_ctrl(struct ovpn_softc *sc, struct mbuf *m, int off) 2064 { 2065 /* Lop off the IP and UDP headers */ 2066 m_adj_decap(m, off); 2067 2068 /* Keep in the local ring until userspace fetches it. */ 2069 if (buf_ring_enqueue(sc->rxring, m) != 0) { 2070 OVPN_COUNTER_ADD(sc, lost_ctrl_pkts_in, 1); 2071 m_freem(m); 2072 return; 2073 } 2074 2075 OVPN_COUNTER_ADD(sc, received_ctrl_pkts, 1); 2076 } 2077 2078 static bool 2079 ovpn_check_replay(struct ovpn_kkey_dir *key, uint32_t seq) 2080 { 2081 uint32_t d; 2082 2083 mtx_lock(&key->replay_mtx); 2084 2085 /* Sequence number must be strictly greater than rx_seq */ 2086 if (seq <= key->rx_seq) { 2087 mtx_unlock(&key->replay_mtx); 2088 return (false); 2089 } 2090 2091 /* Large jump. The packet authenticated okay, so just accept that. */ 2092 if (seq > (key->rx_seq + (sizeof(key->rx_window) * 8))) { 2093 key->rx_seq = seq; 2094 key->rx_window = 0; 2095 mtx_unlock(&key->replay_mtx); 2096 return (true); 2097 } 2098 2099 /* Happy case. */ 2100 if ((seq == key->rx_seq + 1) && key->rx_window == 0) { 2101 key->rx_seq++; 2102 mtx_unlock(&key->replay_mtx); 2103 return (true); 2104 } 2105 2106 d = seq - key->rx_seq - 1; 2107 2108 if (key->rx_window & ((uint64_t)1 << d)) { 2109 /* Dupe! */ 2110 mtx_unlock(&key->replay_mtx); 2111 return (false); 2112 } 2113 2114 key->rx_window |= (uint64_t)1 << d; 2115 2116 while (key->rx_window & 1) { 2117 key->rx_seq++; 2118 key->rx_window >>= 1; 2119 } 2120 2121 mtx_unlock(&key->replay_mtx); 2122 2123 return (true); 2124 } 2125 2126 static struct ovpn_kpeer * 2127 ovpn_peer_from_mbuf(struct ovpn_softc *sc, struct mbuf *m, int off) 2128 { 2129 struct ovpn_wire_header ohdr; 2130 uint32_t peerid; 2131 const size_t hdrlen = sizeof(ohdr) - sizeof(ohdr.auth_tag); 2132 2133 OVPN_RASSERT(sc); 2134 2135 if (m_length(m, NULL) < (off + sizeof(struct udphdr) + hdrlen)) 2136 return (NULL); 2137 2138 m_copydata(m, off + sizeof(struct udphdr), hdrlen, (caddr_t)&ohdr); 2139 2140 peerid = ntohl(ohdr.opcode) & 0x00ffffff; 2141 2142 return (ovpn_find_peer(sc, peerid)); 2143 } 2144 2145 static bool 2146 ovpn_udp_input(struct mbuf *m, int off, struct inpcb *inp, 2147 const struct sockaddr *sa, void *ctx) 2148 { 2149 struct ovpn_softc *sc = ctx; 2150 struct ovpn_wire_header *ohdr; 2151 struct udphdr *uhdr; 2152 struct ovpn_kkey *key; 2153 struct cryptop *crp; 2154 struct ovpn_kpeer *peer; 2155 size_t ohdrlen; 2156 int ret; 2157 uint8_t op; 2158 2159 OVPN_RLOCK_TRACKER; 2160 2161 M_ASSERTPKTHDR(m); 2162 2163 OVPN_COUNTER_ADD(sc, transport_bytes_received, m->m_pkthdr.len - off); 2164 2165 ohdrlen = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2166 2167 OVPN_RLOCK(sc); 2168 2169 peer = ovpn_peer_from_mbuf(sc, m, off); 2170 if (peer == NULL) { 2171 OVPN_RUNLOCK(sc); 2172 return (false); 2173 } 2174 2175 m = m_pullup(m, off + sizeof(*uhdr) + ohdrlen); 2176 if (m == NULL) { 2177 OVPN_RUNLOCK(sc); 2178 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2179 return (true); 2180 } 2181 uhdr = mtodo(m, off); 2182 ohdr = mtodo(m, off + sizeof(*uhdr)); 2183 2184 op = ntohl(ohdr->opcode) >> 24 >> OVPN_OP_SHIFT; 2185 2186 /* 2187 * Simplify things by getting rid of the preceding headers, we don't 2188 * care about them. 2189 */ 2190 m_adj_decap(m, off); 2191 2192 uhdr = mtodo(m, 0); 2193 ohdr = mtodo(m, sizeof(*uhdr)); 2194 2195 if (op != OVPN_OP_DATA_V2) { 2196 OVPN_RUNLOCK(sc); 2197 ovpn_rcv_ctrl(sc, m, sizeof(struct udphdr)); 2198 INP_WLOCK(inp); 2199 udp_notify(inp, EAGAIN); 2200 INP_WUNLOCK(inp); 2201 return (true); 2202 } 2203 2204 key = ovpn_find_key(sc, peer, ohdr); 2205 if (key == NULL || key->decrypt == NULL) { 2206 OVPN_RUNLOCK(sc); 2207 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2208 m_freem(m); 2209 return (true); 2210 } 2211 2212 if (key->decrypt->cipher == OVPN_CIPHER_ALG_NONE) { 2213 /* Now remove the outer headers */ 2214 m_adj_decap(m, sizeof(struct udphdr) + ohdrlen); 2215 2216 ohdr = mtodo(m, sizeof(*uhdr)); 2217 2218 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), 2219 _ovpn_lock_trackerp); 2220 OVPN_UNLOCK_ASSERT(sc); 2221 return (true); 2222 } 2223 2224 ohdrlen += sizeof(ohdr->auth_tag); 2225 2226 m = m_pullup(m, sizeof(*uhdr) + ohdrlen); 2227 if (m == NULL) { 2228 OVPN_RUNLOCK(sc); 2229 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2230 return (true); 2231 } 2232 uhdr = mtodo(m, 0); 2233 ohdr = mtodo(m, sizeof(*uhdr)); 2234 2235 /* Decrypt */ 2236 crp = crypto_getreq(key->decrypt->cryptoid, M_NOWAIT); 2237 if (crp == NULL) { 2238 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2239 OVPN_RUNLOCK(sc); 2240 m_freem(m); 2241 return (true); 2242 } 2243 2244 crp->crp_payload_start = sizeof(struct udphdr) + sizeof(*ohdr); 2245 crp->crp_payload_length = ntohs(uhdr->uh_ulen) - 2246 sizeof(*uhdr) - sizeof(*ohdr); 2247 crp->crp_op = CRYPTO_OP_DECRYPT; 2248 2249 /* AAD validation. */ 2250 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2251 crp->crp_aad = ohdr; 2252 crp->crp_aad_start = 0; 2253 crp->crp_op |= CRYPTO_OP_VERIFY_DIGEST; 2254 crp->crp_digest_start = sizeof(struct udphdr) + 2255 offsetof(struct ovpn_wire_header, auth_tag); 2256 2257 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 2258 memcpy(crp->crp_iv, &ohdr->seq, sizeof(ohdr->seq)); 2259 memcpy(crp->crp_iv + sizeof(ohdr->seq), key->decrypt->nonce, 2260 key->decrypt->noncelen); 2261 2262 crypto_use_mbuf(crp, m); 2263 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 2264 crp->crp_callback = ovpn_decrypt_rx_cb; 2265 crp->crp_opaque = sc; 2266 2267 atomic_add_int(&sc->refcount, 1); 2268 OVPN_RUNLOCK(sc); 2269 ret = crypto_dispatch(crp); 2270 if (ret != 0) { 2271 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2272 } 2273 2274 return (true); 2275 } 2276 2277 static void 2278 ovpn_qflush(struct ifnet *ifp __unused) 2279 { 2280 2281 } 2282 2283 static void 2284 ovpn_flush_rxring(struct ovpn_softc *sc) 2285 { 2286 struct mbuf *m; 2287 struct ovpn_notification *n; 2288 2289 OVPN_WASSERT(sc); 2290 2291 while (! buf_ring_empty(sc->rxring)) { 2292 m = buf_ring_dequeue_sc(sc->rxring); 2293 m_freem(m); 2294 } 2295 2296 while (! buf_ring_empty(sc->notifring)) { 2297 n = buf_ring_dequeue_sc(sc->notifring); 2298 free(n, M_OVPN); 2299 } 2300 } 2301 2302 #ifdef VIMAGE 2303 static void 2304 ovpn_reassign(struct ifnet *ifp, struct vnet *new_vnet __unused, 2305 char *unused __unused) 2306 { 2307 struct ovpn_softc *sc = ifp->if_softc; 2308 int i; 2309 int ret __diagused; 2310 2311 i = 0; 2312 2313 OVPN_WLOCK(sc); 2314 2315 /* Flush keys & configuration. */ 2316 do { 2317 if (sc->peers[i] != NULL) { 2318 ret = _ovpn_del_peer(sc, sc->peers[i]->peerid); 2319 MPASS(ret == 0); 2320 } 2321 i++; 2322 } while (i < OVPN_MAX_PEERS); 2323 2324 ovpn_flush_rxring(sc); 2325 2326 OVPN_WUNLOCK(sc); 2327 } 2328 #endif 2329 2330 static int 2331 ovpn_clone_match(struct if_clone *ifc, const char *name) 2332 { 2333 /* 2334 * Allow all names that start with 'ovpn', specifically because pfSense 2335 * uses ovpnc1 / ovpns2 2336 */ 2337 return (strncmp(ovpnname, name, strlen(ovpnname)) == 0); 2338 } 2339 2340 static int 2341 ovpn_clone_create(struct if_clone *ifc, char *name, size_t len, 2342 struct ifc_data *ifd, struct ifnet **ifpp) 2343 { 2344 struct ovpn_softc *sc; 2345 struct ifnet *ifp; 2346 char *dp; 2347 int error, unit, wildcard; 2348 2349 /* Try to see if a special unit was requested. */ 2350 error = ifc_name2unit(name, &unit); 2351 if (error != 0) 2352 return (error); 2353 wildcard = (unit < 0); 2354 2355 error = ifc_alloc_unit(ifc, &unit); 2356 if (error != 0) 2357 return (error); 2358 2359 /* 2360 * If no unit had been given, we need to adjust the ifName. 2361 */ 2362 for (dp = name; *dp != '\0'; dp++); 2363 if (wildcard) { 2364 error = snprintf(dp, len - (dp - name), "%d", unit); 2365 if (error > len - (dp - name)) { 2366 /* ifName too long. */ 2367 ifc_free_unit(ifc, unit); 2368 return (ENOSPC); 2369 } 2370 dp += error; 2371 } 2372 2373 /* Make sure it doesn't already exist. */ 2374 if (ifunit(name) != NULL) 2375 return (EEXIST); 2376 2377 sc = malloc(sizeof(struct ovpn_softc), M_OVPN, M_WAITOK | M_ZERO); 2378 sc->ifp = if_alloc(IFT_ENC); 2379 rm_init_flags(&sc->lock, "if_ovpn_lock", RM_RECURSE); 2380 sc->refcount = 0; 2381 2382 sc->rxring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL); 2383 sc->notifring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL); 2384 2385 COUNTER_ARRAY_ALLOC(sc->counters, OVPN_COUNTER_SIZE, M_WAITOK); 2386 2387 ifp = sc->ifp; 2388 ifp->if_softc = sc; 2389 strlcpy(ifp->if_xname, name, IFNAMSIZ); 2390 ifp->if_dname = ovpngroupname; 2391 ifp->if_dunit = unit; 2392 2393 ifp->if_addrlen = 0; 2394 ifp->if_mtu = 1428; 2395 ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST; 2396 ifp->if_ioctl = ovpn_ioctl; 2397 ifp->if_transmit = ovpn_transmit; 2398 ifp->if_output = ovpn_output; 2399 ifp->if_qflush = ovpn_qflush; 2400 #ifdef VIMAGE 2401 ifp->if_reassign = ovpn_reassign; 2402 #endif 2403 ifp->if_capabilities |= IFCAP_LINKSTATE; 2404 ifp->if_capenable |= IFCAP_LINKSTATE; 2405 2406 if_attach(ifp); 2407 bpfattach(ifp, DLT_NULL, sizeof(uint32_t)); 2408 *ifpp = ifp; 2409 2410 return (0); 2411 } 2412 2413 static void 2414 ovpn_clone_destroy_cb(struct epoch_context *ctx) 2415 { 2416 struct ovpn_softc *sc; 2417 2418 sc = __containerof(ctx, struct ovpn_softc, epoch_ctx); 2419 2420 MPASS(sc->peercount == 0); 2421 for (int i = 0; i < OVPN_MAX_PEERS; i++) { 2422 MPASS(sc->peers[i] == NULL); 2423 } 2424 2425 COUNTER_ARRAY_FREE(sc->counters, OVPN_COUNTER_SIZE); 2426 2427 if_free(sc->ifp); 2428 free(sc, M_OVPN); 2429 } 2430 2431 static int 2432 ovpn_clone_destroy(struct if_clone *ifc, struct ifnet *ifp, uint32_t flags) 2433 { 2434 struct ovpn_softc *sc; 2435 int unit; 2436 int i; 2437 int ret __diagused; 2438 2439 sc = ifp->if_softc; 2440 unit = ifp->if_dunit; 2441 2442 OVPN_WLOCK(sc); 2443 2444 if (atomic_load_int(&sc->refcount) > 0) { 2445 OVPN_WUNLOCK(sc); 2446 return (EBUSY); 2447 } 2448 2449 i = 0; 2450 do { 2451 if (sc->peers[i] != NULL) { 2452 ret = _ovpn_del_peer(sc, sc->peers[i]->peerid); 2453 MPASS(ret == 0); 2454 } 2455 i++; 2456 } while (i < OVPN_MAX_PEERS); 2457 2458 ovpn_flush_rxring(sc); 2459 buf_ring_free(sc->rxring, M_OVPN); 2460 buf_ring_free(sc->notifring, M_OVPN); 2461 2462 OVPN_WUNLOCK(sc); 2463 2464 bpfdetach(ifp); 2465 if_detach(ifp); 2466 ifp->if_softc = NULL; 2467 2468 NET_EPOCH_CALL(ovpn_clone_destroy_cb, &sc->epoch_ctx); 2469 2470 if (unit != IF_DUNIT_NONE) 2471 ifc_free_unit(ifc, unit); 2472 2473 NET_EPOCH_DRAIN_CALLBACKS(); 2474 2475 return (0); 2476 } 2477 2478 static void 2479 vnet_ovpn_init(const void *unused __unused) 2480 { 2481 struct if_clone_addreq req = { 2482 .match_f = ovpn_clone_match, 2483 .create_f = ovpn_clone_create, 2484 .destroy_f = ovpn_clone_destroy, 2485 }; 2486 V_ovpn_cloner = ifc_attach_cloner(ovpngroupname, &req); 2487 } 2488 VNET_SYSINIT(vnet_ovpn_init, SI_SUB_PSEUDO, SI_ORDER_ANY, 2489 vnet_ovpn_init, NULL); 2490 2491 static void 2492 vnet_ovpn_uninit(const void *unused __unused) 2493 { 2494 if_clone_detach(V_ovpn_cloner); 2495 } 2496 VNET_SYSUNINIT(vnet_ovpn_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY, 2497 vnet_ovpn_uninit, NULL); 2498 2499 static int 2500 ovpnmodevent(module_t mod, int type, void *data) 2501 { 2502 switch (type) { 2503 case MOD_LOAD: 2504 /* Done in vnet_ovpn_init() */ 2505 break; 2506 case MOD_UNLOAD: 2507 /* Done in vnet_ovpn_uninit() */ 2508 break; 2509 default: 2510 return (EOPNOTSUPP); 2511 } 2512 2513 return (0); 2514 } 2515 2516 static moduledata_t ovpn_mod = { 2517 "if_ovpn", 2518 ovpnmodevent, 2519 0 2520 }; 2521 2522 DECLARE_MODULE(if_ovpn, ovpn_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 2523 MODULE_VERSION(if_ovpn, 1); 2524