1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2021-2022 Rubicon Communications, LLC (Netgate) 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 */ 28 #include "opt_inet.h" 29 #include "opt_inet6.h" 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/buf_ring.h> 34 #include <sys/epoch.h> 35 #include <sys/file.h> 36 #include <sys/filedesc.h> 37 #include <sys/kernel.h> 38 #include <sys/malloc.h> 39 #include <sys/mbuf.h> 40 #include <sys/module.h> 41 #include <sys/nv.h> 42 #include <sys/priv.h> 43 #include <sys/protosw.h> 44 #include <sys/rmlock.h> 45 #include <sys/sdt.h> 46 #include <sys/smp.h> 47 #include <sys/socket.h> 48 #include <sys/socketvar.h> 49 #include <sys/sockio.h> 50 #include <sys/sysctl.h> 51 #include <sys/time.h> 52 53 #include <machine/atomic.h> 54 55 #include <net/bpf.h> 56 #include <net/if.h> 57 #include <net/if_clone.h> 58 #include <net/if_types.h> 59 #include <net/if_var.h> 60 #include <net/netisr.h> 61 #include <net/route/nhop.h> 62 63 #include <netinet/in.h> 64 #include <netinet/in_fib.h> 65 #include <netinet/ip.h> 66 #include <netinet/ip6.h> 67 #include <netinet/ip_var.h> 68 #include <netinet/udp.h> 69 #include <netinet/udp_var.h> 70 71 #include <netinet6/ip6_var.h> 72 #include <netinet6/in6_fib.h> 73 74 #include <machine/in_cksum.h> 75 76 #include <opencrypto/cryptodev.h> 77 78 #include "if_ovpn.h" 79 80 struct ovpn_kkey_dir { 81 int refcount; 82 uint8_t key[32]; 83 uint8_t keylen; 84 uint8_t nonce[8]; 85 uint8_t noncelen; 86 enum ovpn_key_cipher cipher; 87 crypto_session_t cryptoid; 88 89 struct mtx replay_mtx; 90 /* 91 * Last seen gapless sequence number. New rx seq numbers must be 92 * strictly higher than this. 93 */ 94 uint32_t rx_seq; 95 /* Seen packets, relative to rx_seq. bit(0) will always be 0. */ 96 uint64_t rx_window; 97 }; 98 99 struct ovpn_kkey { 100 struct ovpn_kkey_dir *encrypt; 101 struct ovpn_kkey_dir *decrypt; 102 uint8_t keyid; 103 uint32_t peerid; 104 }; 105 106 struct ovpn_keepalive { 107 uint32_t interval; 108 uint32_t timeout; 109 }; 110 111 struct ovpn_wire_header { 112 uint32_t opcode; /* opcode, key id, peer id */ 113 uint32_t seq; 114 uint8_t auth_tag[16]; 115 }; 116 117 struct ovpn_notification { 118 enum ovpn_notif_type type; 119 uint32_t peerid; 120 }; 121 122 struct ovpn_softc; 123 124 struct ovpn_kpeer { 125 int refcount; 126 uint32_t peerid; 127 128 struct ovpn_softc *sc; 129 struct sockaddr_storage local; 130 struct sockaddr_storage remote; 131 132 struct in_addr vpn4; 133 struct in6_addr vpn6; 134 135 struct ovpn_kkey keys[2]; 136 uint32_t tx_seq; 137 138 struct ovpn_keepalive keepalive; 139 uint32_t *last_active; 140 struct callout ping_send; 141 struct callout ping_rcv; 142 }; 143 144 #define OVPN_MAX_PEERS 128 145 146 struct ovpn_counters { 147 uint64_t lost_ctrl_pkts_in; 148 uint64_t lost_ctrl_pkts_out; 149 uint64_t lost_data_pkts_in; 150 uint64_t lost_data_pkts_out; 151 uint64_t nomem_data_pkts_in; 152 uint64_t nomem_data_pkts_out; 153 uint64_t received_ctrl_pkts; 154 uint64_t received_data_pkts; 155 uint64_t sent_ctrl_pkts; 156 uint64_t sent_data_pkts; 157 158 uint64_t transport_bytes_sent; 159 uint64_t transport_bytes_received; 160 uint64_t tunnel_bytes_sent; 161 uint64_t tunnel_bytes_received; 162 }; 163 #define OVPN_COUNTER_SIZE (sizeof(struct ovpn_counters)/sizeof(uint64_t)) 164 165 struct ovpn_softc { 166 int refcount; 167 struct rmlock lock; 168 struct ifnet *ifp; 169 struct socket *so; 170 int peercount; 171 struct ovpn_kpeer *peers[OVPN_MAX_PEERS]; /* XXX Hard limit for now? */ 172 173 /* Pending packets */ 174 struct buf_ring *rxring; 175 struct buf_ring *notifring; 176 177 counter_u64_t counters[OVPN_COUNTER_SIZE]; 178 179 struct epoch_context epoch_ctx; 180 }; 181 182 static struct ovpn_kpeer *ovpn_find_peer(struct ovpn_softc *, uint32_t); 183 static bool ovpn_udp_input(struct mbuf *, int, struct inpcb *, 184 const struct sockaddr *, void *); 185 static int ovpn_transmit_to_peer(struct ifnet *, struct mbuf *, 186 struct ovpn_kpeer *, struct rm_priotracker *); 187 static int ovpn_encap(struct ovpn_softc *, uint32_t, struct mbuf *); 188 static int ovpn_get_af(struct mbuf *); 189 static void ovpn_free_kkey_dir(struct ovpn_kkey_dir *); 190 static bool ovpn_check_replay(struct ovpn_kkey_dir *, uint32_t); 191 192 #define OVPN_MTU_MIN 576 193 #define OVPN_MTU_MAX (IP_MAXPACKET - sizeof(struct ip) - \ 194 sizeof(struct udphdr) - sizeof(struct ovpn_wire_header)) 195 196 #define OVPN_OP_DATA_V2 0x09 197 #define OVPN_OP_SHIFT 3 198 199 VNET_DEFINE_STATIC(struct if_clone *, ovpn_cloner); 200 #define V_ovpn_cloner VNET(ovpn_cloner) 201 202 #define OVPN_RLOCK_TRACKER struct rm_priotracker _ovpn_lock_tracker; \ 203 struct rm_priotracker *_ovpn_lock_trackerp = &_ovpn_lock_tracker 204 #define OVPN_RLOCK(sc) rm_rlock(&(sc)->lock, _ovpn_lock_trackerp) 205 #define OVPN_RUNLOCK(sc) rm_runlock(&(sc)->lock, _ovpn_lock_trackerp) 206 #define OVPN_WLOCK(sc) rm_wlock(&(sc)->lock) 207 #define OVPN_WUNLOCK(sc) rm_wunlock(&(sc)->lock) 208 #define OVPN_ASSERT(sc) rm_assert(&(sc)->lock, RA_LOCKED) 209 #define OVPN_RASSERT(sc) rm_assert(&(sc)->lock, RA_RLOCKED) 210 #define OVPN_WASSERT(sc) rm_assert(&(sc)->lock, RA_WLOCKED) 211 #define OVPN_UNLOCK_ASSERT(sc) rm_assert(&(sc)->lock, RA_UNLOCKED) 212 213 #define OVPN_COUNTER_ADD(sc, name, val) \ 214 counter_u64_add(sc->counters[offsetof(struct ovpn_counters, name) / \ 215 sizeof(uint64_t)], val) 216 217 #define TO_IN(x) ((struct sockaddr_in *)(x)) 218 #define TO_IN6(x) ((struct sockaddr_in6 *)(x)) 219 220 SDT_PROVIDER_DEFINE(if_ovpn); 221 SDT_PROBE_DEFINE1(if_ovpn, tx, transmit, start, "struct mbuf *"); 222 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip4, "struct in_addr *", "struct ovpn_kpeer *"); 223 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip6, "struct in6_addr *", "struct ovpn_kpeer *"); 224 225 static const char ovpnname[] = "ovpn"; 226 static const char ovpngroupname[] = "openvpn"; 227 228 static MALLOC_DEFINE(M_OVPN, ovpnname, "OpenVPN DCO Interface"); 229 230 SYSCTL_DECL(_net_link); 231 static SYSCTL_NODE(_net_link, IFT_OTHER, openvpn, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 232 "OpenVPN DCO Interface"); 233 VNET_DEFINE_STATIC(int, replay_protection) = 0; 234 #define V_replay_protection VNET(replay_protection) 235 SYSCTL_INT(_net_link_openvpn, OID_AUTO, replay_protection, CTLFLAG_VNET | CTLFLAG_RW, 236 &VNET_NAME(replay_protection), 0, "Validate sequence numbers"); 237 238 static struct ovpn_kpeer * 239 ovpn_find_peer(struct ovpn_softc *sc, uint32_t peerid) 240 { 241 struct ovpn_kpeer *p = NULL; 242 243 OVPN_ASSERT(sc); 244 245 for (int i = 0; i < OVPN_MAX_PEERS; i++) { 246 p = sc->peers[i]; 247 if (p == NULL) 248 continue; 249 250 if (p->peerid == peerid) { 251 MPASS(p->sc == sc); 252 break; 253 } 254 } 255 256 return (p); 257 } 258 259 static struct ovpn_kpeer * 260 ovpn_find_only_peer(struct ovpn_softc *sc) 261 { 262 OVPN_ASSERT(sc); 263 264 for (int i = 0; i < OVPN_MAX_PEERS; i++) { 265 if (sc->peers[i] == NULL) 266 continue; 267 return (sc->peers[i]); 268 } 269 270 MPASS(false); 271 272 return (NULL); 273 } 274 275 static uint16_t 276 ovpn_get_port(struct sockaddr_storage *s) 277 { 278 switch (s->ss_family) { 279 case AF_INET: { 280 struct sockaddr_in *in = (struct sockaddr_in *)s; 281 return (in->sin_port); 282 } 283 case AF_INET6: { 284 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)s; 285 return (in6->sin6_port); 286 } 287 default: 288 panic("Unsupported address family %d", s->ss_family); 289 } 290 } 291 292 static int 293 ovpn_nvlist_to_sockaddr(const nvlist_t *nvl, struct sockaddr_storage *sa) 294 { 295 int af; 296 297 if (! nvlist_exists_number(nvl, "af")) 298 return (EINVAL); 299 if (! nvlist_exists_binary(nvl, "address")) 300 return (EINVAL); 301 if (! nvlist_exists_number(nvl, "port")) 302 return (EINVAL); 303 304 af = nvlist_get_number(nvl, "af"); 305 306 switch (af) { 307 #ifdef INET 308 case AF_INET: { 309 struct sockaddr_in *in = (struct sockaddr_in *)sa; 310 size_t len; 311 const void *addr = nvlist_get_binary(nvl, "address", &len); 312 in->sin_family = af; 313 if (len != sizeof(in->sin_addr)) 314 return (EINVAL); 315 316 memcpy(&in->sin_addr, addr, sizeof(in->sin_addr)); 317 in->sin_port = nvlist_get_number(nvl, "port"); 318 break; 319 } 320 #endif 321 #ifdef INET6 322 case AF_INET6: { 323 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)sa; 324 size_t len; 325 const void *addr = nvlist_get_binary(nvl, "address", &len); 326 in6->sin6_family = af; 327 if (len != sizeof(in6->sin6_addr)) 328 return (EINVAL); 329 330 memcpy(&in6->sin6_addr, addr, sizeof(in6->sin6_addr)); 331 in6->sin6_port = nvlist_get_number(nvl, "port"); 332 break; 333 } 334 #endif 335 default: 336 return (EINVAL); 337 } 338 339 return (0); 340 } 341 342 static bool 343 ovpn_has_peers(struct ovpn_softc *sc) 344 { 345 OVPN_ASSERT(sc); 346 347 return (sc->peercount > 0); 348 } 349 350 static void 351 ovpn_rele_so(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 352 { 353 bool has_peers; 354 355 OVPN_WASSERT(sc); 356 357 if (sc->so == NULL) 358 return; 359 360 has_peers = ovpn_has_peers(sc); 361 362 /* Only remove the tunnel function if we're releasing the socket for 363 * the last peer. */ 364 if (! has_peers) 365 (void)udp_set_kernel_tunneling(sc->so, NULL, NULL, NULL); 366 367 sorele(sc->so); 368 369 if (! has_peers) 370 sc->so = NULL; 371 } 372 373 static void 374 ovpn_notify_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 375 { 376 struct ovpn_notification *n; 377 378 OVPN_WASSERT(sc); 379 380 n = malloc(sizeof(*n), M_OVPN, M_NOWAIT); 381 if (n == NULL) 382 return; 383 384 n->peerid = peer->peerid; 385 n->type = OVPN_NOTIF_DEL_PEER; 386 if (buf_ring_enqueue(sc->notifring, n) != 0) { 387 free(n, M_OVPN); 388 } else if (sc->so != NULL) { 389 /* Wake up userspace */ 390 sc->so->so_error = EAGAIN; 391 sorwakeup(sc->so); 392 sowwakeup(sc->so); 393 } 394 } 395 396 static void 397 ovpn_peer_release_ref(struct ovpn_kpeer *peer, bool locked) 398 { 399 struct ovpn_softc *sc; 400 401 atomic_add_int(&peer->refcount, -1); 402 403 if (atomic_load_int(&peer->refcount) > 0) 404 return; 405 406 sc = peer->sc; 407 408 if (! locked) { 409 OVPN_WLOCK(sc); 410 411 /* Might have changed before we acquired the lock. */ 412 if (atomic_load_int(&peer->refcount) > 0) { 413 OVPN_WUNLOCK(sc); 414 return; 415 } 416 } 417 418 /* The peer should have been removed from the list already. */ 419 MPASS(ovpn_find_peer(sc, peer->peerid) == NULL); 420 421 ovpn_notify_del_peer(sc, peer); 422 423 for (int i = 0; i < 2; i++) { 424 ovpn_free_kkey_dir(peer->keys[i].encrypt); 425 ovpn_free_kkey_dir(peer->keys[i].decrypt); 426 } 427 428 ovpn_rele_so(sc, peer); 429 430 callout_stop(&peer->ping_send); 431 callout_stop(&peer->ping_rcv); 432 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 433 free(peer, M_OVPN); 434 435 if (! locked) 436 OVPN_WUNLOCK(sc); 437 } 438 439 static int 440 ovpn_new_peer(struct ifnet *ifp, const nvlist_t *nvl) 441 { 442 #ifdef INET6 443 struct epoch_tracker et; 444 #endif 445 struct sockaddr_storage remote; 446 struct ovpn_kpeer *peer = NULL; 447 struct file *fp = NULL; 448 struct sockaddr *name = NULL; 449 struct ovpn_softc *sc = ifp->if_softc; 450 struct thread *td = curthread; 451 struct socket *so = NULL; 452 u_int fflag; 453 int fd; 454 uint32_t peerid; 455 int ret = 0, i; 456 457 if (nvl == NULL) 458 return (EINVAL); 459 460 if (! nvlist_exists_number(nvl, "peerid")) 461 return (EINVAL); 462 463 if (! nvlist_exists_number(nvl, "fd")) 464 return (EINVAL); 465 466 if (! nvlist_exists_nvlist(nvl, "remote")) 467 return (EINVAL); 468 469 peerid = nvlist_get_number(nvl, "peerid"); 470 471 ret = ovpn_nvlist_to_sockaddr(nvlist_get_nvlist(nvl, "remote"), 472 &remote); 473 if (ret != 0) 474 return (ret); 475 476 fd = nvlist_get_number(nvl, "fd"); 477 478 /* Look up the userspace process and use the fd to find the socket. */ 479 ret = getsock_cap(td, fd, &cap_connect_rights, &fp, 480 &fflag, NULL); 481 if (ret != 0) 482 return (ret); 483 484 so = fp->f_data; 485 486 peer = malloc(sizeof(*peer), M_OVPN, M_WAITOK | M_ZERO); 487 peer->peerid = peerid; 488 peer->sc = sc; 489 peer->tx_seq = 1; 490 peer->refcount = 1; 491 peer->last_active = uma_zalloc_pcpu(pcpu_zone_4, M_WAITOK | M_ZERO); 492 493 if (nvlist_exists_binary(nvl, "vpn_ipv4")) { 494 size_t len; 495 const void *addr = nvlist_get_binary(nvl, "vpn_ipv4", &len); 496 if (len != sizeof(peer->vpn4)) { 497 ret = EINVAL; 498 goto error; 499 } 500 memcpy(&peer->vpn4, addr, len); 501 } 502 503 if (nvlist_exists_binary(nvl, "vpn_ipv6")) { 504 size_t len; 505 const void *addr = nvlist_get_binary(nvl, "vpn_ipv6", &len); 506 if (len != sizeof(peer->vpn6)) { 507 ret = EINVAL; 508 goto error; 509 } 510 memcpy(&peer->vpn6, addr, len); 511 } 512 513 callout_init_rm(&peer->ping_send, &sc->lock, CALLOUT_SHAREDLOCK); 514 callout_init_rm(&peer->ping_rcv, &sc->lock, 0); 515 516 ret = (*so->so_proto->pr_usrreqs->pru_sockaddr)(so, &name); 517 if (ret) 518 goto error; 519 520 if (ovpn_get_port((struct sockaddr_storage *)name) == 0) { 521 ret = EINVAL; 522 goto error; 523 } 524 if (name->sa_family != remote.ss_family) { 525 ret = EINVAL; 526 goto error; 527 } 528 529 memcpy(&peer->local, name, name->sa_len); 530 memcpy(&peer->remote, &remote, sizeof(remote)); 531 free(name, M_SONAME); 532 name = NULL; 533 534 if (peer->local.ss_family == AF_INET6 && 535 IN6_IS_ADDR_V4MAPPED(&TO_IN6(&peer->remote)->sin6_addr)) { 536 /* V4 mapped address, so treat this as v4, not v6. */ 537 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->local); 538 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->remote); 539 } 540 541 #ifdef INET6 542 if (peer->local.ss_family == AF_INET6 && 543 IN6_IS_ADDR_UNSPECIFIED(&TO_IN6(&peer->local)->sin6_addr)) { 544 NET_EPOCH_ENTER(et); 545 ret = in6_selectsrc_addr(curthread->td_proc->p_fibnum, 546 &TO_IN6(&peer->remote)->sin6_addr, 547 0, NULL, &TO_IN6(&peer->local)->sin6_addr, NULL); 548 NET_EPOCH_EXIT(et); 549 if (ret != 0) { 550 goto error; 551 } 552 } 553 #endif 554 OVPN_WLOCK(sc); 555 556 /* Disallow peer id re-use. */ 557 if (ovpn_find_peer(sc, peerid) != NULL) { 558 ret = EEXIST; 559 goto error_locked; 560 } 561 562 /* Must be the same socket as for other peers on this interface. */ 563 if (sc->so != NULL && so != sc->so) 564 goto error_locked; 565 566 if (sc->so == NULL) 567 sc->so = so; 568 569 /* Insert the peer into the list. */ 570 for (i = 0; i < OVPN_MAX_PEERS; i++) { 571 if (sc->peers[i] != NULL) 572 continue; 573 574 MPASS(sc->peers[i] == NULL); 575 sc->peers[i] = peer; 576 sc->peercount++; 577 soref(sc->so); 578 break; 579 } 580 if (i == OVPN_MAX_PEERS) { 581 ret = ENOSPC; 582 goto error_locked; 583 } 584 585 ret = udp_set_kernel_tunneling(sc->so, ovpn_udp_input, NULL, sc); 586 if (ret == EBUSY) { 587 /* Fine, another peer already set the input function. */ 588 ret = 0; 589 } 590 if (ret != 0) { 591 sc->peers[i] = NULL; 592 sc->peercount--; 593 goto error_locked; 594 } 595 596 OVPN_WUNLOCK(sc); 597 598 goto done; 599 600 error_locked: 601 OVPN_WUNLOCK(sc); 602 error: 603 free(name, M_SONAME); 604 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 605 free(peer, M_OVPN); 606 done: 607 if (fp != NULL) 608 fdrop(fp, td); 609 610 return (ret); 611 } 612 613 static int 614 _ovpn_del_peer(struct ovpn_softc *sc, uint32_t peerid) 615 { 616 struct ovpn_kpeer *peer; 617 int i; 618 619 OVPN_WASSERT(sc); 620 621 for (i = 0; i < OVPN_MAX_PEERS; i++) { 622 if (sc->peers[i] == NULL) 623 continue; 624 if (sc->peers[i]->peerid != peerid) 625 continue; 626 627 peer = sc->peers[i]; 628 break; 629 } 630 if (i == OVPN_MAX_PEERS) 631 return (ENOENT); 632 633 sc->peers[i] = NULL; 634 sc->peercount--; 635 636 ovpn_peer_release_ref(peer, true); 637 638 return (0); 639 } 640 641 static int 642 ovpn_del_peer(struct ifnet *ifp, nvlist_t *nvl) 643 { 644 struct ovpn_softc *sc = ifp->if_softc; 645 uint32_t peerid; 646 int ret; 647 648 OVPN_WASSERT(sc); 649 650 if (nvl == NULL) 651 return (EINVAL); 652 653 if (! nvlist_exists_number(nvl, "peerid")) 654 return (EINVAL); 655 656 peerid = nvlist_get_number(nvl, "peerid"); 657 658 ret = _ovpn_del_peer(sc, peerid); 659 660 return (ret); 661 } 662 663 static int 664 ovpn_create_kkey_dir(struct ovpn_kkey_dir **kdirp, 665 const nvlist_t *nvl) 666 { 667 struct crypto_session_params csp; 668 struct ovpn_kkey_dir *kdir; 669 const char *ciphername; 670 enum ovpn_key_cipher cipher; 671 const void *key, *iv; 672 size_t keylen = 0, ivlen = 0; 673 int error; 674 675 if (! nvlist_exists_string(nvl, "cipher")) 676 return (EINVAL); 677 ciphername = nvlist_get_string(nvl, "cipher"); 678 679 if (strcmp(ciphername, "none") == 0) 680 cipher = OVPN_CIPHER_ALG_NONE; 681 else if (strcmp(ciphername, "AES-256-GCM") == 0) 682 cipher = OVPN_CIPHER_ALG_AES_GCM; 683 else if (strcmp(ciphername, "CHACHA20-POLY1305") == 0) 684 cipher = OVPN_CIPHER_ALG_CHACHA20_POLY1305; 685 else 686 return (EINVAL); 687 688 if (cipher != OVPN_CIPHER_ALG_NONE) { 689 if (! nvlist_exists_binary(nvl, "key")) 690 return (EINVAL); 691 key = nvlist_get_binary(nvl, "key", &keylen); 692 if (keylen > sizeof(kdir->key)) 693 return (E2BIG); 694 695 if (! nvlist_exists_binary(nvl, "iv")) 696 return (EINVAL); 697 iv = nvlist_get_binary(nvl, "iv", &ivlen); 698 if (ivlen != 8) 699 return (E2BIG); 700 } 701 702 kdir = malloc(sizeof(struct ovpn_kkey_dir), M_OVPN, 703 M_WAITOK | M_ZERO); 704 705 kdir->cipher = cipher; 706 kdir->keylen = keylen; 707 memcpy(kdir->key, key, keylen); 708 kdir->noncelen = ivlen; 709 memcpy(kdir->nonce, iv, ivlen); 710 711 if (kdir->cipher != OVPN_CIPHER_ALG_NONE) { 712 /* Crypto init */ 713 bzero(&csp, sizeof(csp)); 714 csp.csp_mode = CSP_MODE_AEAD; 715 716 if (kdir->cipher == OVPN_CIPHER_ALG_CHACHA20_POLY1305) 717 csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305; 718 else 719 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 720 721 csp.csp_flags |= CSP_F_SEPARATE_AAD; 722 723 csp.csp_cipher_klen = kdir->keylen; 724 csp.csp_cipher_key = kdir->key; 725 csp.csp_ivlen = 96 / 8; 726 727 error = crypto_newsession(&kdir->cryptoid, &csp, 728 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE); 729 if (error) { 730 free(kdir, M_OVPN); 731 return (error); 732 } 733 } 734 735 mtx_init(&kdir->replay_mtx, "if_ovpn rx replay", NULL, MTX_DEF); 736 *kdirp = kdir; 737 738 return (0); 739 } 740 741 static void 742 ovpn_free_kkey_dir(struct ovpn_kkey_dir *kdir) 743 { 744 if (kdir == NULL) 745 return; 746 747 mtx_destroy(&kdir->replay_mtx); 748 749 crypto_freesession(kdir->cryptoid); 750 free(kdir, M_OVPN); 751 } 752 753 static int 754 ovpn_set_key(struct ifnet *ifp, const nvlist_t *nvl) 755 { 756 struct ovpn_softc *sc = ifp->if_softc; 757 struct ovpn_kkey_dir *enc, *dec; 758 struct ovpn_kpeer *peer; 759 int slot, keyid, peerid; 760 int error; 761 762 if (nvl == NULL) 763 return (EINVAL); 764 765 if (! nvlist_exists_number(nvl, "slot")) 766 return (EINVAL); 767 slot = nvlist_get_number(nvl, "slot"); 768 769 if (! nvlist_exists_number(nvl, "keyid")) 770 return (EINVAL); 771 keyid = nvlist_get_number(nvl, "keyid"); 772 773 if (! nvlist_exists_number(nvl, "peerid")) 774 return (EINVAL); 775 peerid = nvlist_get_number(nvl, "peerid"); 776 777 if (slot != OVPN_KEY_SLOT_PRIMARY && 778 slot != OVPN_KEY_SLOT_SECONDARY) 779 return (EINVAL); 780 781 if (! nvlist_exists_nvlist(nvl, "encrypt") || 782 ! nvlist_exists_nvlist(nvl, "decrypt")) 783 return (EINVAL); 784 785 error = ovpn_create_kkey_dir(&enc, nvlist_get_nvlist(nvl, "encrypt")); 786 if (error) 787 return (error); 788 789 error = ovpn_create_kkey_dir(&dec, nvlist_get_nvlist(nvl, "decrypt")); 790 if (error) { 791 ovpn_free_kkey_dir(enc); 792 return (error); 793 } 794 795 OVPN_WLOCK(sc); 796 797 peer = ovpn_find_peer(sc, peerid); 798 if (peer == NULL) { 799 ovpn_free_kkey_dir(dec); 800 ovpn_free_kkey_dir(enc); 801 OVPN_WUNLOCK(sc); 802 return (ENOENT); 803 } 804 805 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 806 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 807 808 peer->keys[slot].encrypt = enc; 809 peer->keys[slot].decrypt = dec; 810 811 peer->keys[slot].keyid = keyid; 812 peer->keys[slot].peerid = peerid; 813 814 OVPN_WUNLOCK(sc); 815 816 return (0); 817 } 818 819 static int 820 ovpn_check_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, enum ovpn_key_slot slot) 821 { 822 OVPN_ASSERT(sc); 823 824 if (peer->keys[slot].encrypt == NULL) 825 return (ENOLINK); 826 827 if (peer->keys[slot].decrypt == NULL) 828 return (ENOLINK); 829 830 return (0); 831 } 832 833 static int 834 ovpn_start(struct ifnet *ifp) 835 { 836 struct ovpn_softc *sc = ifp->if_softc; 837 838 OVPN_WLOCK(sc); 839 840 ifp->if_flags |= IFF_UP; 841 ifp->if_drv_flags |= IFF_DRV_RUNNING; 842 if_link_state_change(ifp, LINK_STATE_UP); 843 844 OVPN_WUNLOCK(sc); 845 846 return (0); 847 } 848 849 static int 850 ovpn_swap_keys(struct ifnet *ifp, nvlist_t *nvl) 851 { 852 struct ovpn_softc *sc = ifp->if_softc; 853 struct ovpn_kpeer *peer; 854 struct ovpn_kkey tmpkey; 855 int error; 856 857 if (nvl == NULL) 858 return (EINVAL); 859 860 if (! nvlist_exists_number(nvl, "peerid")) 861 return (EINVAL); 862 863 OVPN_WLOCK(sc); 864 865 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 866 if (peer == NULL) { 867 OVPN_WUNLOCK(sc); 868 return (ENOENT); 869 } 870 871 /* Check that we have a second key to swap to. */ 872 error = ovpn_check_key(sc, peer, OVPN_KEY_SLOT_SECONDARY); 873 if (error) { 874 OVPN_WUNLOCK(sc); 875 return (error); 876 } 877 878 tmpkey = peer->keys[0]; 879 peer->keys[0] = peer->keys[1]; 880 peer->keys[1] = tmpkey; 881 882 OVPN_WUNLOCK(sc); 883 884 return (0); 885 } 886 887 static int 888 ovpn_del_key(struct ifnet *ifp, const nvlist_t *nvl) 889 { 890 enum ovpn_key_slot slot; 891 struct ovpn_kpeer *peer; 892 struct ovpn_softc *sc = ifp->if_softc; 893 894 if (nvl == NULL) 895 return (EINVAL); 896 897 if (! nvlist_exists_number(nvl, "peerid")) 898 return (EINVAL); 899 900 if (! nvlist_exists_number(nvl, "slot")) 901 return (EINVAL); 902 slot = nvlist_get_number(nvl, "slot"); 903 904 if (slot != OVPN_KEY_SLOT_PRIMARY && 905 slot != OVPN_KEY_SLOT_SECONDARY) 906 return (EINVAL); 907 908 OVPN_WLOCK(sc); 909 910 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 911 if (peer == NULL) { 912 OVPN_WUNLOCK(sc); 913 return (ENOENT); 914 } 915 916 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 917 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 918 919 peer->keys[slot].encrypt = NULL; 920 peer->keys[slot].decrypt = NULL; 921 922 peer->keys[slot].keyid = 0; 923 peer->keys[slot].peerid = 0; 924 925 OVPN_WUNLOCK(sc); 926 927 return (0); 928 } 929 930 static int 931 ovpn_send_pkt(struct ifnet *ifp, const nvlist_t *nvl) 932 { 933 struct epoch_tracker et; 934 struct ovpn_softc *sc = ifp->if_softc; 935 struct mbuf *m; 936 const uint8_t *pkt; 937 size_t pktlen; 938 uint32_t peerid; 939 int ret; 940 941 if (nvl == NULL) 942 return (EINVAL); 943 944 if (! nvlist_exists_binary(nvl, "packet")) 945 return (EINVAL); 946 pkt = nvlist_get_binary(nvl, "packet", &pktlen); 947 948 if (! nvlist_exists_number(nvl, "peerid")) 949 return (EINVAL); 950 951 peerid = nvlist_get_number(nvl, "peerid"); 952 953 /* 954 * Check that userspace isn't giving us a data packet. That might lead 955 * to IV re-use, which would be bad. 956 */ 957 if ((pkt[0] >> OVPN_OP_SHIFT) == OVPN_OP_DATA_V2) 958 return (EINVAL); 959 960 m = m_get2(pktlen, M_WAITOK, MT_DATA, M_PKTHDR); 961 if (m == NULL) 962 return (ENOMEM); 963 964 m->m_len = m->m_pkthdr.len = pktlen; 965 m_copyback(m, 0, m->m_len, pkt); 966 967 /* Now prepend IP/UDP headers and transmit the mbuf. */ 968 NET_EPOCH_ENTER(et); 969 ret = ovpn_encap(sc, peerid, m); 970 NET_EPOCH_EXIT(et); 971 if (ret == 0) 972 OVPN_COUNTER_ADD(sc, sent_ctrl_pkts, 1); 973 else 974 OVPN_COUNTER_ADD(sc, lost_ctrl_pkts_out, 1); 975 976 return (ret); 977 } 978 979 static void 980 ovpn_send_ping(void *arg) 981 { 982 static const uint8_t ping_str[] = { 983 0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb, 984 0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48 985 }; 986 987 struct epoch_tracker et; 988 struct ovpn_kpeer *peer = arg; 989 struct ovpn_softc *sc = peer->sc; 990 struct mbuf *m; 991 992 OVPN_RASSERT(sc); 993 994 /* Ensure we repeat! */ 995 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 996 ovpn_send_ping, peer); 997 998 m = m_get2(sizeof(ping_str), M_NOWAIT, MT_DATA, M_PKTHDR); 999 if (m == NULL) 1000 return; 1001 1002 m_copyback(m, 0, sizeof(ping_str), ping_str); 1003 m->m_len = m->m_pkthdr.len = sizeof(ping_str); 1004 1005 CURVNET_SET(sc->ifp->if_vnet); 1006 NET_EPOCH_ENTER(et); 1007 (void)ovpn_transmit_to_peer(sc->ifp, m, peer, NULL); 1008 NET_EPOCH_EXIT(et); 1009 CURVNET_RESTORE(); 1010 } 1011 1012 static void 1013 ovpn_timeout(void *arg) 1014 { 1015 struct ovpn_kpeer *peer = arg; 1016 struct ovpn_softc *sc = peer->sc; 1017 uint32_t last, _last_active; 1018 int ret __diagused; 1019 int cpu; 1020 1021 OVPN_WASSERT(sc); 1022 1023 last = 0; 1024 CPU_FOREACH(cpu) { 1025 _last_active = *zpcpu_get_cpu(peer->last_active, cpu); 1026 if (_last_active > last) 1027 last = _last_active; 1028 } 1029 1030 if (last + peer->keepalive.timeout > time_uptime) { 1031 callout_reset(&peer->ping_rcv, 1032 (peer->keepalive.timeout - (time_uptime - last)) * hz, 1033 ovpn_timeout, peer); 1034 return; 1035 } 1036 1037 CURVNET_SET(sc->ifp->if_vnet); 1038 ret = _ovpn_del_peer(sc, peer->peerid); 1039 MPASS(ret == 0); 1040 CURVNET_RESTORE(); 1041 } 1042 1043 static int 1044 ovpn_set_peer(struct ifnet *ifp, const nvlist_t *nvl) 1045 { 1046 struct ovpn_softc *sc = ifp->if_softc; 1047 struct ovpn_kpeer *peer; 1048 1049 if (nvl == NULL) 1050 return (EINVAL); 1051 1052 if (! nvlist_exists_number(nvl, "interval") || 1053 ! nvlist_exists_number(nvl, "timeout") || 1054 ! nvlist_exists_number(nvl, "peerid")) 1055 return (EINVAL); 1056 1057 OVPN_WLOCK(sc); 1058 1059 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 1060 if (peer == NULL) { 1061 OVPN_WUNLOCK(sc); 1062 return (ENOENT); 1063 } 1064 1065 peer->keepalive.interval = nvlist_get_number(nvl, "interval"); 1066 peer->keepalive.timeout = nvlist_get_number(nvl, "timeout"); 1067 1068 if (peer->keepalive.interval > 0) 1069 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 1070 ovpn_send_ping, peer); 1071 if (peer->keepalive.timeout > 0) 1072 callout_reset(&peer->ping_rcv, peer->keepalive.timeout * hz, 1073 ovpn_timeout, peer); 1074 1075 OVPN_WUNLOCK(sc); 1076 1077 return (0); 1078 } 1079 1080 static int 1081 ovpn_ioctl_set(struct ifnet *ifp, struct ifdrv *ifd) 1082 { 1083 struct ovpn_softc *sc = ifp->if_softc; 1084 uint8_t *buf = NULL; 1085 nvlist_t *nvl = NULL; 1086 int ret; 1087 1088 if (ifd->ifd_len != 0) { 1089 if (ifd->ifd_len > OVPN_MAX_REQUEST_SIZE) 1090 return (E2BIG); 1091 1092 buf = malloc(ifd->ifd_len, M_OVPN, M_WAITOK); 1093 1094 ret = copyin(ifd->ifd_data, buf, ifd->ifd_len); 1095 if (ret != 0) { 1096 free(buf, M_OVPN); 1097 return (ret); 1098 } 1099 1100 nvl = nvlist_unpack(buf, ifd->ifd_len, 0); 1101 free(buf, M_OVPN); 1102 if (nvl == NULL) { 1103 return (EINVAL); 1104 } 1105 } 1106 1107 switch (ifd->ifd_cmd) { 1108 case OVPN_NEW_PEER: 1109 ret = ovpn_new_peer(ifp, nvl); 1110 break; 1111 case OVPN_DEL_PEER: 1112 OVPN_WLOCK(sc); 1113 ret = ovpn_del_peer(ifp, nvl); 1114 OVPN_WUNLOCK(sc); 1115 break; 1116 case OVPN_NEW_KEY: 1117 ret = ovpn_set_key(ifp, nvl); 1118 break; 1119 case OVPN_START_VPN: 1120 ret = ovpn_start(ifp); 1121 break; 1122 case OVPN_SWAP_KEYS: 1123 ret = ovpn_swap_keys(ifp, nvl); 1124 break; 1125 case OVPN_DEL_KEY: 1126 ret = ovpn_del_key(ifp, nvl); 1127 break; 1128 case OVPN_SEND_PKT: 1129 ret = ovpn_send_pkt(ifp, nvl); 1130 break; 1131 case OVPN_SET_PEER: 1132 ret = ovpn_set_peer(ifp, nvl); 1133 break; 1134 default: 1135 ret = ENOTSUP; 1136 } 1137 1138 nvlist_destroy(nvl); 1139 return (ret); 1140 } 1141 1142 static int 1143 ovpn_add_counters(nvlist_t *parent, const char *name, counter_u64_t in, 1144 counter_u64_t out) 1145 { 1146 nvlist_t *nvl; 1147 1148 nvl = nvlist_create(0); 1149 if (nvl == NULL) 1150 return (ENOMEM); 1151 1152 nvlist_add_number(nvl, "in", counter_u64_fetch(in)); 1153 nvlist_add_number(nvl, "out", counter_u64_fetch(out)); 1154 1155 nvlist_add_nvlist(parent, name, nvl); 1156 1157 nvlist_destroy(nvl); 1158 1159 return (0); 1160 } 1161 1162 static int 1163 ovpn_get_stats(struct ovpn_softc *sc, nvlist_t **onvl) 1164 { 1165 nvlist_t *nvl; 1166 int ret; 1167 1168 nvl = nvlist_create(0); 1169 if (nvl == NULL) 1170 return (ENOMEM); 1171 1172 #define OVPN_COUNTER_OUT(name, in, out) \ 1173 do { \ 1174 ret = ovpn_add_counters(nvl, name, \ 1175 sc->counters[offsetof(struct ovpn_counters, in) / \ 1176 sizeof(uint64_t)], \ 1177 sc->counters[offsetof(struct ovpn_counters, out) / \ 1178 sizeof(uint64_t)]); \ 1179 if (ret != 0) \ 1180 goto error; \ 1181 } while(0) 1182 1183 OVPN_COUNTER_OUT("lost_ctrl", lost_ctrl_pkts_in, lost_ctrl_pkts_out); 1184 OVPN_COUNTER_OUT("lost_data", lost_data_pkts_in, lost_data_pkts_out); 1185 OVPN_COUNTER_OUT("nomem_data", nomem_data_pkts_in, 1186 nomem_data_pkts_out); 1187 OVPN_COUNTER_OUT("data", received_data_pkts, sent_data_pkts); 1188 OVPN_COUNTER_OUT("ctrl", received_ctrl_pkts, sent_ctrl_pkts); 1189 OVPN_COUNTER_OUT("tunnel", tunnel_bytes_received, 1190 tunnel_bytes_received); 1191 OVPN_COUNTER_OUT("transport", transport_bytes_received, 1192 transport_bytes_received); 1193 #undef OVPN_COUNTER_OUT 1194 1195 *onvl = nvl; 1196 1197 return (0); 1198 1199 error: 1200 nvlist_destroy(nvl); 1201 return (ret); 1202 } 1203 1204 static int 1205 ovpn_poll_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1206 { 1207 nvlist_t *nvl; 1208 1209 nvl = nvlist_create(0); 1210 if (nvl == NULL) 1211 return (ENOMEM); 1212 1213 nvlist_add_number(nvl, "pending", 1214 buf_ring_count(sc->rxring) + buf_ring_count(sc->notifring)); 1215 1216 *onvl = nvl; 1217 1218 return (0); 1219 } 1220 1221 static int 1222 opvn_get_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1223 { 1224 struct ovpn_notification *n; 1225 struct ovpn_wire_header *ohdr; 1226 struct mbuf *m; 1227 uint8_t *buf; 1228 nvlist_t *nvl; 1229 uint32_t peerid; 1230 u_int mlength; 1231 1232 /* Check if we have notifications pending. */ 1233 n = buf_ring_dequeue_mc(sc->notifring); 1234 if (n != NULL) { 1235 nvl = nvlist_create(0); 1236 if (nvl == NULL) { 1237 free(n, M_OVPN); 1238 return (ENOMEM); 1239 } 1240 nvlist_add_number(nvl, "peerid", n->peerid); 1241 nvlist_add_number(nvl, "notification", n->type); 1242 free(n, M_OVPN); 1243 1244 *onvl = nvl; 1245 1246 return (0); 1247 } 1248 1249 /* Queued packet. */ 1250 m = buf_ring_dequeue_mc(sc->rxring); 1251 if (m == NULL) 1252 return (ENOENT); 1253 1254 mlength = m_length(m, NULL); 1255 buf = malloc(mlength, M_NVLIST, M_WAITOK); 1256 m_copydata(m, 0, mlength, buf); 1257 ohdr = (struct ovpn_wire_header *)buf; 1258 peerid = ntohl(ohdr->opcode) & 0x00ffffff; 1259 1260 nvl = nvlist_create(0); 1261 if (nvl == NULL) { 1262 OVPN_COUNTER_ADD(sc, lost_ctrl_pkts_in, 1); 1263 m_freem(m); 1264 free(buf, M_NVLIST); 1265 return (ENOMEM); 1266 } 1267 1268 nvlist_move_binary(nvl, "packet", buf, mlength); 1269 buf = NULL; 1270 nvlist_add_number(nvl, "peerid", peerid); 1271 1272 *onvl = nvl; 1273 1274 m_freem(m); 1275 1276 return (0); 1277 } 1278 1279 static int 1280 ovpn_ioctl_get(struct ifnet *ifp, struct ifdrv *ifd) 1281 { 1282 struct ovpn_softc *sc = ifp->if_softc; 1283 nvlist_t *nvl = NULL; 1284 int error; 1285 1286 switch (ifd->ifd_cmd) { 1287 case OVPN_GET_STATS: 1288 error = ovpn_get_stats(sc, &nvl); 1289 break; 1290 case OVPN_POLL_PKT: 1291 error = ovpn_poll_pkt(sc, &nvl); 1292 break; 1293 case OVPN_GET_PKT: 1294 error = opvn_get_pkt(sc, &nvl); 1295 break; 1296 default: 1297 error = ENOTSUP; 1298 break; 1299 } 1300 1301 if (error == 0) { 1302 void *packed = NULL; 1303 size_t len; 1304 1305 MPASS(nvl != NULL); 1306 1307 packed = nvlist_pack(nvl, &len); 1308 if (! packed) { 1309 nvlist_destroy(nvl); 1310 return (ENOMEM); 1311 } 1312 1313 if (len > ifd->ifd_len) { 1314 free(packed, M_NVLIST); 1315 nvlist_destroy(nvl); 1316 return (ENOSPC); 1317 } 1318 1319 error = copyout(packed, ifd->ifd_data, len); 1320 ifd->ifd_len = len; 1321 1322 free(packed, M_NVLIST); 1323 nvlist_destroy(nvl); 1324 } 1325 1326 return (error); 1327 } 1328 1329 static int 1330 ovpn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 1331 { 1332 struct ifdrv *ifd; 1333 int error; 1334 1335 switch (cmd) { 1336 case SIOCSDRVSPEC: 1337 case SIOCGDRVSPEC: 1338 error = priv_check(curthread, PRIV_NET_OVPN); 1339 if (error) 1340 return (error); 1341 break; 1342 } 1343 1344 switch (cmd) { 1345 case SIOCSDRVSPEC: 1346 ifd = (struct ifdrv *)data; 1347 error = ovpn_ioctl_set(ifp, ifd); 1348 break; 1349 case SIOCGDRVSPEC: 1350 ifd = (struct ifdrv *)data; 1351 error = ovpn_ioctl_get(ifp, ifd); 1352 break; 1353 case SIOCSIFMTU: { 1354 struct ifreq *ifr = (struct ifreq *)data; 1355 if (ifr->ifr_mtu < OVPN_MTU_MIN || ifr->ifr_mtu > OVPN_MTU_MAX) 1356 return (EINVAL); 1357 1358 ifp->if_mtu = ifr->ifr_mtu; 1359 return (0); 1360 } 1361 case SIOCSIFADDR: 1362 case SIOCADDMULTI: 1363 case SIOCDELMULTI: 1364 case SIOCGIFMTU: 1365 case SIOCSIFFLAGS: 1366 return (0); 1367 default: 1368 error = EINVAL; 1369 } 1370 1371 return (error); 1372 } 1373 1374 static int 1375 ovpn_encrypt_tx_cb(struct cryptop *crp) 1376 { 1377 struct ovpn_kpeer *peer = crp->crp_opaque; 1378 struct ovpn_softc *sc = peer->sc; 1379 struct mbuf *m = crp->crp_buf.cb_mbuf; 1380 int ret; 1381 1382 if (crp->crp_etype != 0) { 1383 crypto_freereq(crp); 1384 ovpn_peer_release_ref(peer, false); 1385 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1386 m_freem(m); 1387 return (0); 1388 } 1389 1390 CURVNET_SET(sc->ifp->if_vnet); 1391 1392 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1393 1394 ret = ovpn_encap(sc, peer->peerid, m); 1395 if (ret == 0) { 1396 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1397 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, m->m_pkthdr.len - 1398 sizeof(struct ovpn_wire_header)); 1399 } 1400 1401 CURVNET_RESTORE(); 1402 1403 crypto_freereq(crp); 1404 ovpn_peer_release_ref(peer, false); 1405 1406 return (0); 1407 } 1408 1409 static void 1410 ovpn_finish_rx(struct ovpn_softc *sc, struct mbuf *m, 1411 struct ovpn_kpeer *peer, struct ovpn_kkey *key, uint32_t seq, 1412 struct rm_priotracker *_ovpn_lock_trackerp) 1413 { 1414 uint32_t af; 1415 int ret __diagused; 1416 1417 OVPN_RASSERT(sc); 1418 1419 /* Replay protection. */ 1420 if (V_replay_protection && ! ovpn_check_replay(key->decrypt, seq)) { 1421 OVPN_RUNLOCK(sc); 1422 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1423 m_freem(m); 1424 return; 1425 } 1426 1427 critical_enter(); 1428 *zpcpu_get(peer->last_active) = time_uptime; 1429 critical_exit(); 1430 1431 OVPN_RUNLOCK(sc); 1432 1433 OVPN_COUNTER_ADD(sc, received_data_pkts, 1); 1434 OVPN_COUNTER_ADD(sc, tunnel_bytes_received, m->m_pkthdr.len); 1435 1436 /* Receive the packet on our interface. */ 1437 m->m_pkthdr.rcvif = sc->ifp; 1438 1439 /* Clear checksum flags in case the real hardware set them. */ 1440 m->m_pkthdr.csum_flags = 0; 1441 1442 /* Ensure we can read the first byte. */ 1443 m = m_pullup(m, 1); 1444 if (m == NULL) { 1445 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 1446 return; 1447 } 1448 1449 /* 1450 * Check for address family, and disregard any control packets (e.g. 1451 * keepalive). 1452 */ 1453 af = ovpn_get_af(m); 1454 if (af != 0) { 1455 BPF_MTAP2(sc->ifp, &af, sizeof(af), m); 1456 ret = netisr_dispatch(af == AF_INET ? NETISR_IP : NETISR_IPV6, 1457 m); 1458 MPASS(ret == 0); 1459 } else { 1460 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1461 m_freem(m); 1462 } 1463 } 1464 1465 static struct ovpn_kkey * 1466 ovpn_find_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, 1467 const struct ovpn_wire_header *ohdr) 1468 { 1469 struct ovpn_kkey *key = NULL; 1470 uint8_t keyid; 1471 1472 OVPN_RASSERT(sc); 1473 1474 keyid = (ntohl(ohdr->opcode) >> 24) & 0x07; 1475 1476 if (peer->keys[0].keyid == keyid) 1477 key = &peer->keys[0]; 1478 else if (peer->keys[1].keyid == keyid) 1479 key = &peer->keys[1]; 1480 1481 return (key); 1482 } 1483 1484 static int 1485 ovpn_decrypt_rx_cb(struct cryptop *crp) 1486 { 1487 struct ovpn_softc *sc = crp->crp_opaque; 1488 struct mbuf *m = crp->crp_buf.cb_mbuf; 1489 struct ovpn_kkey *key; 1490 struct ovpn_kpeer *peer; 1491 struct ovpn_wire_header *ohdr; 1492 uint32_t peerid; 1493 1494 OVPN_RLOCK_TRACKER; 1495 1496 OVPN_RLOCK(sc); 1497 1498 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1499 1500 if (crp->crp_etype != 0) { 1501 crypto_freereq(crp); 1502 atomic_add_int(&sc->refcount, -1); 1503 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1504 OVPN_RUNLOCK(sc); 1505 m_freem(m); 1506 return (0); 1507 } 1508 1509 CURVNET_SET(sc->ifp->if_vnet); 1510 1511 ohdr = mtodo(m, sizeof(struct udphdr)); 1512 1513 peerid = ntohl(ohdr->opcode) & 0x00ffffff; 1514 peer = ovpn_find_peer(sc, peerid); 1515 if (peer == NULL) { 1516 /* No such peer. Drop packet. */ 1517 crypto_freereq(crp); 1518 atomic_add_int(&sc->refcount, -1); 1519 OVPN_RUNLOCK(sc); 1520 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1521 m_freem(m); 1522 CURVNET_RESTORE(); 1523 return (0); 1524 } 1525 1526 key = ovpn_find_key(sc, peer, ohdr); 1527 if (key == NULL) { 1528 crypto_freereq(crp); 1529 atomic_add_int(&sc->refcount, -1); 1530 /* 1531 * Has this key been removed between us starting the decrypt 1532 * and finishing it? 1533 */ 1534 OVPN_RUNLOCK(sc); 1535 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1536 m_freem(m); 1537 CURVNET_RESTORE(); 1538 return (0); 1539 } 1540 1541 /* Now remove the outer headers */ 1542 m_adj_decap(m, sizeof(struct udphdr) + 1543 sizeof(struct ovpn_wire_header)); 1544 1545 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), _ovpn_lock_trackerp); 1546 OVPN_UNLOCK_ASSERT(sc); 1547 1548 CURVNET_RESTORE(); 1549 1550 crypto_freereq(crp); 1551 atomic_add_int(&sc->refcount, -1); 1552 1553 return (0); 1554 } 1555 1556 static uint8_t EMPTY_BUFFER[AES_BLOCK_LEN]; 1557 1558 static int 1559 ovpn_get_af(struct mbuf *m) 1560 { 1561 struct ip *ip; 1562 struct ip6_hdr *ip6; 1563 1564 /* 1565 * We should pullup, but we're only interested in the first byte, so 1566 * that'll always be contiguous. 1567 */ 1568 ip = mtod(m, struct ip *); 1569 if (ip->ip_v == IPVERSION) 1570 return (AF_INET); 1571 1572 ip6 = mtod(m, struct ip6_hdr *); 1573 if (ip6->ip6_vfc == IPV6_VERSION) 1574 return (AF_INET6); 1575 1576 return (0); 1577 } 1578 1579 static struct ovpn_kpeer * 1580 ovpn_find_peer_by_ip(struct ovpn_softc *sc, const struct in_addr addr) 1581 { 1582 struct ovpn_kpeer *peer = NULL; 1583 1584 OVPN_ASSERT(sc); 1585 1586 for (int i = 0; i < OVPN_MAX_PEERS; i++) { 1587 if (sc->peers[i] == NULL) 1588 continue; 1589 if (addr.s_addr == sc->peers[i]->vpn4.s_addr) { 1590 peer = sc->peers[i]; 1591 break; 1592 } 1593 } 1594 1595 return (peer); 1596 } 1597 1598 static struct ovpn_kpeer * 1599 ovpn_find_peer_by_ip6(struct ovpn_softc *sc, const struct in6_addr *addr) 1600 { 1601 struct ovpn_kpeer *peer = NULL; 1602 1603 OVPN_ASSERT(sc); 1604 1605 for (int i = 0; i < OVPN_MAX_PEERS; i++) { 1606 if (sc->peers[i] == NULL) 1607 continue; 1608 if (memcmp(addr, &sc->peers[i]->vpn6, sizeof(*addr)) == 0) { 1609 peer = sc->peers[i]; 1610 break; 1611 } 1612 } 1613 1614 return (peer); 1615 } 1616 1617 static struct ovpn_kpeer * 1618 ovpn_route_peer(struct ovpn_softc *sc, struct mbuf **m0, 1619 const struct sockaddr *dst) 1620 { 1621 struct ovpn_kpeer *peer = NULL; 1622 int af; 1623 1624 NET_EPOCH_ASSERT(); 1625 OVPN_ASSERT(sc); 1626 1627 /* Shortcut if we're a client (or are a server and have only one client). */ 1628 if (sc->peercount == 1) 1629 return (ovpn_find_only_peer(sc)); 1630 1631 if (dst != NULL) 1632 af = dst->sa_family; 1633 else 1634 af = ovpn_get_af(*m0); 1635 1636 switch (af) { 1637 #ifdef INET 1638 case AF_INET: { 1639 const struct sockaddr_in *sa = (const struct sockaddr_in *)dst; 1640 struct nhop_object *nh; 1641 const struct in_addr *ip_dst; 1642 1643 if (sa != NULL) { 1644 ip_dst = &sa->sin_addr; 1645 } else { 1646 struct ip *ip; 1647 1648 *m0 = m_pullup(*m0, sizeof(struct ip)); 1649 if (*m0 == NULL) 1650 return (NULL); 1651 ip = mtod(*m0, struct ip *); 1652 ip_dst = &ip->ip_dst; 1653 } 1654 1655 peer = ovpn_find_peer_by_ip(sc, *ip_dst); 1656 SDT_PROBE2(if_ovpn, tx, route, ip4, ip_dst, peer); 1657 if (peer == NULL) { 1658 nh = fib4_lookup(M_GETFIB(*m0), *ip_dst, 0, 1659 NHR_NONE, 0); 1660 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1661 peer = ovpn_find_peer_by_ip(sc, 1662 nh->gw4_sa.sin_addr); 1663 SDT_PROBE2(if_ovpn, tx, route, ip4, 1664 &nh->gw4_sa.sin_addr, peer); 1665 } 1666 } 1667 break; 1668 } 1669 #endif 1670 #ifdef INET6 1671 case AF_INET6: { 1672 const struct sockaddr_in6 *sa6 = 1673 (const struct sockaddr_in6 *)dst; 1674 struct nhop_object *nh; 1675 const struct in6_addr *ip6_dst; 1676 1677 if (sa6 != NULL) { 1678 ip6_dst = &sa6->sin6_addr; 1679 } else { 1680 struct ip6_hdr *ip6; 1681 1682 *m0 = m_pullup(*m0, sizeof(struct ip6_hdr)); 1683 if (*m0 == NULL) 1684 return (NULL); 1685 ip6 = mtod(*m0, struct ip6_hdr *); 1686 ip6_dst = &ip6->ip6_dst; 1687 } 1688 1689 peer = ovpn_find_peer_by_ip6(sc, ip6_dst); 1690 SDT_PROBE2(if_ovpn, tx, route, ip6, ip6_dst, peer); 1691 if (peer == NULL) { 1692 nh = fib6_lookup(M_GETFIB(*m0), ip6_dst, 0, 1693 NHR_NONE, 0); 1694 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1695 peer = ovpn_find_peer_by_ip6(sc, 1696 &nh->gw6_sa.sin6_addr); 1697 SDT_PROBE2(if_ovpn, tx, route, ip6, 1698 &nh->gw6_sa.sin6_addr, peer); 1699 } 1700 } 1701 break; 1702 } 1703 #endif 1704 } 1705 1706 return (peer); 1707 } 1708 1709 static int 1710 ovpn_transmit(struct ifnet *ifp, struct mbuf *m) 1711 { 1712 return (ifp->if_output(ifp, m, NULL, NULL)); 1713 } 1714 1715 static int 1716 ovpn_transmit_to_peer(struct ifnet *ifp, struct mbuf *m, 1717 struct ovpn_kpeer *peer, struct rm_priotracker *_ovpn_lock_trackerp) 1718 { 1719 struct ovpn_wire_header *ohdr; 1720 struct ovpn_kkey *key; 1721 struct ovpn_softc *sc; 1722 struct cryptop *crp; 1723 uint32_t af, seq; 1724 size_t len, real_len, ovpn_hdr_len; 1725 int tunnel_len; 1726 int ret; 1727 1728 sc = ifp->if_softc; 1729 1730 OVPN_RASSERT(sc); 1731 1732 tunnel_len = m->m_pkthdr.len; 1733 1734 key = &peer->keys[OVPN_KEY_SLOT_PRIMARY]; 1735 if (key->encrypt == NULL) { 1736 if (_ovpn_lock_trackerp != NULL) 1737 OVPN_RUNLOCK(sc); 1738 m_freem(m); 1739 return (ENOLINK); 1740 } 1741 1742 af = ovpn_get_af(m); 1743 /* Don't capture control packets. */ 1744 if (af != 0) 1745 BPF_MTAP2(ifp, &af, sizeof(af), m); 1746 1747 real_len = len = m->m_pkthdr.len; 1748 MPASS(real_len <= ifp->if_mtu); 1749 1750 ovpn_hdr_len = sizeof(struct ovpn_wire_header); 1751 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) 1752 ovpn_hdr_len -= 16; /* No auth tag. */ 1753 else { 1754 /* Round up the len to a multiple of our block size. */ 1755 len = roundup2(real_len, AES_BLOCK_LEN); 1756 1757 /* Now extend the mbuf. */ 1758 m_append(m, len - real_len, EMPTY_BUFFER); 1759 } 1760 1761 M_PREPEND(m, ovpn_hdr_len, M_NOWAIT); 1762 if (m == NULL) { 1763 if (_ovpn_lock_trackerp != NULL) 1764 OVPN_RUNLOCK(sc); 1765 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1766 return (ENOBUFS); 1767 } 1768 ohdr = mtod(m, struct ovpn_wire_header *); 1769 ohdr->opcode = (OVPN_OP_DATA_V2 << OVPN_OP_SHIFT) | key->keyid; 1770 ohdr->opcode <<= 24; 1771 ohdr->opcode |= key->peerid; 1772 ohdr->opcode = htonl(ohdr->opcode); 1773 1774 seq = atomic_fetchadd_32(&peer->tx_seq, 1); 1775 seq = htonl(seq); 1776 ohdr->seq = seq; 1777 1778 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) { 1779 ret = ovpn_encap(sc, peer->peerid, m); 1780 if (_ovpn_lock_trackerp != NULL) 1781 OVPN_RUNLOCK(sc); 1782 if (ret == 0) { 1783 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1784 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 1785 } 1786 return (ret); 1787 } 1788 1789 crp = crypto_getreq(key->encrypt->cryptoid, M_NOWAIT); 1790 if (crp == NULL) { 1791 if (_ovpn_lock_trackerp != NULL) 1792 OVPN_RUNLOCK(sc); 1793 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1794 m_freem(m); 1795 return (ENOBUFS); 1796 } 1797 1798 /* Encryption covers only the payload, not the header. */ 1799 crp->crp_payload_start = sizeof(*ohdr); 1800 crp->crp_payload_length = len; 1801 crp->crp_op = CRYPTO_OP_ENCRYPT; 1802 1803 /* 1804 * AAD data covers the ovpn_wire_header minus the auth 1805 * tag. 1806 */ 1807 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 1808 crp->crp_aad = ohdr; 1809 crp->crp_aad_start = 0; 1810 crp->crp_op |= CRYPTO_OP_COMPUTE_DIGEST; 1811 crp->crp_digest_start = offsetof(struct ovpn_wire_header, auth_tag); 1812 1813 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 1814 memcpy(crp->crp_iv, &seq, sizeof(seq)); 1815 memcpy(crp->crp_iv + sizeof(seq), key->encrypt->nonce, 1816 key->encrypt->noncelen); 1817 1818 crypto_use_mbuf(crp, m); 1819 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 1820 crp->crp_callback = ovpn_encrypt_tx_cb; 1821 crp->crp_opaque = peer; 1822 1823 atomic_add_int(&peer->refcount, 1); 1824 if (_ovpn_lock_trackerp != NULL) 1825 OVPN_RUNLOCK(sc); 1826 ret = crypto_dispatch(crp); 1827 if (ret) { 1828 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1829 } 1830 1831 return (ret); 1832 } 1833 1834 /* 1835 * Note: Expects to hold the read lock on entry, and will release it itself. 1836 */ 1837 static int 1838 ovpn_encap(struct ovpn_softc *sc, uint32_t peerid, struct mbuf *m) 1839 { 1840 struct udphdr *udp; 1841 struct ovpn_kpeer *peer; 1842 int len; 1843 1844 OVPN_RLOCK_TRACKER; 1845 1846 OVPN_RLOCK(sc); 1847 NET_EPOCH_ASSERT(); 1848 1849 peer = ovpn_find_peer(sc, peerid); 1850 if (peer == NULL || sc->ifp->if_link_state != LINK_STATE_UP) { 1851 OVPN_RUNLOCK(sc); 1852 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1853 m_freem(m); 1854 return (ENETDOWN); 1855 } 1856 1857 len = m->m_pkthdr.len; 1858 1859 M_PREPEND(m, sizeof(struct udphdr), M_NOWAIT); 1860 if (m == NULL) { 1861 OVPN_RUNLOCK(sc); 1862 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1863 m_freem(m); 1864 return (ENOBUFS); 1865 } 1866 udp = mtod(m, struct udphdr *); 1867 1868 MPASS(peer->local.ss_family == peer->remote.ss_family); 1869 1870 udp->uh_sport = ovpn_get_port(&peer->local); 1871 udp->uh_dport = ovpn_get_port(&peer->remote); 1872 udp->uh_ulen = htons(sizeof(struct udphdr) + len); 1873 1874 switch (peer->remote.ss_family) { 1875 #ifdef INET 1876 case AF_INET: { 1877 struct sockaddr_in *in_local = TO_IN(&peer->local); 1878 struct sockaddr_in *in_remote = TO_IN(&peer->remote); 1879 struct ip *ip; 1880 1881 /* 1882 * This requires knowing the source IP, which we don't. Happily 1883 * we're allowed to keep this at 0, and the checksum won't do 1884 * anything the crypto won't already do. 1885 */ 1886 udp->uh_sum = 0; 1887 1888 /* Set the checksum flags so we recalculate checksums. */ 1889 m->m_pkthdr.csum_flags |= CSUM_IP; 1890 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 1891 1892 M_PREPEND(m, sizeof(struct ip), M_NOWAIT); 1893 if (m == NULL) { 1894 OVPN_RUNLOCK(sc); 1895 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1896 return (ENOBUFS); 1897 } 1898 ip = mtod(m, struct ip *); 1899 1900 ip->ip_tos = 0; 1901 ip->ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) + 1902 len); 1903 ip->ip_off = 0; 1904 ip->ip_ttl = V_ip_defttl; 1905 ip->ip_p = IPPROTO_UDP; 1906 ip->ip_sum = 0; 1907 if (in_local->sin_port != 0) 1908 ip->ip_src = in_local->sin_addr; 1909 else 1910 ip->ip_src.s_addr = INADDR_ANY; 1911 ip->ip_dst = in_remote->sin_addr; 1912 1913 OVPN_RUNLOCK(sc); 1914 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 1915 1916 return (ip_output(m, NULL, NULL, 0, NULL, NULL)); 1917 } 1918 #endif 1919 #ifdef INET6 1920 case AF_INET6: { 1921 struct sockaddr_in6 *in6_local = TO_IN6(&peer->local); 1922 struct sockaddr_in6 *in6_remote = TO_IN6(&peer->remote); 1923 struct ip6_hdr *ip6; 1924 1925 M_PREPEND(m, sizeof(struct ip6_hdr), M_NOWAIT); 1926 if (m == NULL) { 1927 OVPN_RUNLOCK(sc); 1928 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1929 return (ENOBUFS); 1930 } 1931 m = m_pullup(m, sizeof(*ip6) + sizeof(*udp)); 1932 if (m == NULL) { 1933 OVPN_RUNLOCK(sc); 1934 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1935 return (ENOBUFS); 1936 } 1937 1938 ip6 = mtod(m, struct ip6_hdr *); 1939 1940 ip6->ip6_vfc = IPV6_VERSION; 1941 ip6->ip6_flow &= ~IPV6_FLOWINFO_MASK; 1942 ip6->ip6_plen = htons(sizeof(*ip6) + sizeof(struct udphdr) + 1943 len); 1944 ip6->ip6_nxt = IPPROTO_UDP; 1945 ip6->ip6_hlim = V_ip6_defhlim; 1946 1947 memcpy(&ip6->ip6_src, &in6_local->sin6_addr, 1948 sizeof(ip6->ip6_src)); 1949 memcpy(&ip6->ip6_dst, &in6_remote->sin6_addr, 1950 sizeof(ip6->ip6_dst)); 1951 1952 udp = mtodo(m, sizeof(*ip6)); 1953 udp->uh_sum = in6_cksum_pseudo(ip6, 1954 m->m_pkthdr.len - sizeof(struct ip6_hdr), 1955 IPPROTO_UDP, 0); 1956 1957 m->m_pkthdr.csum_flags |= CSUM_UDP_IPV6; 1958 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 1959 1960 OVPN_RUNLOCK(sc); 1961 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 1962 1963 return (ip6_output(m, NULL, NULL, IPV6_UNSPECSRC, NULL, NULL, 1964 NULL)); 1965 } 1966 #endif 1967 default: 1968 panic("Unsupported address family %d", 1969 peer->remote.ss_family); 1970 } 1971 } 1972 1973 static int 1974 ovpn_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 1975 struct route *ro) 1976 { 1977 struct ovpn_softc *sc; 1978 struct ovpn_kpeer *peer; 1979 1980 OVPN_RLOCK_TRACKER; 1981 1982 sc = ifp->if_softc; 1983 1984 OVPN_RLOCK(sc); 1985 1986 SDT_PROBE1(if_ovpn, tx, transmit, start, m); 1987 1988 if (__predict_false(ifp->if_link_state != LINK_STATE_UP)) { 1989 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1990 OVPN_RUNLOCK(sc); 1991 m_freem(m); 1992 return (ENETDOWN); 1993 } 1994 1995 /** 1996 * Only obey 'dst' (i.e. the gateway) if no route is supplied. 1997 * That's our indication that we're being called through pf's route-to, 1998 * and we should route according to 'dst' instead. We can't do so 1999 * consistently, because the usual openvpn configuration sets the first 2000 * non-server IP in the subnet as the gateway. If we always use that 2001 * one we'd end up routing all traffic to the first client. 2002 * tl;dr: 'ro == NULL' tells us pf is doing a route-to, and then but 2003 * only then, we should treat 'dst' as the destination. */ 2004 peer = ovpn_route_peer(sc, &m, ro == NULL ? dst : NULL); 2005 if (peer == NULL) { 2006 /* No destination. */ 2007 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2008 OVPN_RUNLOCK(sc); 2009 m_freem(m); 2010 return (ENETDOWN); 2011 } 2012 2013 return (ovpn_transmit_to_peer(ifp, m, peer, _ovpn_lock_trackerp)); 2014 } 2015 2016 static void 2017 ovpn_rcv_ctrl(struct ovpn_softc *sc, struct mbuf *m, int off) 2018 { 2019 /* Lop off the IP and UDP headers */ 2020 m_adj_decap(m, off); 2021 2022 /* Keep in the local ring until userspace fetches it. */ 2023 if (buf_ring_enqueue(sc->rxring, m) != 0) { 2024 OVPN_COUNTER_ADD(sc, lost_ctrl_pkts_in, 1); 2025 m_freem(m); 2026 return; 2027 } 2028 2029 OVPN_COUNTER_ADD(sc, received_ctrl_pkts, 1); 2030 } 2031 2032 static bool 2033 ovpn_check_replay(struct ovpn_kkey_dir *key, uint32_t seq) 2034 { 2035 uint32_t d; 2036 2037 mtx_lock(&key->replay_mtx); 2038 2039 /* Sequence number must be strictly greater than rx_seq */ 2040 if (seq <= key->rx_seq) { 2041 mtx_unlock(&key->replay_mtx); 2042 return (false); 2043 } 2044 2045 /* Large jump. The packet authenticated okay, so just accept that. */ 2046 if (seq > (key->rx_seq + (sizeof(key->rx_window) * 8))) { 2047 key->rx_seq = seq; 2048 key->rx_window = 0; 2049 mtx_unlock(&key->replay_mtx); 2050 return (true); 2051 } 2052 2053 /* Happy case. */ 2054 if ((seq == key->rx_seq + 1) && key->rx_window == 0) { 2055 key->rx_seq++; 2056 mtx_unlock(&key->replay_mtx); 2057 return (true); 2058 } 2059 2060 d = seq - key->rx_seq - 1; 2061 2062 if (key->rx_window & ((uint64_t)1 << d)) { 2063 /* Dupe! */ 2064 mtx_unlock(&key->replay_mtx); 2065 return (false); 2066 } 2067 2068 key->rx_window |= (uint64_t)1 << d; 2069 2070 while (key->rx_window & 1) { 2071 key->rx_seq++; 2072 key->rx_window >>= 1; 2073 } 2074 2075 mtx_unlock(&key->replay_mtx); 2076 2077 return (true); 2078 } 2079 2080 static struct ovpn_kpeer * 2081 ovpn_peer_from_mbuf(struct ovpn_softc *sc, struct mbuf *m, int off) 2082 { 2083 struct ovpn_wire_header ohdr; 2084 uint32_t peerid; 2085 const size_t hdrlen = sizeof(ohdr) - sizeof(ohdr.auth_tag); 2086 2087 OVPN_RASSERT(sc); 2088 2089 if (m_length(m, NULL) < (off + sizeof(struct udphdr) + hdrlen)) 2090 return (NULL); 2091 2092 m_copydata(m, off + sizeof(struct udphdr), hdrlen, (caddr_t)&ohdr); 2093 2094 peerid = ntohl(ohdr.opcode) & 0x00ffffff; 2095 2096 return (ovpn_find_peer(sc, peerid)); 2097 } 2098 2099 static bool 2100 ovpn_udp_input(struct mbuf *m, int off, struct inpcb *inp, 2101 const struct sockaddr *sa, void *ctx) 2102 { 2103 struct ovpn_softc *sc = ctx; 2104 struct ovpn_wire_header *ohdr; 2105 struct udphdr *uhdr; 2106 struct ovpn_kkey *key; 2107 struct cryptop *crp; 2108 struct ovpn_kpeer *peer; 2109 size_t ohdrlen; 2110 int ret; 2111 uint8_t op; 2112 2113 OVPN_RLOCK_TRACKER; 2114 2115 M_ASSERTPKTHDR(m); 2116 2117 OVPN_COUNTER_ADD(sc, transport_bytes_received, m->m_pkthdr.len - off); 2118 2119 ohdrlen = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2120 2121 OVPN_RLOCK(sc); 2122 2123 peer = ovpn_peer_from_mbuf(sc, m, off); 2124 if (peer == NULL) { 2125 OVPN_RUNLOCK(sc); 2126 return (false); 2127 } 2128 2129 m = m_pullup(m, off + sizeof(*uhdr) + ohdrlen); 2130 if (m == NULL) { 2131 OVPN_RUNLOCK(sc); 2132 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2133 return (true); 2134 } 2135 uhdr = mtodo(m, off); 2136 ohdr = mtodo(m, off + sizeof(*uhdr)); 2137 2138 op = ntohl(ohdr->opcode) >> 24 >> OVPN_OP_SHIFT; 2139 2140 /* 2141 * Simplify things by getting rid of the preceding headers, we don't 2142 * care about them. 2143 */ 2144 m_adj_decap(m, off); 2145 2146 uhdr = mtodo(m, 0); 2147 ohdr = mtodo(m, sizeof(*uhdr)); 2148 2149 if (op != OVPN_OP_DATA_V2) { 2150 OVPN_RUNLOCK(sc); 2151 ovpn_rcv_ctrl(sc, m, sizeof(struct udphdr)); 2152 INP_WLOCK(inp); 2153 udp_notify(inp, EAGAIN); 2154 INP_WUNLOCK(inp); 2155 return (true); 2156 } 2157 2158 key = ovpn_find_key(sc, peer, ohdr); 2159 if (key == NULL || key->decrypt == NULL) { 2160 OVPN_RUNLOCK(sc); 2161 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2162 m_freem(m); 2163 return (true); 2164 } 2165 2166 if (key->decrypt->cipher == OVPN_CIPHER_ALG_NONE) { 2167 /* Now remove the outer headers */ 2168 m_adj_decap(m, sizeof(struct udphdr) + ohdrlen); 2169 2170 ohdr = mtodo(m, sizeof(*uhdr)); 2171 2172 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), 2173 _ovpn_lock_trackerp); 2174 OVPN_UNLOCK_ASSERT(sc); 2175 return (true); 2176 } 2177 2178 ohdrlen += sizeof(ohdr->auth_tag); 2179 2180 m = m_pullup(m, sizeof(*uhdr) + ohdrlen); 2181 if (m == NULL) { 2182 OVPN_RUNLOCK(sc); 2183 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2184 return (true); 2185 } 2186 uhdr = mtodo(m, 0); 2187 ohdr = mtodo(m, sizeof(*uhdr)); 2188 2189 /* Decrypt */ 2190 crp = crypto_getreq(key->decrypt->cryptoid, M_NOWAIT); 2191 if (crp == NULL) { 2192 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2193 OVPN_RUNLOCK(sc); 2194 m_freem(m); 2195 return (true); 2196 } 2197 2198 crp->crp_payload_start = sizeof(struct udphdr) + sizeof(*ohdr); 2199 crp->crp_payload_length = ntohs(uhdr->uh_ulen) - 2200 sizeof(*uhdr) - sizeof(*ohdr); 2201 crp->crp_op = CRYPTO_OP_DECRYPT; 2202 2203 /* AAD validation. */ 2204 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2205 crp->crp_aad = ohdr; 2206 crp->crp_aad_start = 0; 2207 crp->crp_op |= CRYPTO_OP_VERIFY_DIGEST; 2208 crp->crp_digest_start = sizeof(struct udphdr) + 2209 offsetof(struct ovpn_wire_header, auth_tag); 2210 2211 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 2212 memcpy(crp->crp_iv, &ohdr->seq, sizeof(ohdr->seq)); 2213 memcpy(crp->crp_iv + sizeof(ohdr->seq), key->decrypt->nonce, 2214 key->decrypt->noncelen); 2215 2216 crypto_use_mbuf(crp, m); 2217 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 2218 crp->crp_callback = ovpn_decrypt_rx_cb; 2219 crp->crp_opaque = sc; 2220 2221 atomic_add_int(&sc->refcount, 1); 2222 OVPN_RUNLOCK(sc); 2223 ret = crypto_dispatch(crp); 2224 if (ret != 0) { 2225 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2226 } 2227 2228 return (true); 2229 } 2230 2231 static void 2232 ovpn_qflush(struct ifnet *ifp __unused) 2233 { 2234 2235 } 2236 2237 static void 2238 ovpn_flush_rxring(struct ovpn_softc *sc) 2239 { 2240 struct mbuf *m; 2241 struct ovpn_notification *n; 2242 2243 OVPN_WASSERT(sc); 2244 2245 while (! buf_ring_empty(sc->rxring)) { 2246 m = buf_ring_dequeue_sc(sc->rxring); 2247 m_freem(m); 2248 } 2249 2250 while (! buf_ring_empty(sc->notifring)) { 2251 n = buf_ring_dequeue_sc(sc->notifring); 2252 free(n, M_OVPN); 2253 } 2254 } 2255 2256 #ifdef VIMAGE 2257 static void 2258 ovpn_reassign(struct ifnet *ifp, struct vnet *new_vnet __unused, 2259 char *unused __unused) 2260 { 2261 struct ovpn_softc *sc = ifp->if_softc; 2262 int i; 2263 int ret __diagused; 2264 2265 i = 0; 2266 2267 OVPN_WLOCK(sc); 2268 2269 /* Flush keys & configuration. */ 2270 do { 2271 if (sc->peers[i] != NULL) { 2272 ret = _ovpn_del_peer(sc, sc->peers[i]->peerid); 2273 MPASS(ret == 0); 2274 } 2275 i++; 2276 } while (i < OVPN_MAX_PEERS); 2277 2278 ovpn_flush_rxring(sc); 2279 2280 OVPN_WUNLOCK(sc); 2281 } 2282 #endif 2283 2284 static int 2285 ovpn_clone_match(struct if_clone *ifc, const char *name) 2286 { 2287 /* 2288 * Allow all names that start with 'ovpn', specifically because pfSense 2289 * uses ovpnc1 / ovpns2 2290 */ 2291 return (strncmp(ovpnname, name, strlen(ovpnname)) == 0); 2292 } 2293 2294 static int 2295 ovpn_clone_create(struct if_clone *ifc, char *name, size_t len, caddr_t params) 2296 { 2297 struct ovpn_softc *sc; 2298 struct ifnet *ifp; 2299 char *dp; 2300 int error, unit, wildcard; 2301 2302 /* Try to see if a special unit was requested. */ 2303 error = ifc_name2unit(name, &unit); 2304 if (error != 0) 2305 return (error); 2306 wildcard = (unit < 0); 2307 2308 error = ifc_alloc_unit(ifc, &unit); 2309 if (error != 0) 2310 return (error); 2311 2312 /* 2313 * If no unit had been given, we need to adjust the ifName. 2314 */ 2315 for (dp = name; *dp != '\0'; dp++); 2316 if (wildcard) { 2317 error = snprintf(dp, len - (dp - name), "%d", unit); 2318 if (error > len - (dp - name)) { 2319 /* ifName too long. */ 2320 ifc_free_unit(ifc, unit); 2321 return (ENOSPC); 2322 } 2323 dp += error; 2324 } 2325 2326 /* Make sure it doesn't already exist. */ 2327 if (ifunit(name) != NULL) 2328 return (EEXIST); 2329 2330 sc = malloc(sizeof(struct ovpn_softc), M_OVPN, M_WAITOK | M_ZERO); 2331 sc->ifp = if_alloc(IFT_ENC); 2332 rm_init_flags(&sc->lock, "if_ovpn_lock", RM_RECURSE); 2333 sc->refcount = 0; 2334 2335 sc->rxring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL); 2336 sc->notifring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL); 2337 2338 COUNTER_ARRAY_ALLOC(sc->counters, OVPN_COUNTER_SIZE, M_WAITOK); 2339 2340 ifp = sc->ifp; 2341 ifp->if_softc = sc; 2342 strlcpy(ifp->if_xname, name, IFNAMSIZ); 2343 ifp->if_dname = ovpngroupname; 2344 ifp->if_dunit = unit; 2345 2346 ifp->if_addrlen = 0; 2347 ifp->if_mtu = 1428; 2348 ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST; 2349 ifp->if_ioctl = ovpn_ioctl; 2350 ifp->if_transmit = ovpn_transmit; 2351 ifp->if_output = ovpn_output; 2352 ifp->if_qflush = ovpn_qflush; 2353 #ifdef VIMAGE 2354 ifp->if_reassign = ovpn_reassign; 2355 #endif 2356 ifp->if_capabilities |= IFCAP_LINKSTATE; 2357 ifp->if_capenable |= IFCAP_LINKSTATE; 2358 2359 if_attach(ifp); 2360 bpfattach(ifp, DLT_NULL, sizeof(uint32_t)); 2361 2362 return (0); 2363 } 2364 2365 static void 2366 ovpn_clone_destroy_cb(struct epoch_context *ctx) 2367 { 2368 struct ovpn_softc *sc; 2369 2370 sc = __containerof(ctx, struct ovpn_softc, epoch_ctx); 2371 2372 MPASS(sc->peercount == 0); 2373 for (int i = 0; i < OVPN_MAX_PEERS; i++) { 2374 MPASS(sc->peers[i] == NULL); 2375 } 2376 2377 COUNTER_ARRAY_FREE(sc->counters, OVPN_COUNTER_SIZE); 2378 2379 if_free(sc->ifp); 2380 free(sc, M_OVPN); 2381 } 2382 2383 static int 2384 ovpn_clone_destroy(struct if_clone *ifc, struct ifnet *ifp) 2385 { 2386 struct ovpn_softc *sc; 2387 int unit; 2388 int i; 2389 int ret __diagused; 2390 2391 sc = ifp->if_softc; 2392 unit = ifp->if_dunit; 2393 2394 OVPN_WLOCK(sc); 2395 2396 if (atomic_load_int(&sc->refcount) > 0) { 2397 OVPN_WUNLOCK(sc); 2398 return (EBUSY); 2399 } 2400 2401 i = 0; 2402 do { 2403 if (sc->peers[i] != NULL) { 2404 ret = _ovpn_del_peer(sc, sc->peers[i]->peerid); 2405 MPASS(ret == 0); 2406 } 2407 i++; 2408 } while (i < OVPN_MAX_PEERS); 2409 2410 ovpn_flush_rxring(sc); 2411 buf_ring_free(sc->rxring, M_OVPN); 2412 buf_ring_free(sc->notifring, M_OVPN); 2413 2414 OVPN_WUNLOCK(sc); 2415 2416 bpfdetach(ifp); 2417 if_detach(ifp); 2418 ifp->if_softc = NULL; 2419 2420 NET_EPOCH_CALL(ovpn_clone_destroy_cb, &sc->epoch_ctx); 2421 2422 if (unit != IF_DUNIT_NONE) 2423 ifc_free_unit(ifc, unit); 2424 2425 return (0); 2426 } 2427 2428 static void 2429 vnet_ovpn_init(const void *unused __unused) 2430 { 2431 V_ovpn_cloner = if_clone_advanced(ovpngroupname, 0, ovpn_clone_match, 2432 ovpn_clone_create, ovpn_clone_destroy); 2433 } 2434 VNET_SYSINIT(vnet_ovpn_init, SI_SUB_PSEUDO, SI_ORDER_ANY, 2435 vnet_ovpn_init, NULL); 2436 2437 static void 2438 vnet_ovpn_uninit(const void *unused __unused) 2439 { 2440 if_clone_detach(V_ovpn_cloner); 2441 } 2442 VNET_SYSUNINIT(vnet_ovpn_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY, 2443 vnet_ovpn_uninit, NULL); 2444 2445 static int 2446 ovpnmodevent(module_t mod, int type, void *data) 2447 { 2448 switch (type) { 2449 case MOD_LOAD: 2450 /* Done in vnet_ovpn_init() */ 2451 break; 2452 case MOD_UNLOAD: 2453 /* Done in vnet_ovpn_uninit() */ 2454 break; 2455 default: 2456 return (EOPNOTSUPP); 2457 } 2458 2459 return (0); 2460 } 2461 2462 static moduledata_t ovpn_mod = { 2463 "if_ovpn", 2464 ovpnmodevent, 2465 0 2466 }; 2467 2468 DECLARE_MODULE(if_ovpn, ovpn_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 2469 MODULE_VERSION(if_ovpn, 1); 2470