1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2021-2022 Rubicon Communications, LLC (Netgate) 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 */ 28 #include "opt_inet.h" 29 #include "opt_inet6.h" 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/buf_ring.h> 34 #include <sys/epoch.h> 35 #include <sys/file.h> 36 #include <sys/filedesc.h> 37 #include <sys/kernel.h> 38 #include <sys/malloc.h> 39 #include <sys/mbuf.h> 40 #include <sys/module.h> 41 #include <sys/nv.h> 42 #include <sys/priv.h> 43 #include <sys/protosw.h> 44 #include <sys/rmlock.h> 45 #include <sys/sdt.h> 46 #include <sys/smp.h> 47 #include <sys/socket.h> 48 #include <sys/socketvar.h> 49 #include <sys/sockio.h> 50 #include <sys/sysctl.h> 51 #include <sys/time.h> 52 53 #include <machine/atomic.h> 54 55 #include <net/bpf.h> 56 #include <net/if.h> 57 #include <net/if_clone.h> 58 #include <net/if_types.h> 59 #include <net/if_var.h> 60 #include <net/netisr.h> 61 #include <net/route/nhop.h> 62 63 #include <netinet/in.h> 64 #include <netinet/in_fib.h> 65 #include <netinet/ip.h> 66 #include <netinet/ip6.h> 67 #include <netinet/ip_var.h> 68 #include <netinet/udp.h> 69 #include <netinet/udp_var.h> 70 71 #include <netinet6/ip6_var.h> 72 #include <netinet6/in6_fib.h> 73 74 #include <machine/in_cksum.h> 75 76 #include <opencrypto/cryptodev.h> 77 78 #include "if_ovpn.h" 79 80 struct ovpn_kkey_dir { 81 int refcount; 82 uint8_t key[32]; 83 uint8_t keylen; 84 uint8_t nonce[8]; 85 uint8_t noncelen; 86 enum ovpn_key_cipher cipher; 87 crypto_session_t cryptoid; 88 89 struct mtx replay_mtx; 90 /* 91 * Last seen gapless sequence number. New rx seq numbers must be 92 * strictly higher than this. 93 */ 94 uint32_t rx_seq; 95 /* Seen packets, relative to rx_seq. bit(0) will always be 0. */ 96 uint64_t rx_window; 97 }; 98 99 struct ovpn_kkey { 100 struct ovpn_kkey_dir *encrypt; 101 struct ovpn_kkey_dir *decrypt; 102 uint8_t keyid; 103 uint32_t peerid; 104 }; 105 106 struct ovpn_keepalive { 107 uint32_t interval; 108 uint32_t timeout; 109 }; 110 111 struct ovpn_wire_header { 112 uint32_t opcode; /* opcode, key id, peer id */ 113 uint32_t seq; 114 uint8_t auth_tag[16]; 115 }; 116 117 struct ovpn_notification { 118 enum ovpn_notif_type type; 119 uint32_t peerid; 120 }; 121 122 struct ovpn_softc; 123 124 struct ovpn_kpeer { 125 int refcount; 126 uint32_t peerid; 127 128 struct ovpn_softc *sc; 129 struct sockaddr_storage local; 130 struct sockaddr_storage remote; 131 132 struct in_addr vpn4; 133 struct in6_addr vpn6; 134 135 struct ovpn_kkey keys[2]; 136 uint32_t tx_seq; 137 138 struct ovpn_keepalive keepalive; 139 uint32_t *last_active; 140 struct callout ping_send; 141 struct callout ping_rcv; 142 }; 143 144 #define OVPN_MAX_PEERS 128 145 146 struct ovpn_counters { 147 uint64_t lost_ctrl_pkts_in; 148 uint64_t lost_ctrl_pkts_out; 149 uint64_t lost_data_pkts_in; 150 uint64_t lost_data_pkts_out; 151 uint64_t nomem_data_pkts_in; 152 uint64_t nomem_data_pkts_out; 153 uint64_t received_ctrl_pkts; 154 uint64_t received_data_pkts; 155 uint64_t sent_ctrl_pkts; 156 uint64_t sent_data_pkts; 157 158 uint64_t transport_bytes_sent; 159 uint64_t transport_bytes_received; 160 uint64_t tunnel_bytes_sent; 161 uint64_t tunnel_bytes_received; 162 }; 163 #define OVPN_COUNTER_SIZE (sizeof(struct ovpn_counters)/sizeof(uint64_t)) 164 165 struct ovpn_softc { 166 int refcount; 167 struct rmlock lock; 168 struct ifnet *ifp; 169 struct socket *so; 170 int peercount; 171 struct ovpn_kpeer *peers[OVPN_MAX_PEERS]; /* XXX Hard limit for now? */ 172 173 /* Pending packets */ 174 struct buf_ring *rxring; 175 struct buf_ring *notifring; 176 177 counter_u64_t counters[OVPN_COUNTER_SIZE]; 178 179 struct epoch_context epoch_ctx; 180 }; 181 182 static struct ovpn_kpeer *ovpn_find_peer(struct ovpn_softc *, uint32_t); 183 static bool ovpn_udp_input(struct mbuf *, int, struct inpcb *, 184 const struct sockaddr *, void *); 185 static int ovpn_transmit_to_peer(struct ifnet *, struct mbuf *, 186 struct ovpn_kpeer *, struct rm_priotracker *); 187 static int ovpn_encap(struct ovpn_softc *, uint32_t, struct mbuf *); 188 static int ovpn_get_af(struct mbuf *); 189 static void ovpn_free_kkey_dir(struct ovpn_kkey_dir *); 190 static bool ovpn_check_replay(struct ovpn_kkey_dir *, uint32_t); 191 192 #define OVPN_MTU_MIN 576 193 #define OVPN_MTU_MAX (IP_MAXPACKET - sizeof(struct ip) - \ 194 sizeof(struct udphdr) - sizeof(struct ovpn_wire_header)) 195 196 #define OVPN_OP_DATA_V2 0x09 197 #define OVPN_OP_SHIFT 3 198 199 VNET_DEFINE_STATIC(struct if_clone *, ovpn_cloner); 200 #define V_ovpn_cloner VNET(ovpn_cloner) 201 202 #define OVPN_RLOCK_TRACKER struct rm_priotracker _ovpn_lock_tracker; \ 203 struct rm_priotracker *_ovpn_lock_trackerp = &_ovpn_lock_tracker 204 #define OVPN_RLOCK(sc) rm_rlock(&(sc)->lock, _ovpn_lock_trackerp) 205 #define OVPN_RUNLOCK(sc) rm_runlock(&(sc)->lock, _ovpn_lock_trackerp) 206 #define OVPN_WLOCK(sc) rm_wlock(&(sc)->lock) 207 #define OVPN_WUNLOCK(sc) rm_wunlock(&(sc)->lock) 208 #define OVPN_ASSERT(sc) rm_assert(&(sc)->lock, RA_LOCKED) 209 #define OVPN_RASSERT(sc) rm_assert(&(sc)->lock, RA_RLOCKED) 210 #define OVPN_WASSERT(sc) rm_assert(&(sc)->lock, RA_WLOCKED) 211 #define OVPN_UNLOCK_ASSERT(sc) rm_assert(&(sc)->lock, RA_UNLOCKED) 212 213 #define OVPN_COUNTER_ADD(sc, name, val) \ 214 counter_u64_add(sc->counters[offsetof(struct ovpn_counters, name) / \ 215 sizeof(uint64_t)], val) 216 217 #define TO_IN(x) ((struct sockaddr_in *)(x)) 218 #define TO_IN6(x) ((struct sockaddr_in6 *)(x)) 219 220 SDT_PROVIDER_DEFINE(if_ovpn); 221 SDT_PROBE_DEFINE1(if_ovpn, tx, transmit, start, "struct mbuf *"); 222 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip4, "struct in_addr *", "struct ovpn_kpeer *"); 223 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip6, "struct in6_addr *", "struct ovpn_kpeer *"); 224 225 static const char ovpnname[] = "ovpn"; 226 static const char ovpngroupname[] = "openvpn"; 227 228 static MALLOC_DEFINE(M_OVPN, ovpnname, "OpenVPN DCO Interface"); 229 230 SYSCTL_DECL(_net_link); 231 static SYSCTL_NODE(_net_link, IFT_OTHER, openvpn, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 232 "OpenVPN DCO Interface"); 233 VNET_DEFINE_STATIC(int, replay_protection) = 0; 234 #define V_replay_protection VNET(replay_protection) 235 SYSCTL_INT(_net_link_openvpn, OID_AUTO, replay_protection, CTLFLAG_VNET | CTLFLAG_RW, 236 &VNET_NAME(replay_protection), 0, "Validate sequence numbers"); 237 238 static struct ovpn_kpeer * 239 ovpn_find_peer(struct ovpn_softc *sc, uint32_t peerid) 240 { 241 struct ovpn_kpeer *p = NULL; 242 243 OVPN_ASSERT(sc); 244 245 for (int i = 0; i < OVPN_MAX_PEERS; i++) { 246 p = sc->peers[i]; 247 if (p == NULL) 248 continue; 249 250 if (p->peerid == peerid) { 251 MPASS(p->sc == sc); 252 break; 253 } 254 } 255 256 return (p); 257 } 258 259 static struct ovpn_kpeer * 260 ovpn_find_only_peer(struct ovpn_softc *sc) 261 { 262 OVPN_ASSERT(sc); 263 264 for (int i = 0; i < OVPN_MAX_PEERS; i++) { 265 if (sc->peers[i] == NULL) 266 continue; 267 return (sc->peers[i]); 268 } 269 270 MPASS(false); 271 272 return (NULL); 273 } 274 275 static uint16_t 276 ovpn_get_port(struct sockaddr_storage *s) 277 { 278 switch (s->ss_family) { 279 case AF_INET: { 280 struct sockaddr_in *in = (struct sockaddr_in *)s; 281 return (in->sin_port); 282 } 283 case AF_INET6: { 284 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)s; 285 return (in6->sin6_port); 286 } 287 default: 288 panic("Unsupported address family %d", s->ss_family); 289 } 290 } 291 292 static int 293 ovpn_nvlist_to_sockaddr(const nvlist_t *nvl, struct sockaddr_storage *sa) 294 { 295 int af; 296 297 if (! nvlist_exists_number(nvl, "af")) 298 return (EINVAL); 299 if (! nvlist_exists_binary(nvl, "address")) 300 return (EINVAL); 301 if (! nvlist_exists_number(nvl, "port")) 302 return (EINVAL); 303 304 af = nvlist_get_number(nvl, "af"); 305 306 switch (af) { 307 #ifdef INET 308 case AF_INET: { 309 struct sockaddr_in *in = (struct sockaddr_in *)sa; 310 size_t len; 311 const void *addr = nvlist_get_binary(nvl, "address", &len); 312 in->sin_family = af; 313 if (len != sizeof(in->sin_addr)) 314 return (EINVAL); 315 316 memcpy(&in->sin_addr, addr, sizeof(in->sin_addr)); 317 in->sin_port = nvlist_get_number(nvl, "port"); 318 break; 319 } 320 #endif 321 #ifdef INET6 322 case AF_INET6: { 323 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)sa; 324 size_t len; 325 const void *addr = nvlist_get_binary(nvl, "address", &len); 326 in6->sin6_family = af; 327 if (len != sizeof(in6->sin6_addr)) 328 return (EINVAL); 329 330 memcpy(&in6->sin6_addr, addr, sizeof(in6->sin6_addr)); 331 in6->sin6_port = nvlist_get_number(nvl, "port"); 332 break; 333 } 334 #endif 335 default: 336 return (EINVAL); 337 } 338 339 return (0); 340 } 341 342 static bool 343 ovpn_has_peers(struct ovpn_softc *sc) 344 { 345 OVPN_ASSERT(sc); 346 347 return (sc->peercount > 0); 348 } 349 350 static void 351 ovpn_rele_so(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 352 { 353 bool has_peers; 354 355 OVPN_WASSERT(sc); 356 357 if (sc->so == NULL) 358 return; 359 360 has_peers = ovpn_has_peers(sc); 361 362 /* Only remove the tunnel function if we're releasing the socket for 363 * the last peer. */ 364 if (! has_peers) 365 (void)udp_set_kernel_tunneling(sc->so, NULL, NULL, NULL); 366 367 sorele(sc->so); 368 369 if (! has_peers) 370 sc->so = NULL; 371 } 372 373 static void 374 ovpn_notify_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 375 { 376 struct ovpn_notification *n; 377 378 OVPN_WASSERT(sc); 379 380 n = malloc(sizeof(*n), M_OVPN, M_NOWAIT); 381 if (n == NULL) 382 return; 383 384 n->peerid = peer->peerid; 385 n->type = OVPN_NOTIF_DEL_PEER; 386 if (buf_ring_enqueue(sc->notifring, n) != 0) { 387 free(n, M_OVPN); 388 } else if (sc->so != NULL) { 389 /* Wake up userspace */ 390 sc->so->so_error = EAGAIN; 391 sorwakeup(sc->so); 392 sowwakeup(sc->so); 393 } 394 } 395 396 static void 397 ovpn_peer_release_ref(struct ovpn_kpeer *peer, bool locked) 398 { 399 struct ovpn_softc *sc; 400 401 atomic_add_int(&peer->refcount, -1); 402 403 if (atomic_load_int(&peer->refcount) > 0) 404 return; 405 406 sc = peer->sc; 407 408 if (! locked) { 409 OVPN_WLOCK(sc); 410 411 /* Might have changed before we acquired the lock. */ 412 if (atomic_load_int(&peer->refcount) > 0) { 413 OVPN_WUNLOCK(sc); 414 return; 415 } 416 } 417 418 /* The peer should have been removed from the list already. */ 419 MPASS(ovpn_find_peer(sc, peer->peerid) == NULL); 420 421 ovpn_notify_del_peer(sc, peer); 422 423 for (int i = 0; i < 2; i++) { 424 ovpn_free_kkey_dir(peer->keys[i].encrypt); 425 ovpn_free_kkey_dir(peer->keys[i].decrypt); 426 } 427 428 ovpn_rele_so(sc, peer); 429 430 callout_stop(&peer->ping_send); 431 callout_stop(&peer->ping_rcv); 432 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 433 free(peer, M_OVPN); 434 435 if (! locked) 436 OVPN_WUNLOCK(sc); 437 } 438 439 static int 440 ovpn_new_peer(struct ifnet *ifp, const nvlist_t *nvl) 441 { 442 #ifdef INET6 443 struct epoch_tracker et; 444 #endif 445 struct sockaddr_storage remote; 446 struct ovpn_kpeer *peer = NULL; 447 struct file *fp = NULL; 448 struct sockaddr *name = NULL; 449 struct ovpn_softc *sc = ifp->if_softc; 450 struct thread *td = curthread; 451 struct socket *so = NULL; 452 u_int fflag; 453 int fd; 454 uint32_t peerid; 455 int ret = 0, i; 456 457 if (nvl == NULL) 458 return (EINVAL); 459 460 if (! nvlist_exists_number(nvl, "peerid")) 461 return (EINVAL); 462 463 if (! nvlist_exists_number(nvl, "fd")) 464 return (EINVAL); 465 466 if (! nvlist_exists_nvlist(nvl, "remote")) 467 return (EINVAL); 468 469 peerid = nvlist_get_number(nvl, "peerid"); 470 471 ret = ovpn_nvlist_to_sockaddr(nvlist_get_nvlist(nvl, "remote"), 472 &remote); 473 if (ret != 0) 474 return (ret); 475 476 fd = nvlist_get_number(nvl, "fd"); 477 478 /* Look up the userspace process and use the fd to find the socket. */ 479 ret = getsock_cap(td, fd, &cap_connect_rights, &fp, 480 &fflag, NULL); 481 if (ret != 0) 482 return (ret); 483 484 so = fp->f_data; 485 486 peer = malloc(sizeof(*peer), M_OVPN, M_WAITOK | M_ZERO); 487 peer->peerid = peerid; 488 peer->sc = sc; 489 peer->tx_seq = 1; 490 peer->refcount = 1; 491 peer->last_active = uma_zalloc_pcpu(pcpu_zone_4, M_WAITOK | M_ZERO); 492 493 if (nvlist_exists_binary(nvl, "vpn_ipv4")) { 494 size_t len; 495 const void *addr = nvlist_get_binary(nvl, "vpn_ipv4", &len); 496 if (len != sizeof(peer->vpn4)) { 497 ret = EINVAL; 498 goto error; 499 } 500 memcpy(&peer->vpn4, addr, len); 501 } 502 503 if (nvlist_exists_binary(nvl, "vpn_ipv6")) { 504 size_t len; 505 const void *addr = nvlist_get_binary(nvl, "vpn_ipv6", &len); 506 if (len != sizeof(peer->vpn6)) { 507 ret = EINVAL; 508 goto error; 509 } 510 memcpy(&peer->vpn6, addr, len); 511 } 512 513 callout_init_rm(&peer->ping_send, &sc->lock, CALLOUT_SHAREDLOCK); 514 callout_init_rm(&peer->ping_rcv, &sc->lock, 0); 515 516 ret = (*so->so_proto->pr_usrreqs->pru_sockaddr)(so, &name); 517 if (ret) 518 goto error; 519 520 if (ovpn_get_port((struct sockaddr_storage *)name) == 0) { 521 ret = EINVAL; 522 goto error; 523 } 524 if (name->sa_family != remote.ss_family) { 525 ret = EINVAL; 526 goto error; 527 } 528 529 memcpy(&peer->local, name, name->sa_len); 530 memcpy(&peer->remote, &remote, sizeof(remote)); 531 free(name, M_SONAME); 532 name = NULL; 533 534 #ifdef INET6 535 if (peer->local.ss_family == AF_INET6 && 536 IN6_IS_ADDR_UNSPECIFIED(&TO_IN6(&peer->local)->sin6_addr)) { 537 NET_EPOCH_ENTER(et); 538 ret = in6_selectsrc_addr(curthread->td_proc->p_fibnum, 539 &TO_IN6(&peer->remote)->sin6_addr, 540 0, NULL, &TO_IN6(&peer->local)->sin6_addr, NULL); 541 NET_EPOCH_EXIT(et); 542 if (ret != 0) { 543 goto error; 544 } 545 } 546 #endif 547 OVPN_WLOCK(sc); 548 549 /* Disallow peer id re-use. */ 550 if (ovpn_find_peer(sc, peerid) != NULL) { 551 ret = EEXIST; 552 goto error_locked; 553 } 554 555 /* Must be the same socket as for other peers on this interface. */ 556 if (sc->so != NULL && so != sc->so) 557 goto error_locked; 558 559 if (sc->so == NULL) 560 sc->so = so; 561 562 /* Insert the peer into the list. */ 563 for (i = 0; i < OVPN_MAX_PEERS; i++) { 564 if (sc->peers[i] != NULL) 565 continue; 566 567 MPASS(sc->peers[i] == NULL); 568 sc->peers[i] = peer; 569 sc->peercount++; 570 soref(sc->so); 571 break; 572 } 573 if (i == OVPN_MAX_PEERS) { 574 ret = ENOSPC; 575 goto error_locked; 576 } 577 578 ret = udp_set_kernel_tunneling(sc->so, ovpn_udp_input, NULL, sc); 579 if (ret == EBUSY) { 580 /* Fine, another peer already set the input function. */ 581 ret = 0; 582 } 583 if (ret != 0) { 584 sc->peers[i] = NULL; 585 sc->peercount--; 586 goto error_locked; 587 } 588 589 OVPN_WUNLOCK(sc); 590 591 goto done; 592 593 error_locked: 594 OVPN_WUNLOCK(sc); 595 error: 596 free(name, M_SONAME); 597 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 598 free(peer, M_OVPN); 599 done: 600 if (fp != NULL) 601 fdrop(fp, td); 602 603 return (ret); 604 } 605 606 static int 607 _ovpn_del_peer(struct ovpn_softc *sc, uint32_t peerid) 608 { 609 struct ovpn_kpeer *peer; 610 int i; 611 612 OVPN_WASSERT(sc); 613 614 for (i = 0; i < OVPN_MAX_PEERS; i++) { 615 if (sc->peers[i] == NULL) 616 continue; 617 if (sc->peers[i]->peerid != peerid) 618 continue; 619 620 peer = sc->peers[i]; 621 break; 622 } 623 if (i == OVPN_MAX_PEERS) 624 return (ENOENT); 625 626 sc->peers[i] = NULL; 627 sc->peercount--; 628 629 ovpn_peer_release_ref(peer, true); 630 631 return (0); 632 } 633 634 static int 635 ovpn_del_peer(struct ifnet *ifp, nvlist_t *nvl) 636 { 637 struct ovpn_softc *sc = ifp->if_softc; 638 uint32_t peerid; 639 int ret; 640 641 OVPN_WASSERT(sc); 642 643 if (nvl == NULL) 644 return (EINVAL); 645 646 if (! nvlist_exists_number(nvl, "peerid")) 647 return (EINVAL); 648 649 peerid = nvlist_get_number(nvl, "peerid"); 650 651 ret = _ovpn_del_peer(sc, peerid); 652 653 return (ret); 654 } 655 656 static int 657 ovpn_create_kkey_dir(struct ovpn_kkey_dir **kdirp, 658 const nvlist_t *nvl) 659 { 660 struct crypto_session_params csp; 661 struct ovpn_kkey_dir *kdir; 662 const char *ciphername; 663 enum ovpn_key_cipher cipher; 664 const void *key, *iv; 665 size_t keylen = 0, ivlen = 0; 666 int error; 667 668 if (! nvlist_exists_string(nvl, "cipher")) 669 return (EINVAL); 670 ciphername = nvlist_get_string(nvl, "cipher"); 671 672 if (strcmp(ciphername, "none") == 0) 673 cipher = OVPN_CIPHER_ALG_NONE; 674 else if (strcmp(ciphername, "AES-256-GCM") == 0) 675 cipher = OVPN_CIPHER_ALG_AES_GCM; 676 else if (strcmp(ciphername, "CHACHA20-POLY1305") == 0) 677 cipher = OVPN_CIPHER_ALG_CHACHA20_POLY1305; 678 else 679 return (EINVAL); 680 681 if (cipher != OVPN_CIPHER_ALG_NONE) { 682 if (! nvlist_exists_binary(nvl, "key")) 683 return (EINVAL); 684 key = nvlist_get_binary(nvl, "key", &keylen); 685 if (keylen > sizeof(kdir->key)) 686 return (E2BIG); 687 688 if (! nvlist_exists_binary(nvl, "iv")) 689 return (EINVAL); 690 iv = nvlist_get_binary(nvl, "iv", &ivlen); 691 if (ivlen != 8) 692 return (E2BIG); 693 } 694 695 kdir = malloc(sizeof(struct ovpn_kkey_dir), M_OVPN, 696 M_WAITOK | M_ZERO); 697 698 kdir->cipher = cipher; 699 kdir->keylen = keylen; 700 memcpy(kdir->key, key, keylen); 701 kdir->noncelen = ivlen; 702 memcpy(kdir->nonce, iv, ivlen); 703 704 if (kdir->cipher != OVPN_CIPHER_ALG_NONE) { 705 /* Crypto init */ 706 bzero(&csp, sizeof(csp)); 707 csp.csp_mode = CSP_MODE_AEAD; 708 709 if (kdir->cipher == OVPN_CIPHER_ALG_CHACHA20_POLY1305) 710 csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305; 711 else 712 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 713 714 csp.csp_flags |= CSP_F_SEPARATE_AAD; 715 716 csp.csp_cipher_klen = kdir->keylen; 717 csp.csp_cipher_key = kdir->key; 718 csp.csp_ivlen = 96 / 8; 719 720 error = crypto_newsession(&kdir->cryptoid, &csp, 721 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE); 722 if (error) { 723 free(kdir, M_OVPN); 724 return (error); 725 } 726 } 727 728 mtx_init(&kdir->replay_mtx, "if_ovpn rx replay", NULL, MTX_DEF); 729 *kdirp = kdir; 730 731 return (0); 732 } 733 734 static void 735 ovpn_free_kkey_dir(struct ovpn_kkey_dir *kdir) 736 { 737 if (kdir == NULL) 738 return; 739 740 mtx_destroy(&kdir->replay_mtx); 741 742 crypto_freesession(kdir->cryptoid); 743 free(kdir, M_OVPN); 744 } 745 746 static int 747 ovpn_set_key(struct ifnet *ifp, const nvlist_t *nvl) 748 { 749 struct ovpn_softc *sc = ifp->if_softc; 750 struct ovpn_kkey_dir *enc, *dec; 751 struct ovpn_kpeer *peer; 752 int slot, keyid, peerid; 753 int error; 754 755 if (nvl == NULL) 756 return (EINVAL); 757 758 if (! nvlist_exists_number(nvl, "slot")) 759 return (EINVAL); 760 slot = nvlist_get_number(nvl, "slot"); 761 762 if (! nvlist_exists_number(nvl, "keyid")) 763 return (EINVAL); 764 keyid = nvlist_get_number(nvl, "keyid"); 765 766 if (! nvlist_exists_number(nvl, "peerid")) 767 return (EINVAL); 768 peerid = nvlist_get_number(nvl, "peerid"); 769 770 if (slot != OVPN_KEY_SLOT_PRIMARY && 771 slot != OVPN_KEY_SLOT_SECONDARY) 772 return (EINVAL); 773 774 if (! nvlist_exists_nvlist(nvl, "encrypt") || 775 ! nvlist_exists_nvlist(nvl, "decrypt")) 776 return (EINVAL); 777 778 error = ovpn_create_kkey_dir(&enc, nvlist_get_nvlist(nvl, "encrypt")); 779 if (error) 780 return (error); 781 782 error = ovpn_create_kkey_dir(&dec, nvlist_get_nvlist(nvl, "decrypt")); 783 if (error) { 784 ovpn_free_kkey_dir(enc); 785 return (error); 786 } 787 788 OVPN_WLOCK(sc); 789 790 peer = ovpn_find_peer(sc, peerid); 791 if (peer == NULL) { 792 ovpn_free_kkey_dir(dec); 793 ovpn_free_kkey_dir(enc); 794 OVPN_WUNLOCK(sc); 795 return (ENOENT); 796 } 797 798 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 799 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 800 801 peer->keys[slot].encrypt = enc; 802 peer->keys[slot].decrypt = dec; 803 804 peer->keys[slot].keyid = keyid; 805 peer->keys[slot].peerid = peerid; 806 807 OVPN_WUNLOCK(sc); 808 809 return (0); 810 } 811 812 static int 813 ovpn_check_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, enum ovpn_key_slot slot) 814 { 815 OVPN_ASSERT(sc); 816 817 if (peer->keys[slot].encrypt == NULL) 818 return (ENOLINK); 819 820 if (peer->keys[slot].decrypt == NULL) 821 return (ENOLINK); 822 823 return (0); 824 } 825 826 static int 827 ovpn_start(struct ifnet *ifp) 828 { 829 struct ovpn_softc *sc = ifp->if_softc; 830 831 OVPN_WLOCK(sc); 832 833 ifp->if_flags |= IFF_UP; 834 ifp->if_drv_flags |= IFF_DRV_RUNNING; 835 if_link_state_change(ifp, LINK_STATE_UP); 836 837 OVPN_WUNLOCK(sc); 838 839 return (0); 840 } 841 842 static int 843 ovpn_swap_keys(struct ifnet *ifp, nvlist_t *nvl) 844 { 845 struct ovpn_softc *sc = ifp->if_softc; 846 struct ovpn_kpeer *peer; 847 struct ovpn_kkey tmpkey; 848 int error; 849 850 if (nvl == NULL) 851 return (EINVAL); 852 853 if (! nvlist_exists_number(nvl, "peerid")) 854 return (EINVAL); 855 856 OVPN_WLOCK(sc); 857 858 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 859 if (peer == NULL) { 860 OVPN_WUNLOCK(sc); 861 return (ENOENT); 862 } 863 864 /* Check that we have a second key to swap to. */ 865 error = ovpn_check_key(sc, peer, OVPN_KEY_SLOT_SECONDARY); 866 if (error) { 867 OVPN_WUNLOCK(sc); 868 return (error); 869 } 870 871 tmpkey = peer->keys[0]; 872 peer->keys[0] = peer->keys[1]; 873 peer->keys[1] = tmpkey; 874 875 OVPN_WUNLOCK(sc); 876 877 return (0); 878 } 879 880 static int 881 ovpn_del_key(struct ifnet *ifp, const nvlist_t *nvl) 882 { 883 enum ovpn_key_slot slot; 884 struct ovpn_kpeer *peer; 885 struct ovpn_softc *sc = ifp->if_softc; 886 887 if (nvl == NULL) 888 return (EINVAL); 889 890 if (! nvlist_exists_number(nvl, "peerid")) 891 return (EINVAL); 892 893 if (! nvlist_exists_number(nvl, "slot")) 894 return (EINVAL); 895 slot = nvlist_get_number(nvl, "slot"); 896 897 if (slot != OVPN_KEY_SLOT_PRIMARY && 898 slot != OVPN_KEY_SLOT_SECONDARY) 899 return (EINVAL); 900 901 OVPN_WLOCK(sc); 902 903 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 904 if (peer == NULL) { 905 OVPN_WUNLOCK(sc); 906 return (ENOENT); 907 } 908 909 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 910 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 911 912 peer->keys[slot].encrypt = NULL; 913 peer->keys[slot].decrypt = NULL; 914 915 peer->keys[slot].keyid = 0; 916 peer->keys[slot].peerid = 0; 917 918 OVPN_WUNLOCK(sc); 919 920 return (0); 921 } 922 923 static int 924 ovpn_send_pkt(struct ifnet *ifp, const nvlist_t *nvl) 925 { 926 struct epoch_tracker et; 927 struct ovpn_softc *sc = ifp->if_softc; 928 struct mbuf *m; 929 const uint8_t *pkt; 930 size_t pktlen; 931 uint32_t peerid; 932 int ret; 933 934 if (nvl == NULL) 935 return (EINVAL); 936 937 if (! nvlist_exists_binary(nvl, "packet")) 938 return (EINVAL); 939 pkt = nvlist_get_binary(nvl, "packet", &pktlen); 940 941 if (! nvlist_exists_number(nvl, "peerid")) 942 return (EINVAL); 943 944 peerid = nvlist_get_number(nvl, "peerid"); 945 946 /* 947 * Check that userspace isn't giving us a data packet. That might lead 948 * to IV re-use, which would be bad. 949 */ 950 if ((pkt[0] >> OVPN_OP_SHIFT) == OVPN_OP_DATA_V2) 951 return (EINVAL); 952 953 m = m_get2(pktlen, M_WAITOK, MT_DATA, M_PKTHDR); 954 if (m == NULL) 955 return (ENOMEM); 956 957 m->m_len = m->m_pkthdr.len = pktlen; 958 m_copyback(m, 0, m->m_len, pkt); 959 960 /* Now prepend IP/UDP headers and transmit the mbuf. */ 961 NET_EPOCH_ENTER(et); 962 ret = ovpn_encap(sc, peerid, m); 963 NET_EPOCH_EXIT(et); 964 if (ret == 0) 965 OVPN_COUNTER_ADD(sc, sent_ctrl_pkts, 1); 966 else 967 OVPN_COUNTER_ADD(sc, lost_ctrl_pkts_out, 1); 968 969 return (ret); 970 } 971 972 static void 973 ovpn_send_ping(void *arg) 974 { 975 static const uint8_t ping_str[] = { 976 0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb, 977 0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48 978 }; 979 980 struct epoch_tracker et; 981 struct ovpn_kpeer *peer = arg; 982 struct ovpn_softc *sc = peer->sc; 983 struct mbuf *m; 984 985 OVPN_RASSERT(sc); 986 987 /* Ensure we repeat! */ 988 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 989 ovpn_send_ping, peer); 990 991 m = m_get2(sizeof(ping_str), M_NOWAIT, MT_DATA, M_PKTHDR); 992 if (m == NULL) 993 return; 994 995 m_copyback(m, 0, sizeof(ping_str), ping_str); 996 m->m_len = m->m_pkthdr.len = sizeof(ping_str); 997 998 CURVNET_SET(sc->ifp->if_vnet); 999 NET_EPOCH_ENTER(et); 1000 (void)ovpn_transmit_to_peer(sc->ifp, m, peer, NULL); 1001 NET_EPOCH_EXIT(et); 1002 CURVNET_RESTORE(); 1003 } 1004 1005 static void 1006 ovpn_timeout(void *arg) 1007 { 1008 struct ovpn_kpeer *peer = arg; 1009 struct ovpn_softc *sc = peer->sc; 1010 uint32_t last, _last_active; 1011 int ret __diagused; 1012 int cpu; 1013 1014 OVPN_WASSERT(sc); 1015 1016 last = 0; 1017 CPU_FOREACH(cpu) { 1018 _last_active = *zpcpu_get_cpu(peer->last_active, cpu); 1019 if (_last_active > last) 1020 last = _last_active; 1021 } 1022 1023 if (last + peer->keepalive.timeout > time_uptime) { 1024 callout_reset(&peer->ping_rcv, 1025 (peer->keepalive.timeout - (time_uptime - last)) * hz, 1026 ovpn_timeout, peer); 1027 return; 1028 } 1029 1030 CURVNET_SET(sc->ifp->if_vnet); 1031 ret = _ovpn_del_peer(sc, peer->peerid); 1032 MPASS(ret == 0); 1033 CURVNET_RESTORE(); 1034 } 1035 1036 static int 1037 ovpn_set_peer(struct ifnet *ifp, const nvlist_t *nvl) 1038 { 1039 struct ovpn_softc *sc = ifp->if_softc; 1040 struct ovpn_kpeer *peer; 1041 1042 if (nvl == NULL) 1043 return (EINVAL); 1044 1045 if (! nvlist_exists_number(nvl, "interval") || 1046 ! nvlist_exists_number(nvl, "timeout") || 1047 ! nvlist_exists_number(nvl, "peerid")) 1048 return (EINVAL); 1049 1050 OVPN_WLOCK(sc); 1051 1052 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 1053 if (peer == NULL) { 1054 OVPN_WUNLOCK(sc); 1055 return (ENOENT); 1056 } 1057 1058 peer->keepalive.interval = nvlist_get_number(nvl, "interval"); 1059 peer->keepalive.timeout = nvlist_get_number(nvl, "timeout"); 1060 1061 if (peer->keepalive.interval > 0) 1062 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 1063 ovpn_send_ping, peer); 1064 if (peer->keepalive.timeout > 0) 1065 callout_reset(&peer->ping_rcv, peer->keepalive.timeout * hz, 1066 ovpn_timeout, peer); 1067 1068 OVPN_WUNLOCK(sc); 1069 1070 return (0); 1071 } 1072 1073 static int 1074 ovpn_ioctl_set(struct ifnet *ifp, struct ifdrv *ifd) 1075 { 1076 struct ovpn_softc *sc = ifp->if_softc; 1077 uint8_t *buf = NULL; 1078 nvlist_t *nvl = NULL; 1079 int ret; 1080 1081 if (ifd->ifd_len != 0) { 1082 if (ifd->ifd_len > OVPN_MAX_REQUEST_SIZE) 1083 return (E2BIG); 1084 1085 buf = malloc(ifd->ifd_len, M_OVPN, M_WAITOK); 1086 1087 ret = copyin(ifd->ifd_data, buf, ifd->ifd_len); 1088 if (ret != 0) { 1089 free(buf, M_OVPN); 1090 return (ret); 1091 } 1092 1093 nvl = nvlist_unpack(buf, ifd->ifd_len, 0); 1094 free(buf, M_OVPN); 1095 if (nvl == NULL) { 1096 return (EINVAL); 1097 } 1098 } 1099 1100 switch (ifd->ifd_cmd) { 1101 case OVPN_NEW_PEER: 1102 ret = ovpn_new_peer(ifp, nvl); 1103 break; 1104 case OVPN_DEL_PEER: 1105 OVPN_WLOCK(sc); 1106 ret = ovpn_del_peer(ifp, nvl); 1107 OVPN_WUNLOCK(sc); 1108 break; 1109 case OVPN_NEW_KEY: 1110 ret = ovpn_set_key(ifp, nvl); 1111 break; 1112 case OVPN_START_VPN: 1113 ret = ovpn_start(ifp); 1114 break; 1115 case OVPN_SWAP_KEYS: 1116 ret = ovpn_swap_keys(ifp, nvl); 1117 break; 1118 case OVPN_DEL_KEY: 1119 ret = ovpn_del_key(ifp, nvl); 1120 break; 1121 case OVPN_SEND_PKT: 1122 ret = ovpn_send_pkt(ifp, nvl); 1123 break; 1124 case OVPN_SET_PEER: 1125 ret = ovpn_set_peer(ifp, nvl); 1126 break; 1127 default: 1128 ret = ENOTSUP; 1129 } 1130 1131 nvlist_destroy(nvl); 1132 return (ret); 1133 } 1134 1135 static int 1136 ovpn_add_counters(nvlist_t *parent, const char *name, counter_u64_t in, 1137 counter_u64_t out) 1138 { 1139 nvlist_t *nvl; 1140 1141 nvl = nvlist_create(0); 1142 if (nvl == NULL) 1143 return (ENOMEM); 1144 1145 nvlist_add_number(nvl, "in", counter_u64_fetch(in)); 1146 nvlist_add_number(nvl, "out", counter_u64_fetch(out)); 1147 1148 nvlist_add_nvlist(parent, name, nvl); 1149 1150 nvlist_destroy(nvl); 1151 1152 return (0); 1153 } 1154 1155 static int 1156 ovpn_get_stats(struct ovpn_softc *sc, nvlist_t **onvl) 1157 { 1158 nvlist_t *nvl; 1159 int ret; 1160 1161 nvl = nvlist_create(0); 1162 if (nvl == NULL) 1163 return (ENOMEM); 1164 1165 #define OVPN_COUNTER_OUT(name, in, out) \ 1166 do { \ 1167 ret = ovpn_add_counters(nvl, name, \ 1168 sc->counters[offsetof(struct ovpn_counters, in) / \ 1169 sizeof(uint64_t)], \ 1170 sc->counters[offsetof(struct ovpn_counters, out) / \ 1171 sizeof(uint64_t)]); \ 1172 if (ret != 0) \ 1173 goto error; \ 1174 } while(0) 1175 1176 OVPN_COUNTER_OUT("lost_ctrl", lost_ctrl_pkts_in, lost_ctrl_pkts_out); 1177 OVPN_COUNTER_OUT("lost_data", lost_data_pkts_in, lost_data_pkts_out); 1178 OVPN_COUNTER_OUT("nomem_data", nomem_data_pkts_in, 1179 nomem_data_pkts_out); 1180 OVPN_COUNTER_OUT("data", received_data_pkts, sent_data_pkts); 1181 OVPN_COUNTER_OUT("ctrl", received_ctrl_pkts, sent_ctrl_pkts); 1182 OVPN_COUNTER_OUT("tunnel", tunnel_bytes_received, 1183 tunnel_bytes_received); 1184 OVPN_COUNTER_OUT("transport", transport_bytes_received, 1185 transport_bytes_received); 1186 #undef OVPN_COUNTER_OUT 1187 1188 *onvl = nvl; 1189 1190 return (0); 1191 1192 error: 1193 nvlist_destroy(nvl); 1194 return (ret); 1195 } 1196 1197 static int 1198 ovpn_poll_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1199 { 1200 nvlist_t *nvl; 1201 1202 nvl = nvlist_create(0); 1203 if (nvl == NULL) 1204 return (ENOMEM); 1205 1206 nvlist_add_number(nvl, "pending", 1207 buf_ring_count(sc->rxring) + buf_ring_count(sc->notifring)); 1208 1209 *onvl = nvl; 1210 1211 return (0); 1212 } 1213 1214 static int 1215 opvn_get_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1216 { 1217 struct ovpn_notification *n; 1218 struct ovpn_wire_header *ohdr; 1219 struct mbuf *m; 1220 uint8_t *buf; 1221 nvlist_t *nvl; 1222 uint32_t peerid; 1223 u_int mlength; 1224 1225 /* Check if we have notifications pending. */ 1226 n = buf_ring_dequeue_mc(sc->notifring); 1227 if (n != NULL) { 1228 nvl = nvlist_create(0); 1229 if (nvl == NULL) { 1230 free(n, M_OVPN); 1231 return (ENOMEM); 1232 } 1233 nvlist_add_number(nvl, "peerid", n->peerid); 1234 nvlist_add_number(nvl, "notification", n->type); 1235 free(n, M_OVPN); 1236 1237 *onvl = nvl; 1238 1239 return (0); 1240 } 1241 1242 /* Queued packet. */ 1243 m = buf_ring_dequeue_mc(sc->rxring); 1244 if (m == NULL) 1245 return (ENOENT); 1246 1247 mlength = m_length(m, NULL); 1248 buf = malloc(mlength, M_NVLIST, M_WAITOK); 1249 m_copydata(m, 0, mlength, buf); 1250 ohdr = (struct ovpn_wire_header *)buf; 1251 peerid = ntohl(ohdr->opcode) & 0x00ffffff; 1252 1253 nvl = nvlist_create(0); 1254 if (nvl == NULL) { 1255 OVPN_COUNTER_ADD(sc, lost_ctrl_pkts_in, 1); 1256 m_freem(m); 1257 free(buf, M_NVLIST); 1258 return (ENOMEM); 1259 } 1260 1261 nvlist_move_binary(nvl, "packet", buf, mlength); 1262 buf = NULL; 1263 nvlist_add_number(nvl, "peerid", peerid); 1264 1265 *onvl = nvl; 1266 1267 m_freem(m); 1268 1269 return (0); 1270 } 1271 1272 static int 1273 ovpn_ioctl_get(struct ifnet *ifp, struct ifdrv *ifd) 1274 { 1275 struct ovpn_softc *sc = ifp->if_softc; 1276 nvlist_t *nvl = NULL; 1277 int error; 1278 1279 switch (ifd->ifd_cmd) { 1280 case OVPN_GET_STATS: 1281 error = ovpn_get_stats(sc, &nvl); 1282 break; 1283 case OVPN_POLL_PKT: 1284 error = ovpn_poll_pkt(sc, &nvl); 1285 break; 1286 case OVPN_GET_PKT: 1287 error = opvn_get_pkt(sc, &nvl); 1288 break; 1289 default: 1290 error = ENOTSUP; 1291 break; 1292 } 1293 1294 if (error == 0) { 1295 void *packed = NULL; 1296 size_t len; 1297 1298 MPASS(nvl != NULL); 1299 1300 packed = nvlist_pack(nvl, &len); 1301 if (! packed) { 1302 nvlist_destroy(nvl); 1303 return (ENOMEM); 1304 } 1305 1306 if (len > ifd->ifd_len) { 1307 free(packed, M_NVLIST); 1308 nvlist_destroy(nvl); 1309 return (ENOSPC); 1310 } 1311 1312 error = copyout(packed, ifd->ifd_data, len); 1313 ifd->ifd_len = len; 1314 1315 free(packed, M_NVLIST); 1316 nvlist_destroy(nvl); 1317 } 1318 1319 return (error); 1320 } 1321 1322 static int 1323 ovpn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 1324 { 1325 struct ifdrv *ifd; 1326 int error; 1327 1328 switch (cmd) { 1329 case SIOCSDRVSPEC: 1330 case SIOCGDRVSPEC: 1331 error = priv_check(curthread, PRIV_NET_OVPN); 1332 if (error) 1333 return (error); 1334 break; 1335 } 1336 1337 switch (cmd) { 1338 case SIOCSDRVSPEC: 1339 ifd = (struct ifdrv *)data; 1340 error = ovpn_ioctl_set(ifp, ifd); 1341 break; 1342 case SIOCGDRVSPEC: 1343 ifd = (struct ifdrv *)data; 1344 error = ovpn_ioctl_get(ifp, ifd); 1345 break; 1346 case SIOCSIFMTU: { 1347 struct ifreq *ifr = (struct ifreq *)data; 1348 if (ifr->ifr_mtu < OVPN_MTU_MIN || ifr->ifr_mtu > OVPN_MTU_MAX) 1349 return (EINVAL); 1350 1351 ifp->if_mtu = ifr->ifr_mtu; 1352 return (0); 1353 } 1354 case SIOCSIFADDR: 1355 case SIOCADDMULTI: 1356 case SIOCDELMULTI: 1357 case SIOCGIFMTU: 1358 case SIOCSIFFLAGS: 1359 return (0); 1360 default: 1361 error = EINVAL; 1362 } 1363 1364 return (error); 1365 } 1366 1367 static int 1368 ovpn_encrypt_tx_cb(struct cryptop *crp) 1369 { 1370 struct ovpn_kpeer *peer = crp->crp_opaque; 1371 struct ovpn_softc *sc = peer->sc; 1372 struct mbuf *m = crp->crp_buf.cb_mbuf; 1373 int ret; 1374 1375 if (crp->crp_etype != 0) { 1376 crypto_freereq(crp); 1377 ovpn_peer_release_ref(peer, false); 1378 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1379 m_freem(m); 1380 return (0); 1381 } 1382 1383 CURVNET_SET(sc->ifp->if_vnet); 1384 1385 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1386 1387 ret = ovpn_encap(sc, peer->peerid, m); 1388 if (ret == 0) { 1389 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1390 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, m->m_pkthdr.len - 1391 sizeof(struct ovpn_wire_header)); 1392 } 1393 1394 CURVNET_RESTORE(); 1395 1396 crypto_freereq(crp); 1397 ovpn_peer_release_ref(peer, false); 1398 1399 return (0); 1400 } 1401 1402 static void 1403 ovpn_finish_rx(struct ovpn_softc *sc, struct mbuf *m, 1404 struct ovpn_kpeer *peer, struct ovpn_kkey *key, uint32_t seq, 1405 struct rm_priotracker *_ovpn_lock_trackerp) 1406 { 1407 uint32_t af; 1408 int ret __diagused; 1409 1410 OVPN_RASSERT(sc); 1411 1412 /* Replay protection. */ 1413 if (V_replay_protection && ! ovpn_check_replay(key->decrypt, seq)) { 1414 OVPN_RUNLOCK(sc); 1415 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1416 m_freem(m); 1417 return; 1418 } 1419 1420 critical_enter(); 1421 *zpcpu_get(peer->last_active) = time_uptime; 1422 critical_exit(); 1423 1424 OVPN_RUNLOCK(sc); 1425 1426 OVPN_COUNTER_ADD(sc, received_data_pkts, 1); 1427 OVPN_COUNTER_ADD(sc, tunnel_bytes_received, m->m_pkthdr.len); 1428 1429 /* Receive the packet on our interface. */ 1430 m->m_pkthdr.rcvif = sc->ifp; 1431 1432 /* Clear checksum flags in case the real hardware set them. */ 1433 m->m_pkthdr.csum_flags = 0; 1434 1435 /* Ensure we can read the first byte. */ 1436 m = m_pullup(m, 1); 1437 1438 /* 1439 * Check for address family, and disregard any control packets (e.g. 1440 * keepalive). 1441 */ 1442 af = ovpn_get_af(m); 1443 if (af != 0) { 1444 BPF_MTAP2(sc->ifp, &af, sizeof(af), m); 1445 ret = netisr_dispatch(af == AF_INET ? NETISR_IP : NETISR_IPV6, 1446 m); 1447 MPASS(ret == 0); 1448 } else { 1449 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1450 m_freem(m); 1451 } 1452 } 1453 1454 static struct ovpn_kkey * 1455 ovpn_find_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, 1456 const struct ovpn_wire_header *ohdr) 1457 { 1458 struct ovpn_kkey *key = NULL; 1459 uint8_t keyid; 1460 1461 OVPN_RASSERT(sc); 1462 1463 keyid = (ntohl(ohdr->opcode) >> 24) & 0x07; 1464 1465 if (peer->keys[0].keyid == keyid) 1466 key = &peer->keys[0]; 1467 else if (peer->keys[1].keyid == keyid) 1468 key = &peer->keys[1]; 1469 1470 return (key); 1471 } 1472 1473 static int 1474 ovpn_decrypt_rx_cb(struct cryptop *crp) 1475 { 1476 struct ovpn_softc *sc = crp->crp_opaque; 1477 struct mbuf *m = crp->crp_buf.cb_mbuf; 1478 struct ovpn_kkey *key; 1479 struct ovpn_kpeer *peer; 1480 struct ovpn_wire_header *ohdr; 1481 uint32_t peerid; 1482 1483 OVPN_RLOCK_TRACKER; 1484 1485 OVPN_RLOCK(sc); 1486 1487 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1488 1489 if (crp->crp_etype != 0) { 1490 crypto_freereq(crp); 1491 atomic_add_int(&sc->refcount, -1); 1492 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1493 OVPN_RUNLOCK(sc); 1494 m_freem(m); 1495 return (0); 1496 } 1497 1498 CURVNET_SET(sc->ifp->if_vnet); 1499 1500 ohdr = mtodo(m, sizeof(struct udphdr)); 1501 1502 peerid = ntohl(ohdr->opcode) & 0x00ffffff; 1503 peer = ovpn_find_peer(sc, peerid); 1504 if (peer == NULL) { 1505 /* No such peer. Drop packet. */ 1506 crypto_freereq(crp); 1507 atomic_add_int(&sc->refcount, -1); 1508 OVPN_RUNLOCK(sc); 1509 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1510 m_freem(m); 1511 CURVNET_RESTORE(); 1512 return (0); 1513 } 1514 1515 key = ovpn_find_key(sc, peer, ohdr); 1516 if (key == NULL) { 1517 crypto_freereq(crp); 1518 atomic_add_int(&sc->refcount, -1); 1519 /* 1520 * Has this key been removed between us starting the decrypt 1521 * and finishing it? 1522 */ 1523 OVPN_RUNLOCK(sc); 1524 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1525 m_freem(m); 1526 CURVNET_RESTORE(); 1527 return (0); 1528 } 1529 1530 /* Now remove the outer headers */ 1531 m_adj_decap(m, sizeof(struct udphdr) + 1532 sizeof(struct ovpn_wire_header)); 1533 1534 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), _ovpn_lock_trackerp); 1535 OVPN_UNLOCK_ASSERT(sc); 1536 1537 CURVNET_RESTORE(); 1538 1539 crypto_freereq(crp); 1540 atomic_add_int(&sc->refcount, -1); 1541 1542 return (0); 1543 } 1544 1545 static uint8_t EMPTY_BUFFER[AES_BLOCK_LEN]; 1546 1547 static int 1548 ovpn_get_af(struct mbuf *m) 1549 { 1550 struct ip *ip; 1551 struct ip6_hdr *ip6; 1552 1553 /* 1554 * We should pullup, but we're only interested in the first byte, so 1555 * that'll always be contiguous. 1556 */ 1557 ip = mtod(m, struct ip *); 1558 if (ip->ip_v == IPVERSION) 1559 return (AF_INET); 1560 1561 ip6 = mtod(m, struct ip6_hdr *); 1562 if (ip6->ip6_vfc == IPV6_VERSION) 1563 return (AF_INET6); 1564 1565 return (0); 1566 } 1567 1568 static struct ovpn_kpeer * 1569 ovpn_find_peer_by_ip(struct ovpn_softc *sc, const struct in_addr addr) 1570 { 1571 struct ovpn_kpeer *peer = NULL; 1572 1573 OVPN_ASSERT(sc); 1574 1575 for (int i = 0; i < OVPN_MAX_PEERS; i++) { 1576 if (sc->peers[i] == NULL) 1577 continue; 1578 if (addr.s_addr == sc->peers[i]->vpn4.s_addr) { 1579 peer = sc->peers[i]; 1580 break; 1581 } 1582 } 1583 1584 return (peer); 1585 } 1586 1587 static struct ovpn_kpeer * 1588 ovpn_find_peer_by_ip6(struct ovpn_softc *sc, const struct in6_addr *addr) 1589 { 1590 struct ovpn_kpeer *peer = NULL; 1591 1592 OVPN_ASSERT(sc); 1593 1594 for (int i = 0; i < OVPN_MAX_PEERS; i++) { 1595 if (sc->peers[i] == NULL) 1596 continue; 1597 if (memcmp(addr, &sc->peers[i]->vpn6, sizeof(*addr)) == 0) { 1598 peer = sc->peers[i]; 1599 break; 1600 } 1601 } 1602 1603 return (peer); 1604 } 1605 1606 static struct ovpn_kpeer * 1607 ovpn_route_peer(struct ovpn_softc *sc, struct mbuf **m0, 1608 const struct sockaddr *dst) 1609 { 1610 struct ovpn_kpeer *peer = NULL; 1611 int af; 1612 1613 NET_EPOCH_ASSERT(); 1614 OVPN_ASSERT(sc); 1615 1616 /* Shortcut if we're a client (or are a server and have only one client). */ 1617 if (sc->peercount == 1) 1618 return (ovpn_find_only_peer(sc)); 1619 1620 if (dst != NULL) 1621 af = dst->sa_family; 1622 else 1623 af = ovpn_get_af(*m0); 1624 1625 switch (af) { 1626 #ifdef INET 1627 case AF_INET: { 1628 const struct sockaddr_in *sa = (const struct sockaddr_in *)dst; 1629 struct nhop_object *nh; 1630 const struct in_addr *ip_dst; 1631 1632 if (sa != NULL) { 1633 ip_dst = &sa->sin_addr; 1634 } else { 1635 struct ip *ip; 1636 1637 *m0 = m_pullup(*m0, sizeof(struct ip)); 1638 if (*m0 == NULL) 1639 return (NULL); 1640 ip = mtod(*m0, struct ip *); 1641 ip_dst = &ip->ip_dst; 1642 } 1643 1644 peer = ovpn_find_peer_by_ip(sc, *ip_dst); 1645 SDT_PROBE2(if_ovpn, tx, route, ip4, ip_dst, peer); 1646 if (peer == NULL) { 1647 nh = fib4_lookup(M_GETFIB(*m0), *ip_dst, 0, 1648 NHR_NONE, 0); 1649 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1650 peer = ovpn_find_peer_by_ip(sc, 1651 nh->gw4_sa.sin_addr); 1652 SDT_PROBE2(if_ovpn, tx, route, ip4, 1653 &nh->gw4_sa.sin_addr, peer); 1654 } 1655 } 1656 break; 1657 } 1658 #endif 1659 #ifdef INET6 1660 case AF_INET6: { 1661 const struct sockaddr_in6 *sa6 = 1662 (const struct sockaddr_in6 *)dst; 1663 struct nhop_object *nh; 1664 const struct in6_addr *ip6_dst; 1665 1666 if (sa6 != NULL) { 1667 ip6_dst = &sa6->sin6_addr; 1668 } else { 1669 struct ip6_hdr *ip6; 1670 1671 *m0 = m_pullup(*m0, sizeof(struct ip6_hdr)); 1672 if (*m0 == NULL) 1673 return (NULL); 1674 ip6 = mtod(*m0, struct ip6_hdr *); 1675 ip6_dst = &ip6->ip6_dst; 1676 } 1677 1678 peer = ovpn_find_peer_by_ip6(sc, ip6_dst); 1679 SDT_PROBE2(if_ovpn, tx, route, ip6, ip6_dst, peer); 1680 if (peer == NULL) { 1681 nh = fib6_lookup(M_GETFIB(*m0), ip6_dst, 0, 1682 NHR_NONE, 0); 1683 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1684 peer = ovpn_find_peer_by_ip6(sc, 1685 &nh->gw6_sa.sin6_addr); 1686 SDT_PROBE2(if_ovpn, tx, route, ip6, 1687 &nh->gw6_sa.sin6_addr, peer); 1688 } 1689 } 1690 break; 1691 } 1692 #endif 1693 } 1694 1695 return (peer); 1696 } 1697 1698 static int 1699 ovpn_transmit(struct ifnet *ifp, struct mbuf *m) 1700 { 1701 return (ifp->if_output(ifp, m, NULL, NULL)); 1702 } 1703 1704 static int 1705 ovpn_transmit_to_peer(struct ifnet *ifp, struct mbuf *m, 1706 struct ovpn_kpeer *peer, struct rm_priotracker *_ovpn_lock_trackerp) 1707 { 1708 struct ovpn_wire_header *ohdr; 1709 struct ovpn_kkey *key; 1710 struct ovpn_softc *sc; 1711 struct cryptop *crp; 1712 uint32_t af, seq; 1713 size_t len, real_len, ovpn_hdr_len; 1714 int tunnel_len; 1715 int ret; 1716 1717 sc = ifp->if_softc; 1718 1719 OVPN_RASSERT(sc); 1720 1721 tunnel_len = m->m_pkthdr.len; 1722 1723 key = &peer->keys[OVPN_KEY_SLOT_PRIMARY]; 1724 if (key->encrypt == NULL) { 1725 if (_ovpn_lock_trackerp != NULL) 1726 OVPN_RUNLOCK(sc); 1727 m_freem(m); 1728 return (ENOLINK); 1729 } 1730 1731 af = ovpn_get_af(m); 1732 /* Don't capture control packets. */ 1733 if (af != 0) 1734 BPF_MTAP2(ifp, &af, sizeof(af), m); 1735 1736 real_len = len = m->m_pkthdr.len; 1737 MPASS(real_len <= ifp->if_mtu); 1738 1739 ovpn_hdr_len = sizeof(struct ovpn_wire_header); 1740 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) 1741 ovpn_hdr_len -= 16; /* No auth tag. */ 1742 else { 1743 /* Round up the len to a multiple of our block size. */ 1744 len = roundup2(real_len, AES_BLOCK_LEN); 1745 1746 /* Now extend the mbuf. */ 1747 m_append(m, len - real_len, EMPTY_BUFFER); 1748 } 1749 1750 M_PREPEND(m, ovpn_hdr_len, M_NOWAIT); 1751 if (m == NULL) { 1752 if (_ovpn_lock_trackerp != NULL) 1753 OVPN_RUNLOCK(sc); 1754 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1755 return (ENOBUFS); 1756 } 1757 ohdr = mtod(m, struct ovpn_wire_header *); 1758 ohdr->opcode = (OVPN_OP_DATA_V2 << OVPN_OP_SHIFT) | key->keyid; 1759 ohdr->opcode <<= 24; 1760 ohdr->opcode |= key->peerid; 1761 ohdr->opcode = htonl(ohdr->opcode); 1762 1763 seq = atomic_fetchadd_32(&peer->tx_seq, 1); 1764 seq = htonl(seq); 1765 ohdr->seq = seq; 1766 1767 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) { 1768 ret = ovpn_encap(sc, peer->peerid, m); 1769 if (_ovpn_lock_trackerp != NULL) 1770 OVPN_RUNLOCK(sc); 1771 if (ret == 0) { 1772 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1773 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 1774 } 1775 return (ret); 1776 } 1777 1778 crp = crypto_getreq(key->encrypt->cryptoid, M_NOWAIT); 1779 if (crp == NULL) { 1780 if (_ovpn_lock_trackerp != NULL) 1781 OVPN_RUNLOCK(sc); 1782 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1783 m_freem(m); 1784 return (ENOBUFS); 1785 } 1786 1787 /* Encryption covers only the payload, not the header. */ 1788 crp->crp_payload_start = sizeof(*ohdr); 1789 crp->crp_payload_length = len; 1790 crp->crp_op = CRYPTO_OP_ENCRYPT; 1791 1792 /* 1793 * AAD data covers the ovpn_wire_header minus the auth 1794 * tag. 1795 */ 1796 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 1797 crp->crp_aad = ohdr; 1798 crp->crp_aad_start = 0; 1799 crp->crp_op |= CRYPTO_OP_COMPUTE_DIGEST; 1800 crp->crp_digest_start = offsetof(struct ovpn_wire_header, auth_tag); 1801 1802 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 1803 memcpy(crp->crp_iv, &seq, sizeof(seq)); 1804 memcpy(crp->crp_iv + sizeof(seq), key->encrypt->nonce, 1805 key->encrypt->noncelen); 1806 1807 crypto_use_mbuf(crp, m); 1808 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 1809 crp->crp_callback = ovpn_encrypt_tx_cb; 1810 crp->crp_opaque = peer; 1811 1812 atomic_add_int(&peer->refcount, 1); 1813 if (_ovpn_lock_trackerp != NULL) 1814 OVPN_RUNLOCK(sc); 1815 ret = crypto_dispatch(crp); 1816 if (ret) { 1817 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1818 } 1819 1820 return (ret); 1821 } 1822 1823 /* 1824 * Note: Expects to hold the read lock on entry, and will release it itself. 1825 */ 1826 static int 1827 ovpn_encap(struct ovpn_softc *sc, uint32_t peerid, struct mbuf *m) 1828 { 1829 struct udphdr *udp; 1830 struct ovpn_kpeer *peer; 1831 int len; 1832 1833 OVPN_RLOCK_TRACKER; 1834 1835 OVPN_RLOCK(sc); 1836 NET_EPOCH_ASSERT(); 1837 1838 peer = ovpn_find_peer(sc, peerid); 1839 if (peer == NULL || sc->ifp->if_link_state != LINK_STATE_UP) { 1840 OVPN_RUNLOCK(sc); 1841 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1842 m_freem(m); 1843 return (ENETDOWN); 1844 } 1845 1846 len = m->m_pkthdr.len; 1847 1848 M_PREPEND(m, sizeof(struct udphdr), M_NOWAIT); 1849 if (m == NULL) { 1850 OVPN_RUNLOCK(sc); 1851 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1852 m_freem(m); 1853 return (ENOBUFS); 1854 } 1855 udp = mtod(m, struct udphdr *); 1856 1857 MPASS(peer->local.ss_family == peer->remote.ss_family); 1858 1859 udp->uh_sport = ovpn_get_port(&peer->local); 1860 udp->uh_dport = ovpn_get_port(&peer->remote); 1861 udp->uh_ulen = htons(sizeof(struct udphdr) + len); 1862 1863 switch (peer->remote.ss_family) { 1864 #ifdef INET 1865 case AF_INET: { 1866 struct sockaddr_in *in_local = TO_IN(&peer->local); 1867 struct sockaddr_in *in_remote = TO_IN(&peer->remote); 1868 struct ip *ip; 1869 1870 /* 1871 * This requires knowing the source IP, which we don't. Happily 1872 * we're allowed to keep this at 0, and the checksum won't do 1873 * anything the crypto won't already do. 1874 */ 1875 udp->uh_sum = 0; 1876 1877 /* Set the checksum flags so we recalculate checksums. */ 1878 m->m_pkthdr.csum_flags |= CSUM_IP; 1879 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 1880 1881 M_PREPEND(m, sizeof(struct ip), M_NOWAIT); 1882 if (m == NULL) { 1883 OVPN_RUNLOCK(sc); 1884 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1885 return (ENOBUFS); 1886 } 1887 ip = mtod(m, struct ip *); 1888 1889 ip->ip_tos = 0; 1890 ip->ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) + 1891 len); 1892 ip->ip_off = 0; 1893 ip->ip_ttl = V_ip_defttl; 1894 ip->ip_p = IPPROTO_UDP; 1895 ip->ip_sum = 0; 1896 if (in_local->sin_port != 0) 1897 ip->ip_src = in_local->sin_addr; 1898 else 1899 ip->ip_src.s_addr = INADDR_ANY; 1900 ip->ip_dst = in_remote->sin_addr; 1901 1902 OVPN_RUNLOCK(sc); 1903 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 1904 1905 return (ip_output(m, NULL, NULL, 0, NULL, NULL)); 1906 } 1907 #endif 1908 #ifdef INET6 1909 case AF_INET6: { 1910 struct sockaddr_in6 *in6_local = TO_IN6(&peer->local); 1911 struct sockaddr_in6 *in6_remote = TO_IN6(&peer->remote); 1912 struct ip6_hdr *ip6; 1913 1914 M_PREPEND(m, sizeof(struct ip6_hdr), M_NOWAIT); 1915 if (m == NULL) { 1916 OVPN_RUNLOCK(sc); 1917 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1918 return (ENOBUFS); 1919 } 1920 m = m_pullup(m, sizeof(*ip6) + sizeof(*udp)); 1921 if (m == NULL) { 1922 OVPN_RUNLOCK(sc); 1923 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1924 return (ENOBUFS); 1925 } 1926 1927 ip6 = mtod(m, struct ip6_hdr *); 1928 1929 ip6->ip6_vfc = IPV6_VERSION; 1930 ip6->ip6_flow &= ~IPV6_FLOWINFO_MASK; 1931 ip6->ip6_plen = htons(sizeof(*ip6) + sizeof(struct udphdr) + 1932 len); 1933 ip6->ip6_nxt = IPPROTO_UDP; 1934 ip6->ip6_hlim = V_ip6_defhlim; 1935 1936 memcpy(&ip6->ip6_src, &in6_local->sin6_addr, 1937 sizeof(ip6->ip6_src)); 1938 memcpy(&ip6->ip6_dst, &in6_remote->sin6_addr, 1939 sizeof(ip6->ip6_dst)); 1940 1941 udp = mtodo(m, sizeof(*ip6)); 1942 udp->uh_sum = in6_cksum_pseudo(ip6, 1943 m->m_pkthdr.len - sizeof(struct ip6_hdr), 1944 IPPROTO_UDP, 0); 1945 1946 m->m_pkthdr.csum_flags |= CSUM_UDP_IPV6; 1947 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 1948 1949 OVPN_RUNLOCK(sc); 1950 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 1951 1952 return (ip6_output(m, NULL, NULL, IPV6_UNSPECSRC, NULL, NULL, 1953 NULL)); 1954 } 1955 #endif 1956 default: 1957 panic("Unsupported address family %d", 1958 peer->remote.ss_family); 1959 } 1960 } 1961 1962 static int 1963 ovpn_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 1964 struct route *ro) 1965 { 1966 struct ovpn_softc *sc; 1967 struct ovpn_kpeer *peer; 1968 1969 OVPN_RLOCK_TRACKER; 1970 1971 sc = ifp->if_softc; 1972 1973 OVPN_RLOCK(sc); 1974 1975 SDT_PROBE1(if_ovpn, tx, transmit, start, m); 1976 1977 if (__predict_false(ifp->if_link_state != LINK_STATE_UP)) { 1978 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1979 OVPN_RUNLOCK(sc); 1980 return (ENETDOWN); 1981 } 1982 1983 /** 1984 * Only obey 'dst' (i.e. the gateway) if no route is supplied. 1985 * That's our indication that we're being called through pf's route-to, 1986 * and we should route according to 'dst' instead. We can't do so 1987 * consistently, because the usual openvpn configuration sets the first 1988 * non-server IP in the subnet as the gateway. If we always use that 1989 * one we'd end up routing all traffic to the first client. 1990 * tl;dr: 'ro == NULL' tells us pf is doing a route-to, and then but 1991 * only then, we should treat 'dst' as the destination. */ 1992 peer = ovpn_route_peer(sc, &m, ro == NULL ? dst : NULL); 1993 if (peer == NULL) { 1994 /* No destination. */ 1995 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1996 OVPN_RUNLOCK(sc); 1997 return (ENETDOWN); 1998 } 1999 2000 return (ovpn_transmit_to_peer(ifp, m, peer, _ovpn_lock_trackerp)); 2001 } 2002 2003 static void 2004 ovpn_rcv_ctrl(struct ovpn_softc *sc, struct mbuf *m, int off) 2005 { 2006 /* Lop off the IP and UDP headers */ 2007 m_adj_decap(m, off); 2008 2009 /* Keep in the local ring until userspace fetches it. */ 2010 if (buf_ring_enqueue(sc->rxring, m) != 0) { 2011 OVPN_COUNTER_ADD(sc, lost_ctrl_pkts_in, 1); 2012 m_freem(m); 2013 return; 2014 } 2015 2016 OVPN_COUNTER_ADD(sc, received_ctrl_pkts, 1); 2017 } 2018 2019 static bool 2020 ovpn_check_replay(struct ovpn_kkey_dir *key, uint32_t seq) 2021 { 2022 uint32_t d; 2023 2024 mtx_lock(&key->replay_mtx); 2025 2026 /* Sequence number must be strictly greater than rx_seq */ 2027 if (seq <= key->rx_seq) { 2028 mtx_unlock(&key->replay_mtx); 2029 return (false); 2030 } 2031 2032 /* Large jump. The packet authenticated okay, so just accept that. */ 2033 if (seq > (key->rx_seq + (sizeof(key->rx_window) * 8))) { 2034 key->rx_seq = seq; 2035 key->rx_window = 0; 2036 mtx_unlock(&key->replay_mtx); 2037 return (true); 2038 } 2039 2040 /* Happy case. */ 2041 if ((seq == key->rx_seq + 1) && key->rx_window == 0) { 2042 key->rx_seq++; 2043 mtx_unlock(&key->replay_mtx); 2044 return (true); 2045 } 2046 2047 d = seq - key->rx_seq - 1; 2048 2049 if (key->rx_window & ((uint64_t)1 << d)) { 2050 /* Dupe! */ 2051 mtx_unlock(&key->replay_mtx); 2052 return (false); 2053 } 2054 2055 key->rx_window |= (uint64_t)1 << d; 2056 2057 while (key->rx_window & 1) { 2058 key->rx_seq++; 2059 key->rx_window >>= 1; 2060 } 2061 2062 mtx_unlock(&key->replay_mtx); 2063 2064 return (true); 2065 } 2066 2067 static struct ovpn_kpeer * 2068 ovpn_peer_from_mbuf(struct ovpn_softc *sc, struct mbuf *m, int off) 2069 { 2070 struct ovpn_wire_header ohdr; 2071 uint32_t peerid; 2072 2073 OVPN_RASSERT(sc); 2074 2075 m_copydata(m, off + sizeof(struct udphdr), 2076 sizeof(ohdr) - sizeof(ohdr.auth_tag), (caddr_t)&ohdr); 2077 2078 peerid = ntohl(ohdr.opcode) & 0x00ffffff; 2079 2080 return (ovpn_find_peer(sc, peerid)); 2081 } 2082 2083 static bool 2084 ovpn_udp_input(struct mbuf *m, int off, struct inpcb *inp, 2085 const struct sockaddr *sa, void *ctx) 2086 { 2087 struct ovpn_softc *sc = ctx; 2088 struct ovpn_wire_header *ohdr; 2089 struct udphdr *uhdr; 2090 struct ovpn_kkey *key; 2091 struct cryptop *crp; 2092 struct ovpn_kpeer *peer; 2093 size_t ohdrlen; 2094 int ret; 2095 uint8_t op; 2096 2097 OVPN_RLOCK_TRACKER; 2098 2099 M_ASSERTPKTHDR(m); 2100 2101 OVPN_COUNTER_ADD(sc, transport_bytes_received, m->m_pkthdr.len - off); 2102 2103 ohdrlen = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2104 2105 OVPN_RLOCK(sc); 2106 2107 peer = ovpn_peer_from_mbuf(sc, m, off); 2108 if (peer == NULL) { 2109 OVPN_RUNLOCK(sc); 2110 return (false); 2111 } 2112 2113 m = m_pullup(m, off + sizeof(*uhdr) + ohdrlen); 2114 if (m == NULL) { 2115 OVPN_RUNLOCK(sc); 2116 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2117 return (true); 2118 } 2119 uhdr = mtodo(m, off); 2120 ohdr = mtodo(m, off + sizeof(*uhdr)); 2121 2122 op = ntohl(ohdr->opcode) >> 24 >> OVPN_OP_SHIFT; 2123 2124 /* 2125 * Simplify things by getting rid of the preceding headers, we don't 2126 * care about them. 2127 */ 2128 m_adj_decap(m, off); 2129 2130 uhdr = mtodo(m, 0); 2131 ohdr = mtodo(m, sizeof(*uhdr)); 2132 2133 if (op != OVPN_OP_DATA_V2) { 2134 OVPN_RUNLOCK(sc); 2135 ovpn_rcv_ctrl(sc, m, sizeof(struct udphdr)); 2136 INP_WLOCK(inp); 2137 udp_notify(inp, EAGAIN); 2138 INP_WUNLOCK(inp); 2139 return (true); 2140 } 2141 2142 key = ovpn_find_key(sc, peer, ohdr); 2143 if (key == NULL || key->decrypt == NULL) { 2144 OVPN_RUNLOCK(sc); 2145 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2146 m_freem(m); 2147 return (true); 2148 } 2149 2150 if (key->decrypt->cipher == OVPN_CIPHER_ALG_NONE) { 2151 /* Now remove the outer headers */ 2152 m_adj_decap(m, sizeof(struct udphdr) + ohdrlen); 2153 2154 ohdr = mtodo(m, sizeof(*uhdr)); 2155 2156 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), 2157 _ovpn_lock_trackerp); 2158 OVPN_UNLOCK_ASSERT(sc); 2159 return (true); 2160 } 2161 2162 ohdrlen += sizeof(ohdr->auth_tag); 2163 2164 m = m_pullup(m, sizeof(*uhdr) + ohdrlen); 2165 if (m == NULL) { 2166 OVPN_RUNLOCK(sc); 2167 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2168 return (true); 2169 } 2170 uhdr = mtodo(m, 0); 2171 ohdr = mtodo(m, sizeof(*uhdr)); 2172 2173 /* Decrypt */ 2174 crp = crypto_getreq(key->decrypt->cryptoid, M_NOWAIT); 2175 if (crp == NULL) { 2176 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2177 OVPN_RUNLOCK(sc); 2178 m_freem(m); 2179 return (true); 2180 } 2181 2182 crp->crp_payload_start = sizeof(struct udphdr) + sizeof(*ohdr); 2183 crp->crp_payload_length = ntohs(uhdr->uh_ulen) - 2184 sizeof(*uhdr) - sizeof(*ohdr); 2185 crp->crp_op = CRYPTO_OP_DECRYPT; 2186 2187 /* AAD validation. */ 2188 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2189 crp->crp_aad = ohdr; 2190 crp->crp_aad_start = 0; 2191 crp->crp_op |= CRYPTO_OP_VERIFY_DIGEST; 2192 crp->crp_digest_start = sizeof(struct udphdr) + 2193 offsetof(struct ovpn_wire_header, auth_tag); 2194 2195 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 2196 memcpy(crp->crp_iv, &ohdr->seq, sizeof(ohdr->seq)); 2197 memcpy(crp->crp_iv + sizeof(ohdr->seq), key->decrypt->nonce, 2198 key->decrypt->noncelen); 2199 2200 crypto_use_mbuf(crp, m); 2201 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 2202 crp->crp_callback = ovpn_decrypt_rx_cb; 2203 crp->crp_opaque = sc; 2204 2205 atomic_add_int(&sc->refcount, 1); 2206 OVPN_RUNLOCK(sc); 2207 ret = crypto_dispatch(crp); 2208 if (ret != 0) { 2209 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2210 } 2211 2212 return (true); 2213 } 2214 2215 static void 2216 ovpn_qflush(struct ifnet *ifp __unused) 2217 { 2218 2219 } 2220 2221 static void 2222 ovpn_flush_rxring(struct ovpn_softc *sc) 2223 { 2224 struct mbuf *m; 2225 struct ovpn_notification *n; 2226 2227 OVPN_WASSERT(sc); 2228 2229 while (! buf_ring_empty(sc->rxring)) { 2230 m = buf_ring_dequeue_sc(sc->rxring); 2231 m_freem(m); 2232 } 2233 2234 while (! buf_ring_empty(sc->notifring)) { 2235 n = buf_ring_dequeue_sc(sc->notifring); 2236 free(n, M_OVPN); 2237 } 2238 } 2239 2240 #ifdef VIMAGE 2241 static void 2242 ovpn_reassign(struct ifnet *ifp, struct vnet *new_vnet __unused, 2243 char *unused __unused) 2244 { 2245 struct ovpn_softc *sc = ifp->if_softc; 2246 int i; 2247 int ret __diagused; 2248 2249 i = 0; 2250 2251 OVPN_WLOCK(sc); 2252 2253 /* Flush keys & configuration. */ 2254 do { 2255 if (sc->peers[i] != NULL) { 2256 ret = _ovpn_del_peer(sc, sc->peers[i]->peerid); 2257 MPASS(ret == 0); 2258 } 2259 i++; 2260 } while (i < OVPN_MAX_PEERS); 2261 2262 ovpn_flush_rxring(sc); 2263 2264 OVPN_WUNLOCK(sc); 2265 } 2266 #endif 2267 2268 static int 2269 ovpn_clone_match(struct if_clone *ifc, const char *name) 2270 { 2271 /* 2272 * Allow all names that start with 'ovpn', specifically because pfSense 2273 * uses ovpnc1 / ovpns2 2274 */ 2275 return (strncmp(ovpnname, name, strlen(ovpnname)) == 0); 2276 } 2277 2278 static int 2279 ovpn_clone_create(struct if_clone *ifc, char *name, size_t len, caddr_t params) 2280 { 2281 struct ovpn_softc *sc; 2282 struct ifnet *ifp; 2283 char *dp; 2284 int error, unit, wildcard; 2285 2286 /* Try to see if a special unit was requested. */ 2287 error = ifc_name2unit(name, &unit); 2288 if (error != 0) 2289 return (error); 2290 wildcard = (unit < 0); 2291 2292 error = ifc_alloc_unit(ifc, &unit); 2293 if (error != 0) 2294 return (error); 2295 2296 /* 2297 * If no unit had been given, we need to adjust the ifName. 2298 */ 2299 for (dp = name; *dp != '\0'; dp++); 2300 if (wildcard) { 2301 error = snprintf(dp, len - (dp - name), "%d", unit); 2302 if (error > len - (dp - name)) { 2303 /* ifName too long. */ 2304 ifc_free_unit(ifc, unit); 2305 return (ENOSPC); 2306 } 2307 dp += error; 2308 } 2309 2310 /* Make sure it doesn't already exist. */ 2311 if (ifunit(name) != NULL) 2312 return (EEXIST); 2313 2314 sc = malloc(sizeof(struct ovpn_softc), M_OVPN, M_WAITOK | M_ZERO); 2315 sc->ifp = if_alloc(IFT_ENC); 2316 rm_init_flags(&sc->lock, "if_ovpn_lock", RM_RECURSE); 2317 sc->refcount = 0; 2318 2319 sc->rxring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL); 2320 sc->notifring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL); 2321 2322 COUNTER_ARRAY_ALLOC(sc->counters, OVPN_COUNTER_SIZE, M_WAITOK); 2323 2324 ifp = sc->ifp; 2325 ifp->if_softc = sc; 2326 strlcpy(ifp->if_xname, name, IFNAMSIZ); 2327 ifp->if_dname = ovpngroupname; 2328 ifp->if_dunit = unit; 2329 2330 ifp->if_addrlen = 0; 2331 ifp->if_mtu = 1428; 2332 ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST; 2333 ifp->if_ioctl = ovpn_ioctl; 2334 ifp->if_transmit = ovpn_transmit; 2335 ifp->if_output = ovpn_output; 2336 ifp->if_qflush = ovpn_qflush; 2337 #ifdef VIMAGE 2338 ifp->if_reassign = ovpn_reassign; 2339 #endif 2340 ifp->if_capabilities |= IFCAP_LINKSTATE; 2341 ifp->if_capenable |= IFCAP_LINKSTATE; 2342 2343 if_attach(ifp); 2344 bpfattach(ifp, DLT_NULL, sizeof(uint32_t)); 2345 2346 return (0); 2347 } 2348 2349 static void 2350 ovpn_clone_destroy_cb(struct epoch_context *ctx) 2351 { 2352 struct ovpn_softc *sc; 2353 2354 sc = __containerof(ctx, struct ovpn_softc, epoch_ctx); 2355 2356 MPASS(sc->peercount == 0); 2357 for (int i = 0; i < OVPN_MAX_PEERS; i++) { 2358 MPASS(sc->peers[i] == NULL); 2359 } 2360 2361 COUNTER_ARRAY_FREE(sc->counters, OVPN_COUNTER_SIZE); 2362 2363 if_free(sc->ifp); 2364 free(sc, M_OVPN); 2365 } 2366 2367 static int 2368 ovpn_clone_destroy(struct if_clone *ifc, struct ifnet *ifp) 2369 { 2370 struct ovpn_softc *sc; 2371 int unit; 2372 int i; 2373 int ret __diagused; 2374 2375 sc = ifp->if_softc; 2376 unit = ifp->if_dunit; 2377 2378 OVPN_WLOCK(sc); 2379 2380 if (atomic_load_int(&sc->refcount) > 0) { 2381 OVPN_WUNLOCK(sc); 2382 return (EBUSY); 2383 } 2384 2385 i = 0; 2386 do { 2387 if (sc->peers[i] != NULL) { 2388 ret = _ovpn_del_peer(sc, sc->peers[i]->peerid); 2389 MPASS(ret == 0); 2390 } 2391 i++; 2392 } while (i < OVPN_MAX_PEERS); 2393 2394 ovpn_flush_rxring(sc); 2395 buf_ring_free(sc->rxring, M_OVPN); 2396 buf_ring_free(sc->notifring, M_OVPN); 2397 2398 OVPN_WUNLOCK(sc); 2399 2400 bpfdetach(ifp); 2401 if_detach(ifp); 2402 ifp->if_softc = NULL; 2403 2404 NET_EPOCH_CALL(ovpn_clone_destroy_cb, &sc->epoch_ctx); 2405 2406 if (unit != IF_DUNIT_NONE) 2407 ifc_free_unit(ifc, unit); 2408 2409 return (0); 2410 } 2411 2412 static void 2413 vnet_ovpn_init(const void *unused __unused) 2414 { 2415 V_ovpn_cloner = if_clone_advanced(ovpngroupname, 0, ovpn_clone_match, 2416 ovpn_clone_create, ovpn_clone_destroy); 2417 } 2418 VNET_SYSINIT(vnet_ovpn_init, SI_SUB_PSEUDO, SI_ORDER_ANY, 2419 vnet_ovpn_init, NULL); 2420 2421 static void 2422 vnet_ovpn_uninit(const void *unused __unused) 2423 { 2424 if_clone_detach(V_ovpn_cloner); 2425 } 2426 VNET_SYSUNINIT(vnet_ovpn_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY, 2427 vnet_ovpn_uninit, NULL); 2428 2429 static int 2430 ovpnmodevent(module_t mod, int type, void *data) 2431 { 2432 switch (type) { 2433 case MOD_LOAD: 2434 /* Done in vnet_ovpn_init() */ 2435 break; 2436 case MOD_UNLOAD: 2437 /* Done in vnet_ovpn_uninit() */ 2438 break; 2439 default: 2440 return (EOPNOTSUPP); 2441 } 2442 2443 return (0); 2444 } 2445 2446 static moduledata_t ovpn_mod = { 2447 "if_ovpn", 2448 ovpnmodevent, 2449 0 2450 }; 2451 2452 DECLARE_MODULE(if_ovpn, ovpn_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 2453 MODULE_VERSION(if_ovpn, 1); 2454