1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2021-2022 Rubicon Communications, LLC (Netgate) 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 */ 28 #include "opt_inet.h" 29 #include "opt_inet6.h" 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/buf_ring.h> 34 #include <sys/epoch.h> 35 #include <sys/file.h> 36 #include <sys/filedesc.h> 37 #include <sys/kernel.h> 38 #include <sys/malloc.h> 39 #include <sys/mbuf.h> 40 #include <sys/module.h> 41 #include <sys/nv.h> 42 #include <sys/priv.h> 43 #include <sys/protosw.h> 44 #include <sys/rmlock.h> 45 #include <sys/sdt.h> 46 #include <sys/smp.h> 47 #include <sys/socket.h> 48 #include <sys/socketvar.h> 49 #include <sys/sockio.h> 50 #include <sys/sysctl.h> 51 #include <sys/time.h> 52 53 #include <machine/atomic.h> 54 55 #include <net/bpf.h> 56 #include <net/if.h> 57 #include <net/if_clone.h> 58 #include <net/if_types.h> 59 #include <net/if_var.h> 60 #include <net/netisr.h> 61 #include <net/route/nhop.h> 62 63 #include <netinet/in.h> 64 #include <netinet/in_fib.h> 65 #include <netinet/ip.h> 66 #include <netinet/ip6.h> 67 #include <netinet/ip_var.h> 68 #include <netinet/udp.h> 69 #include <netinet/udp_var.h> 70 71 #include <netinet6/ip6_var.h> 72 #include <netinet6/in6_fib.h> 73 74 #include <machine/in_cksum.h> 75 76 #include <opencrypto/cryptodev.h> 77 78 #include "if_ovpn.h" 79 80 struct ovpn_kkey_dir { 81 int refcount; 82 uint8_t key[32]; 83 uint8_t keylen; 84 uint8_t nonce[8]; 85 uint8_t noncelen; 86 enum ovpn_key_cipher cipher; 87 crypto_session_t cryptoid; 88 89 struct mtx replay_mtx; 90 /* 91 * Last seen gapless sequence number. New rx seq numbers must be 92 * strictly higher than this. 93 */ 94 uint32_t rx_seq; 95 /* Seen packets, relative to rx_seq. bit(0) will always be 0. */ 96 uint64_t rx_window; 97 }; 98 99 struct ovpn_kkey { 100 struct ovpn_kkey_dir *encrypt; 101 struct ovpn_kkey_dir *decrypt; 102 uint8_t keyid; 103 uint32_t peerid; 104 }; 105 106 struct ovpn_keepalive { 107 uint32_t interval; 108 uint32_t timeout; 109 }; 110 111 struct ovpn_wire_header { 112 uint32_t opcode; /* opcode, key id, peer id */ 113 uint32_t seq; 114 uint8_t auth_tag[16]; 115 }; 116 117 struct ovpn_notification { 118 enum ovpn_notif_type type; 119 uint32_t peerid; 120 }; 121 122 struct ovpn_softc; 123 124 struct ovpn_kpeer { 125 int refcount; 126 uint32_t peerid; 127 128 struct ovpn_softc *sc; 129 struct sockaddr_storage local; 130 struct sockaddr_storage remote; 131 132 struct in_addr vpn4; 133 struct in6_addr vpn6; 134 135 struct ovpn_kkey keys[2]; 136 uint32_t tx_seq; 137 138 struct ovpn_keepalive keepalive; 139 uint32_t *last_active; 140 struct callout ping_send; 141 struct callout ping_rcv; 142 }; 143 144 #define OVPN_MAX_PEERS 128 145 146 struct ovpn_counters { 147 uint64_t lost_ctrl_pkts_in; 148 uint64_t lost_ctrl_pkts_out; 149 uint64_t lost_data_pkts_in; 150 uint64_t lost_data_pkts_out; 151 uint64_t nomem_data_pkts_in; 152 uint64_t nomem_data_pkts_out; 153 uint64_t received_ctrl_pkts; 154 uint64_t received_data_pkts; 155 uint64_t sent_ctrl_pkts; 156 uint64_t sent_data_pkts; 157 158 uint64_t transport_bytes_sent; 159 uint64_t transport_bytes_received; 160 uint64_t tunnel_bytes_sent; 161 uint64_t tunnel_bytes_received; 162 }; 163 #define OVPN_COUNTER_SIZE (sizeof(struct ovpn_counters)/sizeof(uint64_t)) 164 165 struct ovpn_softc { 166 int refcount; 167 struct rmlock lock; 168 struct ifnet *ifp; 169 struct socket *so; 170 int peercount; 171 struct ovpn_kpeer *peers[OVPN_MAX_PEERS]; /* XXX Hard limit for now? */ 172 173 /* Pending packets */ 174 struct buf_ring *rxring; 175 struct buf_ring *notifring; 176 177 counter_u64_t counters[OVPN_COUNTER_SIZE]; 178 179 struct epoch_context epoch_ctx; 180 }; 181 182 static struct ovpn_kpeer *ovpn_find_peer(struct ovpn_softc *, uint32_t); 183 static bool ovpn_udp_input(struct mbuf *, int, struct inpcb *, 184 const struct sockaddr *, void *); 185 static int ovpn_transmit_to_peer(struct ifnet *, struct mbuf *, 186 struct ovpn_kpeer *, struct rm_priotracker *); 187 static int ovpn_encap(struct ovpn_softc *, uint32_t, struct mbuf *); 188 static int ovpn_get_af(struct mbuf *); 189 static void ovpn_free_kkey_dir(struct ovpn_kkey_dir *); 190 static bool ovpn_check_replay(struct ovpn_kkey_dir *, uint32_t); 191 192 #define OVPN_MTU_MIN 576 193 #define OVPN_MTU_MAX (IP_MAXPACKET - sizeof(struct ip) - \ 194 sizeof(struct udphdr) - sizeof(struct ovpn_wire_header)) 195 196 #define OVPN_OP_DATA_V2 0x09 197 #define OVPN_OP_SHIFT 3 198 199 VNET_DEFINE_STATIC(struct if_clone *, ovpn_cloner); 200 #define V_ovpn_cloner VNET(ovpn_cloner) 201 202 #define OVPN_RLOCK_TRACKER struct rm_priotracker _ovpn_lock_tracker; \ 203 struct rm_priotracker *_ovpn_lock_trackerp = &_ovpn_lock_tracker 204 #define OVPN_RLOCK(sc) rm_rlock(&(sc)->lock, _ovpn_lock_trackerp) 205 #define OVPN_RUNLOCK(sc) rm_runlock(&(sc)->lock, _ovpn_lock_trackerp) 206 #define OVPN_WLOCK(sc) rm_wlock(&(sc)->lock) 207 #define OVPN_WUNLOCK(sc) rm_wunlock(&(sc)->lock) 208 #define OVPN_ASSERT(sc) rm_assert(&(sc)->lock, RA_LOCKED) 209 #define OVPN_RASSERT(sc) rm_assert(&(sc)->lock, RA_RLOCKED) 210 #define OVPN_WASSERT(sc) rm_assert(&(sc)->lock, RA_WLOCKED) 211 #define OVPN_UNLOCK_ASSERT(sc) rm_assert(&(sc)->lock, RA_UNLOCKED) 212 213 #define OVPN_COUNTER_ADD(sc, name, val) \ 214 counter_u64_add(sc->counters[offsetof(struct ovpn_counters, name) / \ 215 sizeof(uint64_t)], val) 216 217 #define TO_IN(x) ((struct sockaddr_in *)(x)) 218 #define TO_IN6(x) ((struct sockaddr_in6 *)(x)) 219 220 SDT_PROVIDER_DEFINE(if_ovpn); 221 SDT_PROBE_DEFINE1(if_ovpn, tx, transmit, start, "struct mbuf *"); 222 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip4, "struct in_addr *", "struct ovpn_kpeer *"); 223 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip6, "struct in6_addr *", "struct ovpn_kpeer *"); 224 225 static const char ovpnname[] = "ovpn"; 226 static const char ovpngroupname[] = "openvpn"; 227 228 static MALLOC_DEFINE(M_OVPN, ovpnname, "OpenVPN DCO Interface"); 229 230 SYSCTL_DECL(_net_link); 231 static SYSCTL_NODE(_net_link, IFT_OTHER, openvpn, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 232 "OpenVPN DCO Interface"); 233 VNET_DEFINE_STATIC(int, replay_protection) = 0; 234 #define V_replay_protection VNET(replay_protection) 235 SYSCTL_INT(_net_link_openvpn, OID_AUTO, replay_protection, CTLFLAG_VNET | CTLFLAG_RW, 236 &VNET_NAME(replay_protection), 0, "Validate sequence numbers"); 237 238 VNET_DEFINE_STATIC(int, async_crypto); 239 #define V_async_crypto VNET(async_crypto) 240 SYSCTL_INT(_net_link_openvpn, OID_AUTO, async_crypto, 241 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_crypto), 0, 242 "Use asynchronous mode to parallelize crypto jobs."); 243 244 VNET_DEFINE_STATIC(int, async_netisr_queue); 245 #define V_async_netisr_queue VNET(async_netisr_queue) 246 SYSCTL_INT(_net_link_openvpn, OID_AUTO, netisr_queue, 247 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_netisr_queue), 0, 248 "Use netisr_queue() rather than netisr_dispatch()."); 249 250 static struct ovpn_kpeer * 251 ovpn_find_peer(struct ovpn_softc *sc, uint32_t peerid) 252 { 253 struct ovpn_kpeer *p = NULL; 254 255 OVPN_ASSERT(sc); 256 257 for (int i = 0; i < OVPN_MAX_PEERS; i++) { 258 p = sc->peers[i]; 259 if (p == NULL) 260 continue; 261 262 if (p->peerid == peerid) { 263 MPASS(p->sc == sc); 264 break; 265 } 266 } 267 268 return (p); 269 } 270 271 static struct ovpn_kpeer * 272 ovpn_find_only_peer(struct ovpn_softc *sc) 273 { 274 OVPN_ASSERT(sc); 275 276 for (int i = 0; i < OVPN_MAX_PEERS; i++) { 277 if (sc->peers[i] == NULL) 278 continue; 279 return (sc->peers[i]); 280 } 281 282 MPASS(false); 283 284 return (NULL); 285 } 286 287 static uint16_t 288 ovpn_get_port(struct sockaddr_storage *s) 289 { 290 switch (s->ss_family) { 291 case AF_INET: { 292 struct sockaddr_in *in = (struct sockaddr_in *)s; 293 return (in->sin_port); 294 } 295 case AF_INET6: { 296 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)s; 297 return (in6->sin6_port); 298 } 299 default: 300 panic("Unsupported address family %d", s->ss_family); 301 } 302 } 303 304 static int 305 ovpn_nvlist_to_sockaddr(const nvlist_t *nvl, struct sockaddr_storage *sa) 306 { 307 int af; 308 309 if (! nvlist_exists_number(nvl, "af")) 310 return (EINVAL); 311 if (! nvlist_exists_binary(nvl, "address")) 312 return (EINVAL); 313 if (! nvlist_exists_number(nvl, "port")) 314 return (EINVAL); 315 316 af = nvlist_get_number(nvl, "af"); 317 318 switch (af) { 319 #ifdef INET 320 case AF_INET: { 321 struct sockaddr_in *in = (struct sockaddr_in *)sa; 322 size_t len; 323 const void *addr = nvlist_get_binary(nvl, "address", &len); 324 in->sin_family = af; 325 if (len != sizeof(in->sin_addr)) 326 return (EINVAL); 327 328 memcpy(&in->sin_addr, addr, sizeof(in->sin_addr)); 329 in->sin_port = nvlist_get_number(nvl, "port"); 330 break; 331 } 332 #endif 333 #ifdef INET6 334 case AF_INET6: { 335 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)sa; 336 size_t len; 337 const void *addr = nvlist_get_binary(nvl, "address", &len); 338 in6->sin6_family = af; 339 if (len != sizeof(in6->sin6_addr)) 340 return (EINVAL); 341 342 memcpy(&in6->sin6_addr, addr, sizeof(in6->sin6_addr)); 343 in6->sin6_port = nvlist_get_number(nvl, "port"); 344 break; 345 } 346 #endif 347 default: 348 return (EINVAL); 349 } 350 351 return (0); 352 } 353 354 static bool 355 ovpn_has_peers(struct ovpn_softc *sc) 356 { 357 OVPN_ASSERT(sc); 358 359 return (sc->peercount > 0); 360 } 361 362 static void 363 ovpn_rele_so(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 364 { 365 bool has_peers; 366 367 OVPN_WASSERT(sc); 368 369 if (sc->so == NULL) 370 return; 371 372 has_peers = ovpn_has_peers(sc); 373 374 /* Only remove the tunnel function if we're releasing the socket for 375 * the last peer. */ 376 if (! has_peers) 377 (void)udp_set_kernel_tunneling(sc->so, NULL, NULL, NULL); 378 379 sorele(sc->so); 380 381 if (! has_peers) 382 sc->so = NULL; 383 } 384 385 static void 386 ovpn_notify_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 387 { 388 struct ovpn_notification *n; 389 390 OVPN_WASSERT(sc); 391 392 n = malloc(sizeof(*n), M_OVPN, M_NOWAIT); 393 if (n == NULL) 394 return; 395 396 n->peerid = peer->peerid; 397 n->type = OVPN_NOTIF_DEL_PEER; 398 if (buf_ring_enqueue(sc->notifring, n) != 0) { 399 free(n, M_OVPN); 400 } else if (sc->so != NULL) { 401 /* Wake up userspace */ 402 sc->so->so_error = EAGAIN; 403 sorwakeup(sc->so); 404 sowwakeup(sc->so); 405 } 406 } 407 408 static void 409 ovpn_peer_release_ref(struct ovpn_kpeer *peer, bool locked) 410 { 411 struct ovpn_softc *sc; 412 413 atomic_add_int(&peer->refcount, -1); 414 415 if (atomic_load_int(&peer->refcount) > 0) 416 return; 417 418 sc = peer->sc; 419 420 if (! locked) { 421 OVPN_WLOCK(sc); 422 423 /* Might have changed before we acquired the lock. */ 424 if (atomic_load_int(&peer->refcount) > 0) { 425 OVPN_WUNLOCK(sc); 426 return; 427 } 428 } 429 430 /* The peer should have been removed from the list already. */ 431 MPASS(ovpn_find_peer(sc, peer->peerid) == NULL); 432 433 ovpn_notify_del_peer(sc, peer); 434 435 for (int i = 0; i < 2; i++) { 436 ovpn_free_kkey_dir(peer->keys[i].encrypt); 437 ovpn_free_kkey_dir(peer->keys[i].decrypt); 438 } 439 440 ovpn_rele_so(sc, peer); 441 442 callout_stop(&peer->ping_send); 443 callout_stop(&peer->ping_rcv); 444 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 445 free(peer, M_OVPN); 446 447 if (! locked) 448 OVPN_WUNLOCK(sc); 449 } 450 451 static int 452 ovpn_new_peer(struct ifnet *ifp, const nvlist_t *nvl) 453 { 454 #ifdef INET6 455 struct epoch_tracker et; 456 #endif 457 struct sockaddr_storage remote; 458 struct ovpn_kpeer *peer = NULL; 459 struct file *fp = NULL; 460 struct sockaddr *name = NULL; 461 struct ovpn_softc *sc = ifp->if_softc; 462 struct thread *td = curthread; 463 struct socket *so = NULL; 464 int fd; 465 uint32_t peerid; 466 int ret = 0, i; 467 468 if (nvl == NULL) 469 return (EINVAL); 470 471 if (! nvlist_exists_number(nvl, "peerid")) 472 return (EINVAL); 473 474 if (! nvlist_exists_number(nvl, "fd")) 475 return (EINVAL); 476 477 if (! nvlist_exists_nvlist(nvl, "remote")) 478 return (EINVAL); 479 480 peerid = nvlist_get_number(nvl, "peerid"); 481 482 ret = ovpn_nvlist_to_sockaddr(nvlist_get_nvlist(nvl, "remote"), 483 &remote); 484 if (ret != 0) 485 return (ret); 486 487 fd = nvlist_get_number(nvl, "fd"); 488 489 /* Look up the userspace process and use the fd to find the socket. */ 490 ret = getsock(td, fd, &cap_connect_rights, &fp); 491 if (ret != 0) 492 return (ret); 493 494 so = fp->f_data; 495 496 peer = malloc(sizeof(*peer), M_OVPN, M_WAITOK | M_ZERO); 497 peer->peerid = peerid; 498 peer->sc = sc; 499 peer->tx_seq = 1; 500 peer->refcount = 1; 501 peer->last_active = uma_zalloc_pcpu(pcpu_zone_4, M_WAITOK | M_ZERO); 502 503 if (nvlist_exists_binary(nvl, "vpn_ipv4")) { 504 size_t len; 505 const void *addr = nvlist_get_binary(nvl, "vpn_ipv4", &len); 506 if (len != sizeof(peer->vpn4)) { 507 ret = EINVAL; 508 goto error; 509 } 510 memcpy(&peer->vpn4, addr, len); 511 } 512 513 if (nvlist_exists_binary(nvl, "vpn_ipv6")) { 514 size_t len; 515 const void *addr = nvlist_get_binary(nvl, "vpn_ipv6", &len); 516 if (len != sizeof(peer->vpn6)) { 517 ret = EINVAL; 518 goto error; 519 } 520 memcpy(&peer->vpn6, addr, len); 521 } 522 523 callout_init_rm(&peer->ping_send, &sc->lock, CALLOUT_SHAREDLOCK); 524 callout_init_rm(&peer->ping_rcv, &sc->lock, 0); 525 526 ret = so->so_proto->pr_sockaddr(so, &name); 527 if (ret) 528 goto error; 529 530 if (ovpn_get_port((struct sockaddr_storage *)name) == 0) { 531 ret = EINVAL; 532 goto error; 533 } 534 if (name->sa_family != remote.ss_family) { 535 ret = EINVAL; 536 goto error; 537 } 538 539 memcpy(&peer->local, name, name->sa_len); 540 memcpy(&peer->remote, &remote, sizeof(remote)); 541 free(name, M_SONAME); 542 name = NULL; 543 544 if (peer->local.ss_family == AF_INET6 && 545 IN6_IS_ADDR_V4MAPPED(&TO_IN6(&peer->remote)->sin6_addr)) { 546 /* V4 mapped address, so treat this as v4, not v6. */ 547 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->local); 548 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->remote); 549 } 550 551 #ifdef INET6 552 if (peer->local.ss_family == AF_INET6 && 553 IN6_IS_ADDR_UNSPECIFIED(&TO_IN6(&peer->local)->sin6_addr)) { 554 NET_EPOCH_ENTER(et); 555 ret = in6_selectsrc_addr(curthread->td_proc->p_fibnum, 556 &TO_IN6(&peer->remote)->sin6_addr, 557 0, NULL, &TO_IN6(&peer->local)->sin6_addr, NULL); 558 NET_EPOCH_EXIT(et); 559 if (ret != 0) { 560 goto error; 561 } 562 } 563 #endif 564 OVPN_WLOCK(sc); 565 566 /* Disallow peer id re-use. */ 567 if (ovpn_find_peer(sc, peerid) != NULL) { 568 ret = EEXIST; 569 goto error_locked; 570 } 571 572 /* Make sure this is really a UDP socket. */ 573 if (so->so_type != SOCK_DGRAM || so->so_proto->pr_type != SOCK_DGRAM) { 574 ret = EPROTOTYPE; 575 goto error_locked; 576 } 577 578 /* Must be the same socket as for other peers on this interface. */ 579 if (sc->so != NULL && so != sc->so) 580 goto error_locked; 581 582 if (sc->so == NULL) 583 sc->so = so; 584 585 /* Insert the peer into the list. */ 586 for (i = 0; i < OVPN_MAX_PEERS; i++) { 587 if (sc->peers[i] != NULL) 588 continue; 589 590 MPASS(sc->peers[i] == NULL); 591 sc->peers[i] = peer; 592 sc->peercount++; 593 soref(sc->so); 594 break; 595 } 596 if (i == OVPN_MAX_PEERS) { 597 ret = ENOSPC; 598 goto error_locked; 599 } 600 601 ret = udp_set_kernel_tunneling(sc->so, ovpn_udp_input, NULL, sc); 602 if (ret == EBUSY) { 603 /* Fine, another peer already set the input function. */ 604 ret = 0; 605 } 606 if (ret != 0) { 607 sc->peers[i] = NULL; 608 sc->peercount--; 609 goto error_locked; 610 } 611 612 OVPN_WUNLOCK(sc); 613 614 goto done; 615 616 error_locked: 617 OVPN_WUNLOCK(sc); 618 error: 619 free(name, M_SONAME); 620 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 621 free(peer, M_OVPN); 622 done: 623 if (fp != NULL) 624 fdrop(fp, td); 625 626 return (ret); 627 } 628 629 static int 630 _ovpn_del_peer(struct ovpn_softc *sc, uint32_t peerid) 631 { 632 struct ovpn_kpeer *peer; 633 int i; 634 635 OVPN_WASSERT(sc); 636 637 for (i = 0; i < OVPN_MAX_PEERS; i++) { 638 if (sc->peers[i] == NULL) 639 continue; 640 if (sc->peers[i]->peerid != peerid) 641 continue; 642 643 peer = sc->peers[i]; 644 break; 645 } 646 if (i == OVPN_MAX_PEERS) 647 return (ENOENT); 648 649 sc->peers[i] = NULL; 650 sc->peercount--; 651 652 ovpn_peer_release_ref(peer, true); 653 654 return (0); 655 } 656 657 static int 658 ovpn_del_peer(struct ifnet *ifp, nvlist_t *nvl) 659 { 660 struct ovpn_softc *sc = ifp->if_softc; 661 uint32_t peerid; 662 int ret; 663 664 OVPN_WASSERT(sc); 665 666 if (nvl == NULL) 667 return (EINVAL); 668 669 if (! nvlist_exists_number(nvl, "peerid")) 670 return (EINVAL); 671 672 peerid = nvlist_get_number(nvl, "peerid"); 673 674 ret = _ovpn_del_peer(sc, peerid); 675 676 return (ret); 677 } 678 679 static int 680 ovpn_create_kkey_dir(struct ovpn_kkey_dir **kdirp, 681 const nvlist_t *nvl) 682 { 683 struct crypto_session_params csp; 684 struct ovpn_kkey_dir *kdir; 685 const char *ciphername; 686 enum ovpn_key_cipher cipher; 687 const void *key, *iv; 688 size_t keylen = 0, ivlen = 0; 689 int error; 690 691 if (! nvlist_exists_string(nvl, "cipher")) 692 return (EINVAL); 693 ciphername = nvlist_get_string(nvl, "cipher"); 694 695 if (strcmp(ciphername, "none") == 0) 696 cipher = OVPN_CIPHER_ALG_NONE; 697 else if (strcmp(ciphername, "AES-256-GCM") == 0 || 698 strcmp(ciphername, "AES-192-GCM") == 0 || 699 strcmp(ciphername, "AES-128-GCM") == 0) 700 cipher = OVPN_CIPHER_ALG_AES_GCM; 701 else if (strcmp(ciphername, "CHACHA20-POLY1305") == 0) 702 cipher = OVPN_CIPHER_ALG_CHACHA20_POLY1305; 703 else 704 return (EINVAL); 705 706 if (cipher != OVPN_CIPHER_ALG_NONE) { 707 if (! nvlist_exists_binary(nvl, "key")) 708 return (EINVAL); 709 key = nvlist_get_binary(nvl, "key", &keylen); 710 if (keylen > sizeof(kdir->key)) 711 return (E2BIG); 712 713 if (! nvlist_exists_binary(nvl, "iv")) 714 return (EINVAL); 715 iv = nvlist_get_binary(nvl, "iv", &ivlen); 716 if (ivlen != 8) 717 return (E2BIG); 718 } 719 720 kdir = malloc(sizeof(struct ovpn_kkey_dir), M_OVPN, 721 M_WAITOK | M_ZERO); 722 723 kdir->cipher = cipher; 724 kdir->keylen = keylen; 725 memcpy(kdir->key, key, keylen); 726 kdir->noncelen = ivlen; 727 memcpy(kdir->nonce, iv, ivlen); 728 729 if (kdir->cipher != OVPN_CIPHER_ALG_NONE) { 730 /* Crypto init */ 731 bzero(&csp, sizeof(csp)); 732 csp.csp_mode = CSP_MODE_AEAD; 733 734 if (kdir->cipher == OVPN_CIPHER_ALG_CHACHA20_POLY1305) 735 csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305; 736 else 737 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 738 739 csp.csp_flags |= CSP_F_SEPARATE_AAD; 740 741 csp.csp_cipher_klen = kdir->keylen; 742 csp.csp_cipher_key = kdir->key; 743 csp.csp_ivlen = 96 / 8; 744 745 error = crypto_newsession(&kdir->cryptoid, &csp, 746 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE); 747 if (error) { 748 free(kdir, M_OVPN); 749 return (error); 750 } 751 } 752 753 mtx_init(&kdir->replay_mtx, "if_ovpn rx replay", NULL, MTX_DEF); 754 *kdirp = kdir; 755 756 return (0); 757 } 758 759 static void 760 ovpn_free_kkey_dir(struct ovpn_kkey_dir *kdir) 761 { 762 if (kdir == NULL) 763 return; 764 765 mtx_destroy(&kdir->replay_mtx); 766 767 crypto_freesession(kdir->cryptoid); 768 free(kdir, M_OVPN); 769 } 770 771 static int 772 ovpn_set_key(struct ifnet *ifp, const nvlist_t *nvl) 773 { 774 struct ovpn_softc *sc = ifp->if_softc; 775 struct ovpn_kkey_dir *enc, *dec; 776 struct ovpn_kpeer *peer; 777 int slot, keyid, peerid; 778 int error; 779 780 if (nvl == NULL) 781 return (EINVAL); 782 783 if (! nvlist_exists_number(nvl, "slot")) 784 return (EINVAL); 785 slot = nvlist_get_number(nvl, "slot"); 786 787 if (! nvlist_exists_number(nvl, "keyid")) 788 return (EINVAL); 789 keyid = nvlist_get_number(nvl, "keyid"); 790 791 if (! nvlist_exists_number(nvl, "peerid")) 792 return (EINVAL); 793 peerid = nvlist_get_number(nvl, "peerid"); 794 795 if (slot != OVPN_KEY_SLOT_PRIMARY && 796 slot != OVPN_KEY_SLOT_SECONDARY) 797 return (EINVAL); 798 799 if (! nvlist_exists_nvlist(nvl, "encrypt") || 800 ! nvlist_exists_nvlist(nvl, "decrypt")) 801 return (EINVAL); 802 803 error = ovpn_create_kkey_dir(&enc, nvlist_get_nvlist(nvl, "encrypt")); 804 if (error) 805 return (error); 806 807 error = ovpn_create_kkey_dir(&dec, nvlist_get_nvlist(nvl, "decrypt")); 808 if (error) { 809 ovpn_free_kkey_dir(enc); 810 return (error); 811 } 812 813 OVPN_WLOCK(sc); 814 815 peer = ovpn_find_peer(sc, peerid); 816 if (peer == NULL) { 817 ovpn_free_kkey_dir(dec); 818 ovpn_free_kkey_dir(enc); 819 OVPN_WUNLOCK(sc); 820 return (ENOENT); 821 } 822 823 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 824 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 825 826 peer->keys[slot].encrypt = enc; 827 peer->keys[slot].decrypt = dec; 828 829 peer->keys[slot].keyid = keyid; 830 peer->keys[slot].peerid = peerid; 831 832 OVPN_WUNLOCK(sc); 833 834 return (0); 835 } 836 837 static int 838 ovpn_check_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, enum ovpn_key_slot slot) 839 { 840 OVPN_ASSERT(sc); 841 842 if (peer->keys[slot].encrypt == NULL) 843 return (ENOLINK); 844 845 if (peer->keys[slot].decrypt == NULL) 846 return (ENOLINK); 847 848 return (0); 849 } 850 851 static int 852 ovpn_start(struct ifnet *ifp) 853 { 854 struct ovpn_softc *sc = ifp->if_softc; 855 856 OVPN_WLOCK(sc); 857 858 ifp->if_flags |= IFF_UP; 859 ifp->if_drv_flags |= IFF_DRV_RUNNING; 860 if_link_state_change(ifp, LINK_STATE_UP); 861 862 OVPN_WUNLOCK(sc); 863 864 return (0); 865 } 866 867 static int 868 ovpn_swap_keys(struct ifnet *ifp, nvlist_t *nvl) 869 { 870 struct ovpn_softc *sc = ifp->if_softc; 871 struct ovpn_kpeer *peer; 872 struct ovpn_kkey tmpkey; 873 int error; 874 875 if (nvl == NULL) 876 return (EINVAL); 877 878 if (! nvlist_exists_number(nvl, "peerid")) 879 return (EINVAL); 880 881 OVPN_WLOCK(sc); 882 883 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 884 if (peer == NULL) { 885 OVPN_WUNLOCK(sc); 886 return (ENOENT); 887 } 888 889 /* Check that we have a second key to swap to. */ 890 error = ovpn_check_key(sc, peer, OVPN_KEY_SLOT_SECONDARY); 891 if (error) { 892 OVPN_WUNLOCK(sc); 893 return (error); 894 } 895 896 tmpkey = peer->keys[0]; 897 peer->keys[0] = peer->keys[1]; 898 peer->keys[1] = tmpkey; 899 900 OVPN_WUNLOCK(sc); 901 902 return (0); 903 } 904 905 static int 906 ovpn_del_key(struct ifnet *ifp, const nvlist_t *nvl) 907 { 908 enum ovpn_key_slot slot; 909 struct ovpn_kpeer *peer; 910 struct ovpn_softc *sc = ifp->if_softc; 911 912 if (nvl == NULL) 913 return (EINVAL); 914 915 if (! nvlist_exists_number(nvl, "peerid")) 916 return (EINVAL); 917 918 if (! nvlist_exists_number(nvl, "slot")) 919 return (EINVAL); 920 slot = nvlist_get_number(nvl, "slot"); 921 922 if (slot != OVPN_KEY_SLOT_PRIMARY && 923 slot != OVPN_KEY_SLOT_SECONDARY) 924 return (EINVAL); 925 926 OVPN_WLOCK(sc); 927 928 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 929 if (peer == NULL) { 930 OVPN_WUNLOCK(sc); 931 return (ENOENT); 932 } 933 934 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 935 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 936 937 peer->keys[slot].encrypt = NULL; 938 peer->keys[slot].decrypt = NULL; 939 940 peer->keys[slot].keyid = 0; 941 peer->keys[slot].peerid = 0; 942 943 OVPN_WUNLOCK(sc); 944 945 return (0); 946 } 947 948 static int 949 ovpn_send_pkt(struct ifnet *ifp, const nvlist_t *nvl) 950 { 951 struct epoch_tracker et; 952 struct ovpn_softc *sc = ifp->if_softc; 953 struct mbuf *m; 954 const uint8_t *pkt; 955 size_t pktlen; 956 uint32_t peerid; 957 int ret; 958 959 if (nvl == NULL) 960 return (EINVAL); 961 962 if (! nvlist_exists_binary(nvl, "packet")) 963 return (EINVAL); 964 pkt = nvlist_get_binary(nvl, "packet", &pktlen); 965 966 if (! nvlist_exists_number(nvl, "peerid")) 967 return (EINVAL); 968 969 peerid = nvlist_get_number(nvl, "peerid"); 970 971 /* 972 * Check that userspace isn't giving us a data packet. That might lead 973 * to IV re-use, which would be bad. 974 */ 975 if ((pkt[0] >> OVPN_OP_SHIFT) == OVPN_OP_DATA_V2) 976 return (EINVAL); 977 978 m = m_get2(pktlen, M_WAITOK, MT_DATA, M_PKTHDR); 979 if (m == NULL) 980 return (ENOMEM); 981 982 m->m_len = m->m_pkthdr.len = pktlen; 983 m_copyback(m, 0, m->m_len, pkt); 984 985 /* Now prepend IP/UDP headers and transmit the mbuf. */ 986 NET_EPOCH_ENTER(et); 987 ret = ovpn_encap(sc, peerid, m); 988 NET_EPOCH_EXIT(et); 989 if (ret == 0) 990 OVPN_COUNTER_ADD(sc, sent_ctrl_pkts, 1); 991 else 992 OVPN_COUNTER_ADD(sc, lost_ctrl_pkts_out, 1); 993 994 return (ret); 995 } 996 997 static void 998 ovpn_send_ping(void *arg) 999 { 1000 static const uint8_t ping_str[] = { 1001 0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb, 1002 0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48 1003 }; 1004 1005 struct epoch_tracker et; 1006 struct ovpn_kpeer *peer = arg; 1007 struct ovpn_softc *sc = peer->sc; 1008 struct mbuf *m; 1009 1010 OVPN_RASSERT(sc); 1011 1012 /* Ensure we repeat! */ 1013 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 1014 ovpn_send_ping, peer); 1015 1016 m = m_get2(sizeof(ping_str), M_NOWAIT, MT_DATA, M_PKTHDR); 1017 if (m == NULL) 1018 return; 1019 1020 m_copyback(m, 0, sizeof(ping_str), ping_str); 1021 m->m_len = m->m_pkthdr.len = sizeof(ping_str); 1022 1023 CURVNET_SET(sc->ifp->if_vnet); 1024 NET_EPOCH_ENTER(et); 1025 (void)ovpn_transmit_to_peer(sc->ifp, m, peer, NULL); 1026 NET_EPOCH_EXIT(et); 1027 CURVNET_RESTORE(); 1028 } 1029 1030 static void 1031 ovpn_timeout(void *arg) 1032 { 1033 struct ovpn_kpeer *peer = arg; 1034 struct ovpn_softc *sc = peer->sc; 1035 uint32_t last, _last_active; 1036 int ret __diagused; 1037 int cpu; 1038 1039 OVPN_WASSERT(sc); 1040 1041 last = 0; 1042 CPU_FOREACH(cpu) { 1043 _last_active = *zpcpu_get_cpu(peer->last_active, cpu); 1044 if (_last_active > last) 1045 last = _last_active; 1046 } 1047 1048 if (last + peer->keepalive.timeout > time_uptime) { 1049 callout_reset(&peer->ping_rcv, 1050 (peer->keepalive.timeout - (time_uptime - last)) * hz, 1051 ovpn_timeout, peer); 1052 return; 1053 } 1054 1055 CURVNET_SET(sc->ifp->if_vnet); 1056 ret = _ovpn_del_peer(sc, peer->peerid); 1057 MPASS(ret == 0); 1058 CURVNET_RESTORE(); 1059 } 1060 1061 static int 1062 ovpn_set_peer(struct ifnet *ifp, const nvlist_t *nvl) 1063 { 1064 struct ovpn_softc *sc = ifp->if_softc; 1065 struct ovpn_kpeer *peer; 1066 1067 if (nvl == NULL) 1068 return (EINVAL); 1069 1070 if (! nvlist_exists_number(nvl, "interval") || 1071 ! nvlist_exists_number(nvl, "timeout") || 1072 ! nvlist_exists_number(nvl, "peerid")) 1073 return (EINVAL); 1074 1075 OVPN_WLOCK(sc); 1076 1077 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 1078 if (peer == NULL) { 1079 OVPN_WUNLOCK(sc); 1080 return (ENOENT); 1081 } 1082 1083 peer->keepalive.interval = nvlist_get_number(nvl, "interval"); 1084 peer->keepalive.timeout = nvlist_get_number(nvl, "timeout"); 1085 1086 if (peer->keepalive.interval > 0) 1087 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 1088 ovpn_send_ping, peer); 1089 if (peer->keepalive.timeout > 0) 1090 callout_reset(&peer->ping_rcv, peer->keepalive.timeout * hz, 1091 ovpn_timeout, peer); 1092 1093 OVPN_WUNLOCK(sc); 1094 1095 return (0); 1096 } 1097 1098 static int 1099 ovpn_set_ifmode(struct ifnet *ifp, const nvlist_t *nvl) 1100 { 1101 struct ovpn_softc *sc = ifp->if_softc; 1102 int ifmode; 1103 1104 if (nvl == NULL) 1105 return (EINVAL); 1106 1107 if (! nvlist_exists_number(nvl, "ifmode") ) 1108 return (EINVAL); 1109 1110 ifmode = nvlist_get_number(nvl, "ifmode"); 1111 1112 OVPN_WLOCK(sc); 1113 1114 /* deny this if UP */ 1115 if (ifp->if_flags & IFF_UP) { 1116 OVPN_WUNLOCK(sc); 1117 return (EBUSY); 1118 } 1119 1120 switch (ifmode & ~IFF_MULTICAST) { 1121 case IFF_POINTOPOINT: 1122 case IFF_BROADCAST: 1123 ifp->if_flags &= 1124 ~(IFF_BROADCAST|IFF_POINTOPOINT|IFF_MULTICAST); 1125 ifp->if_flags |= ifmode; 1126 break; 1127 default: 1128 OVPN_WUNLOCK(sc); 1129 return (EINVAL); 1130 } 1131 1132 OVPN_WUNLOCK(sc); 1133 1134 return (0); 1135 } 1136 1137 static int 1138 ovpn_ioctl_set(struct ifnet *ifp, struct ifdrv *ifd) 1139 { 1140 struct ovpn_softc *sc = ifp->if_softc; 1141 uint8_t *buf = NULL; 1142 nvlist_t *nvl = NULL; 1143 int ret; 1144 1145 if (ifd->ifd_len != 0) { 1146 if (ifd->ifd_len > OVPN_MAX_REQUEST_SIZE) 1147 return (E2BIG); 1148 1149 buf = malloc(ifd->ifd_len, M_OVPN, M_WAITOK); 1150 1151 ret = copyin(ifd->ifd_data, buf, ifd->ifd_len); 1152 if (ret != 0) { 1153 free(buf, M_OVPN); 1154 return (ret); 1155 } 1156 1157 nvl = nvlist_unpack(buf, ifd->ifd_len, 0); 1158 free(buf, M_OVPN); 1159 if (nvl == NULL) { 1160 return (EINVAL); 1161 } 1162 } 1163 1164 switch (ifd->ifd_cmd) { 1165 case OVPN_NEW_PEER: 1166 ret = ovpn_new_peer(ifp, nvl); 1167 break; 1168 case OVPN_DEL_PEER: 1169 OVPN_WLOCK(sc); 1170 ret = ovpn_del_peer(ifp, nvl); 1171 OVPN_WUNLOCK(sc); 1172 break; 1173 case OVPN_NEW_KEY: 1174 ret = ovpn_set_key(ifp, nvl); 1175 break; 1176 case OVPN_START_VPN: 1177 ret = ovpn_start(ifp); 1178 break; 1179 case OVPN_SWAP_KEYS: 1180 ret = ovpn_swap_keys(ifp, nvl); 1181 break; 1182 case OVPN_DEL_KEY: 1183 ret = ovpn_del_key(ifp, nvl); 1184 break; 1185 case OVPN_SEND_PKT: 1186 ret = ovpn_send_pkt(ifp, nvl); 1187 break; 1188 case OVPN_SET_PEER: 1189 ret = ovpn_set_peer(ifp, nvl); 1190 break; 1191 case OVPN_SET_IFMODE: 1192 ret = ovpn_set_ifmode(ifp, nvl); 1193 break; 1194 default: 1195 ret = ENOTSUP; 1196 } 1197 1198 nvlist_destroy(nvl); 1199 return (ret); 1200 } 1201 1202 static int 1203 ovpn_add_counters(nvlist_t *parent, const char *name, counter_u64_t in, 1204 counter_u64_t out) 1205 { 1206 nvlist_t *nvl; 1207 1208 nvl = nvlist_create(0); 1209 if (nvl == NULL) 1210 return (ENOMEM); 1211 1212 nvlist_add_number(nvl, "in", counter_u64_fetch(in)); 1213 nvlist_add_number(nvl, "out", counter_u64_fetch(out)); 1214 1215 nvlist_add_nvlist(parent, name, nvl); 1216 1217 nvlist_destroy(nvl); 1218 1219 return (0); 1220 } 1221 1222 static int 1223 ovpn_get_stats(struct ovpn_softc *sc, nvlist_t **onvl) 1224 { 1225 nvlist_t *nvl; 1226 int ret; 1227 1228 nvl = nvlist_create(0); 1229 if (nvl == NULL) 1230 return (ENOMEM); 1231 1232 #define OVPN_COUNTER_OUT(name, in, out) \ 1233 do { \ 1234 ret = ovpn_add_counters(nvl, name, \ 1235 sc->counters[offsetof(struct ovpn_counters, in) / \ 1236 sizeof(uint64_t)], \ 1237 sc->counters[offsetof(struct ovpn_counters, out) / \ 1238 sizeof(uint64_t)]); \ 1239 if (ret != 0) \ 1240 goto error; \ 1241 } while(0) 1242 1243 OVPN_COUNTER_OUT("lost_ctrl", lost_ctrl_pkts_in, lost_ctrl_pkts_out); 1244 OVPN_COUNTER_OUT("lost_data", lost_data_pkts_in, lost_data_pkts_out); 1245 OVPN_COUNTER_OUT("nomem_data", nomem_data_pkts_in, 1246 nomem_data_pkts_out); 1247 OVPN_COUNTER_OUT("data", received_data_pkts, sent_data_pkts); 1248 OVPN_COUNTER_OUT("ctrl", received_ctrl_pkts, sent_ctrl_pkts); 1249 OVPN_COUNTER_OUT("tunnel", tunnel_bytes_received, 1250 tunnel_bytes_received); 1251 OVPN_COUNTER_OUT("transport", transport_bytes_received, 1252 transport_bytes_received); 1253 #undef OVPN_COUNTER_OUT 1254 1255 *onvl = nvl; 1256 1257 return (0); 1258 1259 error: 1260 nvlist_destroy(nvl); 1261 return (ret); 1262 } 1263 1264 static int 1265 ovpn_poll_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1266 { 1267 nvlist_t *nvl; 1268 1269 nvl = nvlist_create(0); 1270 if (nvl == NULL) 1271 return (ENOMEM); 1272 1273 nvlist_add_number(nvl, "pending", 1274 buf_ring_count(sc->rxring) + buf_ring_count(sc->notifring)); 1275 1276 *onvl = nvl; 1277 1278 return (0); 1279 } 1280 1281 static int 1282 opvn_get_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1283 { 1284 struct ovpn_notification *n; 1285 struct ovpn_wire_header *ohdr; 1286 struct mbuf *m; 1287 uint8_t *buf; 1288 nvlist_t *nvl; 1289 uint32_t peerid; 1290 u_int mlength; 1291 1292 /* Check if we have notifications pending. */ 1293 n = buf_ring_dequeue_mc(sc->notifring); 1294 if (n != NULL) { 1295 nvl = nvlist_create(0); 1296 if (nvl == NULL) { 1297 free(n, M_OVPN); 1298 return (ENOMEM); 1299 } 1300 nvlist_add_number(nvl, "peerid", n->peerid); 1301 nvlist_add_number(nvl, "notification", n->type); 1302 free(n, M_OVPN); 1303 1304 *onvl = nvl; 1305 1306 return (0); 1307 } 1308 1309 /* Queued packet. */ 1310 m = buf_ring_dequeue_mc(sc->rxring); 1311 if (m == NULL) 1312 return (ENOENT); 1313 1314 mlength = m_length(m, NULL); 1315 buf = malloc(mlength, M_NVLIST, M_WAITOK); 1316 m_copydata(m, 0, mlength, buf); 1317 ohdr = (struct ovpn_wire_header *)buf; 1318 peerid = ntohl(ohdr->opcode) & 0x00ffffff; 1319 1320 nvl = nvlist_create(0); 1321 if (nvl == NULL) { 1322 OVPN_COUNTER_ADD(sc, lost_ctrl_pkts_in, 1); 1323 m_freem(m); 1324 free(buf, M_NVLIST); 1325 return (ENOMEM); 1326 } 1327 1328 nvlist_move_binary(nvl, "packet", buf, mlength); 1329 buf = NULL; 1330 nvlist_add_number(nvl, "peerid", peerid); 1331 1332 *onvl = nvl; 1333 1334 m_freem(m); 1335 1336 return (0); 1337 } 1338 1339 static int 1340 ovpn_ioctl_get(struct ifnet *ifp, struct ifdrv *ifd) 1341 { 1342 struct ovpn_softc *sc = ifp->if_softc; 1343 nvlist_t *nvl = NULL; 1344 int error; 1345 1346 switch (ifd->ifd_cmd) { 1347 case OVPN_GET_STATS: 1348 error = ovpn_get_stats(sc, &nvl); 1349 break; 1350 case OVPN_POLL_PKT: 1351 error = ovpn_poll_pkt(sc, &nvl); 1352 break; 1353 case OVPN_GET_PKT: 1354 error = opvn_get_pkt(sc, &nvl); 1355 break; 1356 default: 1357 error = ENOTSUP; 1358 break; 1359 } 1360 1361 if (error == 0) { 1362 void *packed = NULL; 1363 size_t len; 1364 1365 MPASS(nvl != NULL); 1366 1367 packed = nvlist_pack(nvl, &len); 1368 if (! packed) { 1369 nvlist_destroy(nvl); 1370 return (ENOMEM); 1371 } 1372 1373 if (len > ifd->ifd_len) { 1374 free(packed, M_NVLIST); 1375 nvlist_destroy(nvl); 1376 return (ENOSPC); 1377 } 1378 1379 error = copyout(packed, ifd->ifd_data, len); 1380 ifd->ifd_len = len; 1381 1382 free(packed, M_NVLIST); 1383 nvlist_destroy(nvl); 1384 } 1385 1386 return (error); 1387 } 1388 1389 static int 1390 ovpn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 1391 { 1392 struct ifdrv *ifd; 1393 int error; 1394 1395 switch (cmd) { 1396 case SIOCSDRVSPEC: 1397 case SIOCGDRVSPEC: 1398 error = priv_check(curthread, PRIV_NET_OVPN); 1399 if (error) 1400 return (error); 1401 break; 1402 } 1403 1404 switch (cmd) { 1405 case SIOCSDRVSPEC: 1406 ifd = (struct ifdrv *)data; 1407 error = ovpn_ioctl_set(ifp, ifd); 1408 break; 1409 case SIOCGDRVSPEC: 1410 ifd = (struct ifdrv *)data; 1411 error = ovpn_ioctl_get(ifp, ifd); 1412 break; 1413 case SIOCSIFMTU: { 1414 struct ifreq *ifr = (struct ifreq *)data; 1415 if (ifr->ifr_mtu < OVPN_MTU_MIN || ifr->ifr_mtu > OVPN_MTU_MAX) 1416 return (EINVAL); 1417 1418 ifp->if_mtu = ifr->ifr_mtu; 1419 return (0); 1420 } 1421 case SIOCSIFADDR: 1422 case SIOCADDMULTI: 1423 case SIOCDELMULTI: 1424 case SIOCGIFMTU: 1425 case SIOCSIFFLAGS: 1426 return (0); 1427 default: 1428 error = EINVAL; 1429 } 1430 1431 return (error); 1432 } 1433 1434 static int 1435 ovpn_encrypt_tx_cb(struct cryptop *crp) 1436 { 1437 struct epoch_tracker et; 1438 struct ovpn_kpeer *peer = crp->crp_opaque; 1439 struct ovpn_softc *sc = peer->sc; 1440 struct mbuf *m = crp->crp_buf.cb_mbuf; 1441 int tunnel_len; 1442 int ret; 1443 1444 if (crp->crp_etype != 0) { 1445 crypto_freereq(crp); 1446 ovpn_peer_release_ref(peer, false); 1447 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1448 m_freem(m); 1449 return (0); 1450 } 1451 1452 NET_EPOCH_ENTER(et); 1453 CURVNET_SET(sc->ifp->if_vnet); 1454 1455 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1456 1457 tunnel_len = m->m_pkthdr.len - sizeof(struct ovpn_wire_header); 1458 ret = ovpn_encap(sc, peer->peerid, m); 1459 if (ret == 0) { 1460 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1461 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 1462 } 1463 1464 CURVNET_RESTORE(); 1465 NET_EPOCH_EXIT(et); 1466 1467 crypto_freereq(crp); 1468 ovpn_peer_release_ref(peer, false); 1469 1470 return (0); 1471 } 1472 1473 static void 1474 ovpn_finish_rx(struct ovpn_softc *sc, struct mbuf *m, 1475 struct ovpn_kpeer *peer, struct ovpn_kkey *key, uint32_t seq, 1476 struct rm_priotracker *_ovpn_lock_trackerp) 1477 { 1478 uint32_t af; 1479 1480 OVPN_RASSERT(sc); 1481 NET_EPOCH_ASSERT(); 1482 1483 /* Replay protection. */ 1484 if (V_replay_protection && ! ovpn_check_replay(key->decrypt, seq)) { 1485 OVPN_RUNLOCK(sc); 1486 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1487 m_freem(m); 1488 return; 1489 } 1490 1491 critical_enter(); 1492 *zpcpu_get(peer->last_active) = time_uptime; 1493 critical_exit(); 1494 1495 OVPN_RUNLOCK(sc); 1496 1497 OVPN_COUNTER_ADD(sc, received_data_pkts, 1); 1498 OVPN_COUNTER_ADD(sc, tunnel_bytes_received, m->m_pkthdr.len); 1499 1500 /* Receive the packet on our interface. */ 1501 m->m_pkthdr.rcvif = sc->ifp; 1502 1503 /* Clear checksum flags in case the real hardware set them. */ 1504 m->m_pkthdr.csum_flags = 0; 1505 1506 /* Ensure we can read the first byte. */ 1507 m = m_pullup(m, 1); 1508 if (m == NULL) { 1509 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 1510 return; 1511 } 1512 1513 /* 1514 * Check for address family, and disregard any control packets (e.g. 1515 * keepalive). 1516 */ 1517 af = ovpn_get_af(m); 1518 if (af != 0) { 1519 BPF_MTAP2(sc->ifp, &af, sizeof(af), m); 1520 if (V_async_netisr_queue) 1521 netisr_queue(af == AF_INET ? NETISR_IP : NETISR_IPV6, m); 1522 else 1523 netisr_dispatch(af == AF_INET ? NETISR_IP : NETISR_IPV6, m); 1524 } else { 1525 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1526 m_freem(m); 1527 } 1528 } 1529 1530 static struct ovpn_kkey * 1531 ovpn_find_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, 1532 const struct ovpn_wire_header *ohdr) 1533 { 1534 struct ovpn_kkey *key = NULL; 1535 uint8_t keyid; 1536 1537 OVPN_RASSERT(sc); 1538 1539 keyid = (ntohl(ohdr->opcode) >> 24) & 0x07; 1540 1541 if (peer->keys[0].keyid == keyid) 1542 key = &peer->keys[0]; 1543 else if (peer->keys[1].keyid == keyid) 1544 key = &peer->keys[1]; 1545 1546 return (key); 1547 } 1548 1549 static int 1550 ovpn_decrypt_rx_cb(struct cryptop *crp) 1551 { 1552 struct epoch_tracker et; 1553 struct ovpn_softc *sc = crp->crp_opaque; 1554 struct mbuf *m = crp->crp_buf.cb_mbuf; 1555 struct ovpn_kkey *key; 1556 struct ovpn_kpeer *peer; 1557 struct ovpn_wire_header *ohdr; 1558 uint32_t peerid; 1559 1560 OVPN_RLOCK_TRACKER; 1561 1562 OVPN_RLOCK(sc); 1563 1564 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1565 1566 if (crp->crp_etype != 0) { 1567 crypto_freereq(crp); 1568 atomic_add_int(&sc->refcount, -1); 1569 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1570 OVPN_RUNLOCK(sc); 1571 m_freem(m); 1572 return (0); 1573 } 1574 1575 CURVNET_SET(sc->ifp->if_vnet); 1576 1577 ohdr = mtodo(m, sizeof(struct udphdr)); 1578 1579 peerid = ntohl(ohdr->opcode) & 0x00ffffff; 1580 peer = ovpn_find_peer(sc, peerid); 1581 if (peer == NULL) { 1582 /* No such peer. Drop packet. */ 1583 crypto_freereq(crp); 1584 atomic_add_int(&sc->refcount, -1); 1585 OVPN_RUNLOCK(sc); 1586 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1587 m_freem(m); 1588 CURVNET_RESTORE(); 1589 return (0); 1590 } 1591 1592 key = ovpn_find_key(sc, peer, ohdr); 1593 if (key == NULL) { 1594 crypto_freereq(crp); 1595 atomic_add_int(&sc->refcount, -1); 1596 /* 1597 * Has this key been removed between us starting the decrypt 1598 * and finishing it? 1599 */ 1600 OVPN_RUNLOCK(sc); 1601 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1602 m_freem(m); 1603 CURVNET_RESTORE(); 1604 return (0); 1605 } 1606 1607 /* Now remove the outer headers */ 1608 m_adj_decap(m, sizeof(struct udphdr) + 1609 sizeof(struct ovpn_wire_header)); 1610 1611 NET_EPOCH_ENTER(et); 1612 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), _ovpn_lock_trackerp); 1613 NET_EPOCH_EXIT(et); 1614 OVPN_UNLOCK_ASSERT(sc); 1615 1616 CURVNET_RESTORE(); 1617 1618 crypto_freereq(crp); 1619 atomic_add_int(&sc->refcount, -1); 1620 1621 return (0); 1622 } 1623 1624 static int 1625 ovpn_get_af(struct mbuf *m) 1626 { 1627 struct ip *ip; 1628 struct ip6_hdr *ip6; 1629 1630 /* 1631 * We should pullup, but we're only interested in the first byte, so 1632 * that'll always be contiguous. 1633 */ 1634 ip = mtod(m, struct ip *); 1635 if (ip->ip_v == IPVERSION) 1636 return (AF_INET); 1637 1638 ip6 = mtod(m, struct ip6_hdr *); 1639 if ((ip6->ip6_vfc & IPV6_VERSION_MASK) == IPV6_VERSION) 1640 return (AF_INET6); 1641 1642 return (0); 1643 } 1644 1645 #ifdef INET 1646 static struct ovpn_kpeer * 1647 ovpn_find_peer_by_ip(struct ovpn_softc *sc, const struct in_addr addr) 1648 { 1649 struct ovpn_kpeer *peer = NULL; 1650 1651 OVPN_ASSERT(sc); 1652 1653 for (int i = 0; i < OVPN_MAX_PEERS; i++) { 1654 if (sc->peers[i] == NULL) 1655 continue; 1656 if (addr.s_addr == sc->peers[i]->vpn4.s_addr) { 1657 peer = sc->peers[i]; 1658 break; 1659 } 1660 } 1661 1662 return (peer); 1663 } 1664 #endif 1665 1666 #ifdef INET6 1667 static struct ovpn_kpeer * 1668 ovpn_find_peer_by_ip6(struct ovpn_softc *sc, const struct in6_addr *addr) 1669 { 1670 struct ovpn_kpeer *peer = NULL; 1671 1672 OVPN_ASSERT(sc); 1673 1674 for (int i = 0; i < OVPN_MAX_PEERS; i++) { 1675 if (sc->peers[i] == NULL) 1676 continue; 1677 if (memcmp(addr, &sc->peers[i]->vpn6, sizeof(*addr)) == 0) { 1678 peer = sc->peers[i]; 1679 break; 1680 } 1681 } 1682 1683 return (peer); 1684 } 1685 #endif 1686 1687 static struct ovpn_kpeer * 1688 ovpn_route_peer(struct ovpn_softc *sc, struct mbuf **m0, 1689 const struct sockaddr *dst) 1690 { 1691 struct ovpn_kpeer *peer = NULL; 1692 int af; 1693 1694 NET_EPOCH_ASSERT(); 1695 OVPN_ASSERT(sc); 1696 1697 /* Shortcut if we're a client (or are a server and have only one client). */ 1698 if (sc->peercount == 1) 1699 return (ovpn_find_only_peer(sc)); 1700 1701 if (dst != NULL) 1702 af = dst->sa_family; 1703 else 1704 af = ovpn_get_af(*m0); 1705 1706 switch (af) { 1707 #ifdef INET 1708 case AF_INET: { 1709 const struct sockaddr_in *sa = (const struct sockaddr_in *)dst; 1710 struct nhop_object *nh; 1711 const struct in_addr *ip_dst; 1712 1713 if (sa != NULL) { 1714 ip_dst = &sa->sin_addr; 1715 } else { 1716 struct ip *ip; 1717 1718 *m0 = m_pullup(*m0, sizeof(struct ip)); 1719 if (*m0 == NULL) 1720 return (NULL); 1721 ip = mtod(*m0, struct ip *); 1722 ip_dst = &ip->ip_dst; 1723 } 1724 1725 peer = ovpn_find_peer_by_ip(sc, *ip_dst); 1726 SDT_PROBE2(if_ovpn, tx, route, ip4, ip_dst, peer); 1727 if (peer == NULL) { 1728 nh = fib4_lookup(M_GETFIB(*m0), *ip_dst, 0, 1729 NHR_NONE, 0); 1730 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1731 peer = ovpn_find_peer_by_ip(sc, 1732 nh->gw4_sa.sin_addr); 1733 SDT_PROBE2(if_ovpn, tx, route, ip4, 1734 &nh->gw4_sa.sin_addr, peer); 1735 } 1736 } 1737 break; 1738 } 1739 #endif 1740 #ifdef INET6 1741 case AF_INET6: { 1742 const struct sockaddr_in6 *sa6 = 1743 (const struct sockaddr_in6 *)dst; 1744 struct nhop_object *nh; 1745 const struct in6_addr *ip6_dst; 1746 1747 if (sa6 != NULL) { 1748 ip6_dst = &sa6->sin6_addr; 1749 } else { 1750 struct ip6_hdr *ip6; 1751 1752 *m0 = m_pullup(*m0, sizeof(struct ip6_hdr)); 1753 if (*m0 == NULL) 1754 return (NULL); 1755 ip6 = mtod(*m0, struct ip6_hdr *); 1756 ip6_dst = &ip6->ip6_dst; 1757 } 1758 1759 peer = ovpn_find_peer_by_ip6(sc, ip6_dst); 1760 SDT_PROBE2(if_ovpn, tx, route, ip6, ip6_dst, peer); 1761 if (peer == NULL) { 1762 nh = fib6_lookup(M_GETFIB(*m0), ip6_dst, 0, 1763 NHR_NONE, 0); 1764 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1765 peer = ovpn_find_peer_by_ip6(sc, 1766 &nh->gw6_sa.sin6_addr); 1767 SDT_PROBE2(if_ovpn, tx, route, ip6, 1768 &nh->gw6_sa.sin6_addr, peer); 1769 } 1770 } 1771 break; 1772 } 1773 #endif 1774 } 1775 1776 return (peer); 1777 } 1778 1779 static int 1780 ovpn_transmit(struct ifnet *ifp, struct mbuf *m) 1781 { 1782 return (ifp->if_output(ifp, m, NULL, NULL)); 1783 } 1784 1785 static int 1786 ovpn_transmit_to_peer(struct ifnet *ifp, struct mbuf *m, 1787 struct ovpn_kpeer *peer, struct rm_priotracker *_ovpn_lock_trackerp) 1788 { 1789 struct ovpn_wire_header *ohdr; 1790 struct ovpn_kkey *key; 1791 struct ovpn_softc *sc; 1792 struct cryptop *crp; 1793 uint32_t af, seq; 1794 size_t len, ovpn_hdr_len; 1795 int tunnel_len; 1796 int ret; 1797 1798 sc = ifp->if_softc; 1799 1800 OVPN_RASSERT(sc); 1801 1802 tunnel_len = m->m_pkthdr.len; 1803 1804 key = &peer->keys[OVPN_KEY_SLOT_PRIMARY]; 1805 if (key->encrypt == NULL) { 1806 if (_ovpn_lock_trackerp != NULL) 1807 OVPN_RUNLOCK(sc); 1808 m_freem(m); 1809 return (ENOLINK); 1810 } 1811 1812 af = ovpn_get_af(m); 1813 /* Don't capture control packets. */ 1814 if (af != 0) 1815 BPF_MTAP2(ifp, &af, sizeof(af), m); 1816 1817 len = m->m_pkthdr.len; 1818 MPASS(len <= ifp->if_mtu); 1819 1820 ovpn_hdr_len = sizeof(struct ovpn_wire_header); 1821 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) 1822 ovpn_hdr_len -= 16; /* No auth tag. */ 1823 1824 M_PREPEND(m, ovpn_hdr_len, M_NOWAIT); 1825 if (m == NULL) { 1826 if (_ovpn_lock_trackerp != NULL) 1827 OVPN_RUNLOCK(sc); 1828 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1829 return (ENOBUFS); 1830 } 1831 ohdr = mtod(m, struct ovpn_wire_header *); 1832 ohdr->opcode = (OVPN_OP_DATA_V2 << OVPN_OP_SHIFT) | key->keyid; 1833 ohdr->opcode <<= 24; 1834 ohdr->opcode |= key->peerid; 1835 ohdr->opcode = htonl(ohdr->opcode); 1836 1837 seq = atomic_fetchadd_32(&peer->tx_seq, 1); 1838 seq = htonl(seq); 1839 ohdr->seq = seq; 1840 1841 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) { 1842 ret = ovpn_encap(sc, peer->peerid, m); 1843 if (_ovpn_lock_trackerp != NULL) 1844 OVPN_RUNLOCK(sc); 1845 if (ret == 0) { 1846 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1847 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 1848 } 1849 return (ret); 1850 } 1851 1852 crp = crypto_getreq(key->encrypt->cryptoid, M_NOWAIT); 1853 if (crp == NULL) { 1854 if (_ovpn_lock_trackerp != NULL) 1855 OVPN_RUNLOCK(sc); 1856 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1857 m_freem(m); 1858 return (ENOBUFS); 1859 } 1860 1861 /* Encryption covers only the payload, not the header. */ 1862 crp->crp_payload_start = sizeof(*ohdr); 1863 crp->crp_payload_length = len; 1864 crp->crp_op = CRYPTO_OP_ENCRYPT; 1865 1866 /* 1867 * AAD data covers the ovpn_wire_header minus the auth 1868 * tag. 1869 */ 1870 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 1871 crp->crp_aad = ohdr; 1872 crp->crp_aad_start = 0; 1873 crp->crp_op |= CRYPTO_OP_COMPUTE_DIGEST; 1874 crp->crp_digest_start = offsetof(struct ovpn_wire_header, auth_tag); 1875 1876 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 1877 memcpy(crp->crp_iv, &seq, sizeof(seq)); 1878 memcpy(crp->crp_iv + sizeof(seq), key->encrypt->nonce, 1879 key->encrypt->noncelen); 1880 1881 crypto_use_mbuf(crp, m); 1882 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 1883 crp->crp_callback = ovpn_encrypt_tx_cb; 1884 crp->crp_opaque = peer; 1885 1886 atomic_add_int(&peer->refcount, 1); 1887 if (_ovpn_lock_trackerp != NULL) 1888 OVPN_RUNLOCK(sc); 1889 if (V_async_crypto) 1890 ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); 1891 else 1892 ret = crypto_dispatch(crp); 1893 if (ret) { 1894 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1895 } 1896 1897 return (ret); 1898 } 1899 1900 /* 1901 * Note: Expects to hold the read lock on entry, and will release it itself. 1902 */ 1903 static int 1904 ovpn_encap(struct ovpn_softc *sc, uint32_t peerid, struct mbuf *m) 1905 { 1906 struct udphdr *udp; 1907 struct ovpn_kpeer *peer; 1908 int len; 1909 1910 OVPN_RLOCK_TRACKER; 1911 1912 OVPN_RLOCK(sc); 1913 NET_EPOCH_ASSERT(); 1914 1915 peer = ovpn_find_peer(sc, peerid); 1916 if (peer == NULL || sc->ifp->if_link_state != LINK_STATE_UP) { 1917 OVPN_RUNLOCK(sc); 1918 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1919 m_freem(m); 1920 return (ENETDOWN); 1921 } 1922 1923 len = m->m_pkthdr.len; 1924 1925 M_PREPEND(m, sizeof(struct udphdr), M_NOWAIT); 1926 if (m == NULL) { 1927 OVPN_RUNLOCK(sc); 1928 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1929 m_freem(m); 1930 return (ENOBUFS); 1931 } 1932 udp = mtod(m, struct udphdr *); 1933 1934 MPASS(peer->local.ss_family == peer->remote.ss_family); 1935 1936 udp->uh_sport = ovpn_get_port(&peer->local); 1937 udp->uh_dport = ovpn_get_port(&peer->remote); 1938 udp->uh_ulen = htons(sizeof(struct udphdr) + len); 1939 1940 switch (peer->remote.ss_family) { 1941 #ifdef INET 1942 case AF_INET: { 1943 struct sockaddr_in *in_local = TO_IN(&peer->local); 1944 struct sockaddr_in *in_remote = TO_IN(&peer->remote); 1945 struct ip *ip; 1946 1947 /* 1948 * This requires knowing the source IP, which we don't. Happily 1949 * we're allowed to keep this at 0, and the checksum won't do 1950 * anything the crypto won't already do. 1951 */ 1952 udp->uh_sum = 0; 1953 1954 /* Set the checksum flags so we recalculate checksums. */ 1955 m->m_pkthdr.csum_flags |= CSUM_IP; 1956 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 1957 1958 M_PREPEND(m, sizeof(struct ip), M_NOWAIT); 1959 if (m == NULL) { 1960 OVPN_RUNLOCK(sc); 1961 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1962 return (ENOBUFS); 1963 } 1964 ip = mtod(m, struct ip *); 1965 1966 ip->ip_tos = 0; 1967 ip->ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) + 1968 len); 1969 ip->ip_off = 0; 1970 ip->ip_ttl = V_ip_defttl; 1971 ip->ip_p = IPPROTO_UDP; 1972 ip->ip_sum = 0; 1973 if (in_local->sin_port != 0) 1974 ip->ip_src = in_local->sin_addr; 1975 else 1976 ip->ip_src.s_addr = INADDR_ANY; 1977 ip->ip_dst = in_remote->sin_addr; 1978 1979 OVPN_RUNLOCK(sc); 1980 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 1981 1982 return (ip_output(m, NULL, NULL, 0, NULL, NULL)); 1983 } 1984 #endif 1985 #ifdef INET6 1986 case AF_INET6: { 1987 struct sockaddr_in6 *in6_local = TO_IN6(&peer->local); 1988 struct sockaddr_in6 *in6_remote = TO_IN6(&peer->remote); 1989 struct ip6_hdr *ip6; 1990 1991 M_PREPEND(m, sizeof(struct ip6_hdr), M_NOWAIT); 1992 if (m == NULL) { 1993 OVPN_RUNLOCK(sc); 1994 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1995 return (ENOBUFS); 1996 } 1997 m = m_pullup(m, sizeof(*ip6) + sizeof(*udp)); 1998 if (m == NULL) { 1999 OVPN_RUNLOCK(sc); 2000 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2001 return (ENOBUFS); 2002 } 2003 2004 ip6 = mtod(m, struct ip6_hdr *); 2005 2006 ip6->ip6_vfc = IPV6_VERSION; 2007 ip6->ip6_flow &= ~IPV6_FLOWINFO_MASK; 2008 ip6->ip6_plen = htons(sizeof(*ip6) + sizeof(struct udphdr) + 2009 len); 2010 ip6->ip6_nxt = IPPROTO_UDP; 2011 ip6->ip6_hlim = V_ip6_defhlim; 2012 2013 memcpy(&ip6->ip6_src, &in6_local->sin6_addr, 2014 sizeof(ip6->ip6_src)); 2015 memcpy(&ip6->ip6_dst, &in6_remote->sin6_addr, 2016 sizeof(ip6->ip6_dst)); 2017 2018 udp = mtodo(m, sizeof(*ip6)); 2019 udp->uh_sum = in6_cksum_pseudo(ip6, 2020 m->m_pkthdr.len - sizeof(struct ip6_hdr), 2021 IPPROTO_UDP, 0); 2022 2023 m->m_pkthdr.csum_flags |= CSUM_UDP_IPV6; 2024 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 2025 2026 OVPN_RUNLOCK(sc); 2027 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 2028 2029 return (ip6_output(m, NULL, NULL, IPV6_UNSPECSRC, NULL, NULL, 2030 NULL)); 2031 } 2032 #endif 2033 default: 2034 panic("Unsupported address family %d", 2035 peer->remote.ss_family); 2036 } 2037 } 2038 2039 static int 2040 ovpn_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 2041 struct route *ro) 2042 { 2043 struct ovpn_softc *sc; 2044 struct ovpn_kpeer *peer; 2045 2046 OVPN_RLOCK_TRACKER; 2047 2048 sc = ifp->if_softc; 2049 2050 OVPN_RLOCK(sc); 2051 2052 SDT_PROBE1(if_ovpn, tx, transmit, start, m); 2053 2054 if (__predict_false(ifp->if_link_state != LINK_STATE_UP)) { 2055 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2056 OVPN_RUNLOCK(sc); 2057 m_freem(m); 2058 return (ENETDOWN); 2059 } 2060 2061 /** 2062 * Only obey 'dst' (i.e. the gateway) if no route is supplied. 2063 * That's our indication that we're being called through pf's route-to, 2064 * and we should route according to 'dst' instead. We can't do so 2065 * consistently, because the usual openvpn configuration sets the first 2066 * non-server IP in the subnet as the gateway. If we always use that 2067 * one we'd end up routing all traffic to the first client. 2068 * tl;dr: 'ro == NULL' tells us pf is doing a route-to, and then but 2069 * only then, we should treat 'dst' as the destination. */ 2070 peer = ovpn_route_peer(sc, &m, ro == NULL ? dst : NULL); 2071 if (peer == NULL) { 2072 /* No destination. */ 2073 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2074 OVPN_RUNLOCK(sc); 2075 m_freem(m); 2076 return (ENETDOWN); 2077 } 2078 2079 return (ovpn_transmit_to_peer(ifp, m, peer, _ovpn_lock_trackerp)); 2080 } 2081 2082 static void 2083 ovpn_rcv_ctrl(struct ovpn_softc *sc, struct mbuf *m, int off) 2084 { 2085 /* Lop off the IP and UDP headers */ 2086 m_adj_decap(m, off); 2087 2088 /* Keep in the local ring until userspace fetches it. */ 2089 if (buf_ring_enqueue(sc->rxring, m) != 0) { 2090 OVPN_COUNTER_ADD(sc, lost_ctrl_pkts_in, 1); 2091 m_freem(m); 2092 return; 2093 } 2094 2095 OVPN_COUNTER_ADD(sc, received_ctrl_pkts, 1); 2096 } 2097 2098 static bool 2099 ovpn_check_replay(struct ovpn_kkey_dir *key, uint32_t seq) 2100 { 2101 uint32_t d; 2102 2103 mtx_lock(&key->replay_mtx); 2104 2105 /* Sequence number must be strictly greater than rx_seq */ 2106 if (seq <= key->rx_seq) { 2107 mtx_unlock(&key->replay_mtx); 2108 return (false); 2109 } 2110 2111 /* Large jump. The packet authenticated okay, so just accept that. */ 2112 if (seq > (key->rx_seq + (sizeof(key->rx_window) * 8))) { 2113 key->rx_seq = seq; 2114 key->rx_window = 0; 2115 mtx_unlock(&key->replay_mtx); 2116 return (true); 2117 } 2118 2119 /* Happy case. */ 2120 if ((seq == key->rx_seq + 1) && key->rx_window == 0) { 2121 key->rx_seq++; 2122 mtx_unlock(&key->replay_mtx); 2123 return (true); 2124 } 2125 2126 d = seq - key->rx_seq - 1; 2127 2128 if (key->rx_window & ((uint64_t)1 << d)) { 2129 /* Dupe! */ 2130 mtx_unlock(&key->replay_mtx); 2131 return (false); 2132 } 2133 2134 key->rx_window |= (uint64_t)1 << d; 2135 2136 while (key->rx_window & 1) { 2137 key->rx_seq++; 2138 key->rx_window >>= 1; 2139 } 2140 2141 mtx_unlock(&key->replay_mtx); 2142 2143 return (true); 2144 } 2145 2146 static struct ovpn_kpeer * 2147 ovpn_peer_from_mbuf(struct ovpn_softc *sc, struct mbuf *m, int off) 2148 { 2149 struct ovpn_wire_header ohdr; 2150 uint32_t peerid; 2151 const size_t hdrlen = sizeof(ohdr) - sizeof(ohdr.auth_tag); 2152 2153 OVPN_RASSERT(sc); 2154 2155 if (m_length(m, NULL) < (off + sizeof(struct udphdr) + hdrlen)) 2156 return (NULL); 2157 2158 m_copydata(m, off + sizeof(struct udphdr), hdrlen, (caddr_t)&ohdr); 2159 2160 peerid = ntohl(ohdr.opcode) & 0x00ffffff; 2161 2162 return (ovpn_find_peer(sc, peerid)); 2163 } 2164 2165 static bool 2166 ovpn_udp_input(struct mbuf *m, int off, struct inpcb *inp, 2167 const struct sockaddr *sa, void *ctx) 2168 { 2169 struct ovpn_softc *sc = ctx; 2170 struct ovpn_wire_header *ohdr; 2171 struct udphdr *uhdr; 2172 struct ovpn_kkey *key; 2173 struct cryptop *crp; 2174 struct ovpn_kpeer *peer; 2175 size_t ohdrlen; 2176 int ret; 2177 uint8_t op; 2178 2179 OVPN_RLOCK_TRACKER; 2180 2181 M_ASSERTPKTHDR(m); 2182 2183 OVPN_COUNTER_ADD(sc, transport_bytes_received, m->m_pkthdr.len - off); 2184 2185 ohdrlen = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2186 2187 OVPN_RLOCK(sc); 2188 2189 peer = ovpn_peer_from_mbuf(sc, m, off); 2190 if (peer == NULL) { 2191 OVPN_RUNLOCK(sc); 2192 return (false); 2193 } 2194 2195 m = m_pullup(m, off + sizeof(*uhdr) + ohdrlen); 2196 if (m == NULL) { 2197 OVPN_RUNLOCK(sc); 2198 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2199 return (true); 2200 } 2201 uhdr = mtodo(m, off); 2202 ohdr = mtodo(m, off + sizeof(*uhdr)); 2203 2204 op = ntohl(ohdr->opcode) >> 24 >> OVPN_OP_SHIFT; 2205 2206 /* 2207 * Simplify things by getting rid of the preceding headers, we don't 2208 * care about them. 2209 */ 2210 m_adj_decap(m, off); 2211 2212 uhdr = mtodo(m, 0); 2213 ohdr = mtodo(m, sizeof(*uhdr)); 2214 2215 if (op != OVPN_OP_DATA_V2) { 2216 OVPN_RUNLOCK(sc); 2217 ovpn_rcv_ctrl(sc, m, sizeof(struct udphdr)); 2218 INP_WLOCK(inp); 2219 udp_notify(inp, EAGAIN); 2220 INP_WUNLOCK(inp); 2221 return (true); 2222 } 2223 2224 key = ovpn_find_key(sc, peer, ohdr); 2225 if (key == NULL || key->decrypt == NULL) { 2226 OVPN_RUNLOCK(sc); 2227 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2228 m_freem(m); 2229 return (true); 2230 } 2231 2232 if (key->decrypt->cipher == OVPN_CIPHER_ALG_NONE) { 2233 /* Now remove the outer headers */ 2234 m_adj_decap(m, sizeof(struct udphdr) + ohdrlen); 2235 2236 ohdr = mtodo(m, sizeof(*uhdr)); 2237 2238 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), 2239 _ovpn_lock_trackerp); 2240 OVPN_UNLOCK_ASSERT(sc); 2241 return (true); 2242 } 2243 2244 ohdrlen += sizeof(ohdr->auth_tag); 2245 2246 m = m_pullup(m, sizeof(*uhdr) + ohdrlen); 2247 if (m == NULL) { 2248 OVPN_RUNLOCK(sc); 2249 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2250 return (true); 2251 } 2252 uhdr = mtodo(m, 0); 2253 ohdr = mtodo(m, sizeof(*uhdr)); 2254 2255 /* Decrypt */ 2256 crp = crypto_getreq(key->decrypt->cryptoid, M_NOWAIT); 2257 if (crp == NULL) { 2258 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2259 OVPN_RUNLOCK(sc); 2260 m_freem(m); 2261 return (true); 2262 } 2263 2264 crp->crp_payload_start = sizeof(struct udphdr) + sizeof(*ohdr); 2265 crp->crp_payload_length = ntohs(uhdr->uh_ulen) - 2266 sizeof(*uhdr) - sizeof(*ohdr); 2267 crp->crp_op = CRYPTO_OP_DECRYPT; 2268 2269 /* AAD validation. */ 2270 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2271 crp->crp_aad = ohdr; 2272 crp->crp_aad_start = 0; 2273 crp->crp_op |= CRYPTO_OP_VERIFY_DIGEST; 2274 crp->crp_digest_start = sizeof(struct udphdr) + 2275 offsetof(struct ovpn_wire_header, auth_tag); 2276 2277 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 2278 memcpy(crp->crp_iv, &ohdr->seq, sizeof(ohdr->seq)); 2279 memcpy(crp->crp_iv + sizeof(ohdr->seq), key->decrypt->nonce, 2280 key->decrypt->noncelen); 2281 2282 crypto_use_mbuf(crp, m); 2283 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 2284 crp->crp_callback = ovpn_decrypt_rx_cb; 2285 crp->crp_opaque = sc; 2286 2287 atomic_add_int(&sc->refcount, 1); 2288 OVPN_RUNLOCK(sc); 2289 if (V_async_crypto) 2290 ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); 2291 else 2292 ret = crypto_dispatch(crp); 2293 if (ret != 0) { 2294 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2295 } 2296 2297 return (true); 2298 } 2299 2300 static void 2301 ovpn_qflush(struct ifnet *ifp __unused) 2302 { 2303 2304 } 2305 2306 static void 2307 ovpn_flush_rxring(struct ovpn_softc *sc) 2308 { 2309 struct mbuf *m; 2310 struct ovpn_notification *n; 2311 2312 OVPN_WASSERT(sc); 2313 2314 while (! buf_ring_empty(sc->rxring)) { 2315 m = buf_ring_dequeue_sc(sc->rxring); 2316 m_freem(m); 2317 } 2318 2319 while (! buf_ring_empty(sc->notifring)) { 2320 n = buf_ring_dequeue_sc(sc->notifring); 2321 free(n, M_OVPN); 2322 } 2323 } 2324 2325 #ifdef VIMAGE 2326 static void 2327 ovpn_reassign(struct ifnet *ifp, struct vnet *new_vnet __unused, 2328 char *unused __unused) 2329 { 2330 struct ovpn_softc *sc = ifp->if_softc; 2331 int i; 2332 int ret __diagused; 2333 2334 i = 0; 2335 2336 OVPN_WLOCK(sc); 2337 2338 /* Flush keys & configuration. */ 2339 do { 2340 if (sc->peers[i] != NULL) { 2341 ret = _ovpn_del_peer(sc, sc->peers[i]->peerid); 2342 MPASS(ret == 0); 2343 } 2344 i++; 2345 } while (i < OVPN_MAX_PEERS); 2346 2347 ovpn_flush_rxring(sc); 2348 2349 OVPN_WUNLOCK(sc); 2350 } 2351 #endif 2352 2353 static int 2354 ovpn_clone_match(struct if_clone *ifc, const char *name) 2355 { 2356 /* 2357 * Allow all names that start with 'ovpn', specifically because pfSense 2358 * uses ovpnc1 / ovpns2 2359 */ 2360 return (strncmp(ovpnname, name, strlen(ovpnname)) == 0); 2361 } 2362 2363 static int 2364 ovpn_clone_create(struct if_clone *ifc, char *name, size_t len, 2365 struct ifc_data *ifd, struct ifnet **ifpp) 2366 { 2367 struct ovpn_softc *sc; 2368 struct ifnet *ifp; 2369 char *dp; 2370 int error, unit, wildcard; 2371 2372 /* Try to see if a special unit was requested. */ 2373 error = ifc_name2unit(name, &unit); 2374 if (error != 0) 2375 return (error); 2376 wildcard = (unit < 0); 2377 2378 error = ifc_alloc_unit(ifc, &unit); 2379 if (error != 0) 2380 return (error); 2381 2382 /* 2383 * If no unit had been given, we need to adjust the ifName. 2384 */ 2385 for (dp = name; *dp != '\0'; dp++); 2386 if (wildcard) { 2387 error = snprintf(dp, len - (dp - name), "%d", unit); 2388 if (error > len - (dp - name)) { 2389 /* ifName too long. */ 2390 ifc_free_unit(ifc, unit); 2391 return (ENOSPC); 2392 } 2393 dp += error; 2394 } 2395 2396 /* Make sure it doesn't already exist. */ 2397 if (ifunit(name) != NULL) 2398 return (EEXIST); 2399 2400 sc = malloc(sizeof(struct ovpn_softc), M_OVPN, M_WAITOK | M_ZERO); 2401 sc->ifp = if_alloc(IFT_ENC); 2402 rm_init_flags(&sc->lock, "if_ovpn_lock", RM_RECURSE); 2403 sc->refcount = 0; 2404 2405 sc->rxring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL); 2406 sc->notifring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL); 2407 2408 COUNTER_ARRAY_ALLOC(sc->counters, OVPN_COUNTER_SIZE, M_WAITOK); 2409 2410 ifp = sc->ifp; 2411 ifp->if_softc = sc; 2412 strlcpy(ifp->if_xname, name, IFNAMSIZ); 2413 ifp->if_dname = ovpngroupname; 2414 ifp->if_dunit = unit; 2415 2416 ifp->if_addrlen = 0; 2417 ifp->if_mtu = 1428; 2418 ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST; 2419 ifp->if_ioctl = ovpn_ioctl; 2420 ifp->if_transmit = ovpn_transmit; 2421 ifp->if_output = ovpn_output; 2422 ifp->if_qflush = ovpn_qflush; 2423 #ifdef VIMAGE 2424 ifp->if_reassign = ovpn_reassign; 2425 #endif 2426 ifp->if_capabilities |= IFCAP_LINKSTATE; 2427 ifp->if_capenable |= IFCAP_LINKSTATE; 2428 2429 if_attach(ifp); 2430 bpfattach(ifp, DLT_NULL, sizeof(uint32_t)); 2431 *ifpp = ifp; 2432 2433 return (0); 2434 } 2435 2436 static void 2437 ovpn_clone_destroy_cb(struct epoch_context *ctx) 2438 { 2439 struct ovpn_softc *sc; 2440 2441 sc = __containerof(ctx, struct ovpn_softc, epoch_ctx); 2442 2443 MPASS(sc->peercount == 0); 2444 for (int i = 0; i < OVPN_MAX_PEERS; i++) { 2445 MPASS(sc->peers[i] == NULL); 2446 } 2447 2448 COUNTER_ARRAY_FREE(sc->counters, OVPN_COUNTER_SIZE); 2449 2450 if_free(sc->ifp); 2451 free(sc, M_OVPN); 2452 } 2453 2454 static int 2455 ovpn_clone_destroy(struct if_clone *ifc, struct ifnet *ifp, uint32_t flags) 2456 { 2457 struct ovpn_softc *sc; 2458 int unit; 2459 int i; 2460 int ret __diagused; 2461 2462 sc = ifp->if_softc; 2463 unit = ifp->if_dunit; 2464 2465 OVPN_WLOCK(sc); 2466 2467 if (atomic_load_int(&sc->refcount) > 0) { 2468 OVPN_WUNLOCK(sc); 2469 return (EBUSY); 2470 } 2471 2472 i = 0; 2473 do { 2474 if (sc->peers[i] != NULL) { 2475 ret = _ovpn_del_peer(sc, sc->peers[i]->peerid); 2476 MPASS(ret == 0); 2477 } 2478 i++; 2479 } while (i < OVPN_MAX_PEERS); 2480 2481 ovpn_flush_rxring(sc); 2482 buf_ring_free(sc->rxring, M_OVPN); 2483 buf_ring_free(sc->notifring, M_OVPN); 2484 2485 OVPN_WUNLOCK(sc); 2486 2487 bpfdetach(ifp); 2488 if_detach(ifp); 2489 ifp->if_softc = NULL; 2490 2491 NET_EPOCH_CALL(ovpn_clone_destroy_cb, &sc->epoch_ctx); 2492 2493 if (unit != IF_DUNIT_NONE) 2494 ifc_free_unit(ifc, unit); 2495 2496 NET_EPOCH_DRAIN_CALLBACKS(); 2497 2498 return (0); 2499 } 2500 2501 static void 2502 vnet_ovpn_init(const void *unused __unused) 2503 { 2504 struct if_clone_addreq req = { 2505 .match_f = ovpn_clone_match, 2506 .create_f = ovpn_clone_create, 2507 .destroy_f = ovpn_clone_destroy, 2508 }; 2509 V_ovpn_cloner = ifc_attach_cloner(ovpngroupname, &req); 2510 } 2511 VNET_SYSINIT(vnet_ovpn_init, SI_SUB_PSEUDO, SI_ORDER_ANY, 2512 vnet_ovpn_init, NULL); 2513 2514 static void 2515 vnet_ovpn_uninit(const void *unused __unused) 2516 { 2517 if_clone_detach(V_ovpn_cloner); 2518 } 2519 VNET_SYSUNINIT(vnet_ovpn_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY, 2520 vnet_ovpn_uninit, NULL); 2521 2522 static int 2523 ovpnmodevent(module_t mod, int type, void *data) 2524 { 2525 switch (type) { 2526 case MOD_LOAD: 2527 /* Done in vnet_ovpn_init() */ 2528 break; 2529 case MOD_UNLOAD: 2530 /* Done in vnet_ovpn_uninit() */ 2531 break; 2532 default: 2533 return (EOPNOTSUPP); 2534 } 2535 2536 return (0); 2537 } 2538 2539 static moduledata_t ovpn_mod = { 2540 "if_ovpn", 2541 ovpnmodevent, 2542 0 2543 }; 2544 2545 DECLARE_MODULE(if_ovpn, ovpn_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 2546 MODULE_VERSION(if_ovpn, 1); 2547