1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2021-2022 Rubicon Communications, LLC (Netgate) 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 */ 28 #include "opt_inet.h" 29 #include "opt_inet6.h" 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/buf_ring.h> 34 #include <sys/epoch.h> 35 #include <sys/file.h> 36 #include <sys/filedesc.h> 37 #include <sys/kernel.h> 38 #include <sys/malloc.h> 39 #include <sys/mbuf.h> 40 #include <sys/module.h> 41 #include <sys/nv.h> 42 #include <sys/priv.h> 43 #include <sys/protosw.h> 44 #include <sys/rmlock.h> 45 #include <sys/sdt.h> 46 #include <sys/smp.h> 47 #include <sys/socket.h> 48 #include <sys/socketvar.h> 49 #include <sys/sockio.h> 50 #include <sys/sysctl.h> 51 #include <sys/time.h> 52 53 #include <machine/atomic.h> 54 55 #include <net/bpf.h> 56 #include <net/if.h> 57 #include <net/if_clone.h> 58 #include <net/if_types.h> 59 #include <net/if_var.h> 60 #include <net/netisr.h> 61 #include <net/route/nhop.h> 62 63 #include <netinet/in.h> 64 #include <netinet/in_fib.h> 65 #include <netinet/ip.h> 66 #include <netinet/ip6.h> 67 #include <netinet/ip_var.h> 68 #include <netinet/udp.h> 69 #include <netinet/udp_var.h> 70 71 #include <netinet6/ip6_var.h> 72 #include <netinet6/in6_fib.h> 73 74 #include <machine/in_cksum.h> 75 76 #include <opencrypto/cryptodev.h> 77 78 #include "if_ovpn.h" 79 80 struct ovpn_kkey_dir { 81 int refcount; 82 uint8_t key[32]; 83 uint8_t keylen; 84 uint8_t nonce[8]; 85 uint8_t noncelen; 86 enum ovpn_key_cipher cipher; 87 crypto_session_t cryptoid; 88 89 struct mtx replay_mtx; 90 /* 91 * Last seen gapless sequence number. New rx seq numbers must be 92 * strictly higher than this. 93 */ 94 uint32_t rx_seq; 95 /* Seen packets, relative to rx_seq. bit(0) will always be 0. */ 96 uint64_t rx_window; 97 }; 98 99 struct ovpn_kkey { 100 struct ovpn_kkey_dir *encrypt; 101 struct ovpn_kkey_dir *decrypt; 102 uint8_t keyid; 103 uint32_t peerid; 104 }; 105 106 struct ovpn_keepalive { 107 uint32_t interval; 108 uint32_t timeout; 109 }; 110 111 struct ovpn_wire_header { 112 uint32_t opcode; /* opcode, key id, peer id */ 113 uint32_t seq; 114 uint8_t auth_tag[16]; 115 }; 116 117 struct ovpn_peer_counters { 118 uint64_t pkt_in; 119 uint64_t pkt_out; 120 uint64_t bytes_in; 121 uint64_t bytes_out; 122 }; 123 #define OVPN_PEER_COUNTER_SIZE (sizeof(struct ovpn_peer_counters)/sizeof(uint64_t)) 124 125 struct ovpn_notification { 126 enum ovpn_notif_type type; 127 uint32_t peerid; 128 129 /* Delete notification */ 130 enum ovpn_del_reason del_reason; 131 struct ovpn_peer_counters counters; 132 }; 133 134 struct ovpn_softc; 135 136 struct ovpn_kpeer { 137 RB_ENTRY(ovpn_kpeer) tree; 138 int refcount; 139 uint32_t peerid; 140 141 struct ovpn_softc *sc; 142 struct sockaddr_storage local; 143 struct sockaddr_storage remote; 144 145 struct in_addr vpn4; 146 struct in6_addr vpn6; 147 148 struct ovpn_kkey keys[2]; 149 uint32_t tx_seq; 150 151 enum ovpn_del_reason del_reason; 152 struct ovpn_keepalive keepalive; 153 uint32_t *last_active; 154 struct callout ping_send; 155 struct callout ping_rcv; 156 157 counter_u64_t counters[OVPN_PEER_COUNTER_SIZE]; 158 }; 159 160 struct ovpn_counters { 161 uint64_t lost_ctrl_pkts_in; 162 uint64_t lost_ctrl_pkts_out; 163 uint64_t lost_data_pkts_in; 164 uint64_t lost_data_pkts_out; 165 uint64_t nomem_data_pkts_in; 166 uint64_t nomem_data_pkts_out; 167 uint64_t received_ctrl_pkts; 168 uint64_t received_data_pkts; 169 uint64_t sent_ctrl_pkts; 170 uint64_t sent_data_pkts; 171 172 uint64_t transport_bytes_sent; 173 uint64_t transport_bytes_received; 174 uint64_t tunnel_bytes_sent; 175 uint64_t tunnel_bytes_received; 176 }; 177 #define OVPN_COUNTER_SIZE (sizeof(struct ovpn_counters)/sizeof(uint64_t)) 178 179 RB_HEAD(ovpn_kpeers, ovpn_kpeer); 180 181 struct ovpn_softc { 182 int refcount; 183 struct rmlock lock; 184 struct ifnet *ifp; 185 struct socket *so; 186 int peercount; 187 struct ovpn_kpeers peers; 188 189 /* Pending notification */ 190 struct buf_ring *notifring; 191 192 counter_u64_t counters[OVPN_COUNTER_SIZE]; 193 194 struct epoch_context epoch_ctx; 195 }; 196 197 static struct ovpn_kpeer *ovpn_find_peer(struct ovpn_softc *, uint32_t); 198 static bool ovpn_udp_input(struct mbuf *, int, struct inpcb *, 199 const struct sockaddr *, void *); 200 static int ovpn_transmit_to_peer(struct ifnet *, struct mbuf *, 201 struct ovpn_kpeer *, struct rm_priotracker *); 202 static int ovpn_encap(struct ovpn_softc *, uint32_t, struct mbuf *); 203 static int ovpn_get_af(struct mbuf *); 204 static void ovpn_free_kkey_dir(struct ovpn_kkey_dir *); 205 static bool ovpn_check_replay(struct ovpn_kkey_dir *, uint32_t); 206 static int ovpn_peer_compare(struct ovpn_kpeer *, struct ovpn_kpeer *); 207 208 static RB_PROTOTYPE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare); 209 static RB_GENERATE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare); 210 211 #define OVPN_MTU_MIN 576 212 #define OVPN_MTU_MAX (IP_MAXPACKET - sizeof(struct ip) - \ 213 sizeof(struct udphdr) - sizeof(struct ovpn_wire_header)) 214 215 #define OVPN_OP_DATA_V2 0x09 216 #define OVPN_OP_SHIFT 3 217 218 VNET_DEFINE_STATIC(struct if_clone *, ovpn_cloner); 219 #define V_ovpn_cloner VNET(ovpn_cloner) 220 221 #define OVPN_RLOCK_TRACKER struct rm_priotracker _ovpn_lock_tracker; \ 222 struct rm_priotracker *_ovpn_lock_trackerp = &_ovpn_lock_tracker 223 #define OVPN_RLOCK(sc) rm_rlock(&(sc)->lock, _ovpn_lock_trackerp) 224 #define OVPN_RUNLOCK(sc) rm_runlock(&(sc)->lock, _ovpn_lock_trackerp) 225 #define OVPN_WLOCK(sc) rm_wlock(&(sc)->lock) 226 #define OVPN_WUNLOCK(sc) rm_wunlock(&(sc)->lock) 227 #define OVPN_ASSERT(sc) rm_assert(&(sc)->lock, RA_LOCKED) 228 #define OVPN_RASSERT(sc) rm_assert(&(sc)->lock, RA_RLOCKED) 229 #define OVPN_WASSERT(sc) rm_assert(&(sc)->lock, RA_WLOCKED) 230 #define OVPN_UNLOCK_ASSERT(sc) rm_assert(&(sc)->lock, RA_UNLOCKED) 231 232 #define OVPN_COUNTER(sc, name) \ 233 ((sc)->counters[offsetof(struct ovpn_counters, name)/sizeof(uint64_t)]) 234 #define OVPN_PEER_COUNTER(peer, name) \ 235 ((peer)->counters[offsetof(struct ovpn_peer_counters, name) / \ 236 sizeof(uint64_t)]) 237 238 #define OVPN_COUNTER_ADD(sc, name, val) \ 239 counter_u64_add(OVPN_COUNTER(sc, name), val) 240 #define OVPN_PEER_COUNTER_ADD(p, name, val) \ 241 counter_u64_add(OVPN_PEER_COUNTER(p, name), val) 242 243 #define TO_IN(x) ((struct sockaddr_in *)(x)) 244 #define TO_IN6(x) ((struct sockaddr_in6 *)(x)) 245 246 SDT_PROVIDER_DEFINE(if_ovpn); 247 SDT_PROBE_DEFINE1(if_ovpn, tx, transmit, start, "struct mbuf *"); 248 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip4, "struct in_addr *", "struct ovpn_kpeer *"); 249 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip6, "struct in6_addr *", "struct ovpn_kpeer *"); 250 251 static const char ovpnname[] = "ovpn"; 252 static const char ovpngroupname[] = "openvpn"; 253 254 static MALLOC_DEFINE(M_OVPN, ovpnname, "OpenVPN DCO Interface"); 255 256 SYSCTL_DECL(_net_link); 257 static SYSCTL_NODE(_net_link, IFT_OTHER, openvpn, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 258 "OpenVPN DCO Interface"); 259 VNET_DEFINE_STATIC(int, replay_protection) = 0; 260 #define V_replay_protection VNET(replay_protection) 261 SYSCTL_INT(_net_link_openvpn, OID_AUTO, replay_protection, CTLFLAG_VNET | CTLFLAG_RW, 262 &VNET_NAME(replay_protection), 0, "Validate sequence numbers"); 263 264 VNET_DEFINE_STATIC(int, async_crypto); 265 #define V_async_crypto VNET(async_crypto) 266 SYSCTL_INT(_net_link_openvpn, OID_AUTO, async_crypto, 267 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_crypto), 0, 268 "Use asynchronous mode to parallelize crypto jobs."); 269 270 VNET_DEFINE_STATIC(int, async_netisr_queue); 271 #define V_async_netisr_queue VNET(async_netisr_queue) 272 SYSCTL_INT(_net_link_openvpn, OID_AUTO, netisr_queue, 273 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_netisr_queue), 0, 274 "Use netisr_queue() rather than netisr_dispatch()."); 275 276 static int 277 ovpn_peer_compare(struct ovpn_kpeer *a, struct ovpn_kpeer *b) 278 { 279 return (a->peerid - b->peerid); 280 } 281 282 static struct ovpn_kpeer * 283 ovpn_find_peer(struct ovpn_softc *sc, uint32_t peerid) 284 { 285 struct ovpn_kpeer p; 286 287 OVPN_ASSERT(sc); 288 289 p.peerid = peerid; 290 291 return (RB_FIND(ovpn_kpeers, &sc->peers, &p)); 292 } 293 294 static struct ovpn_kpeer * 295 ovpn_find_only_peer(struct ovpn_softc *sc) 296 { 297 OVPN_ASSERT(sc); 298 299 return (RB_ROOT(&sc->peers)); 300 } 301 302 static uint16_t 303 ovpn_get_port(struct sockaddr_storage *s) 304 { 305 switch (s->ss_family) { 306 case AF_INET: { 307 struct sockaddr_in *in = (struct sockaddr_in *)s; 308 return (in->sin_port); 309 } 310 case AF_INET6: { 311 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)s; 312 return (in6->sin6_port); 313 } 314 default: 315 panic("Unsupported address family %d", s->ss_family); 316 } 317 } 318 319 static int 320 ovpn_nvlist_to_sockaddr(const nvlist_t *nvl, struct sockaddr_storage *sa) 321 { 322 int af; 323 324 if (! nvlist_exists_number(nvl, "af")) 325 return (EINVAL); 326 if (! nvlist_exists_binary(nvl, "address")) 327 return (EINVAL); 328 if (! nvlist_exists_number(nvl, "port")) 329 return (EINVAL); 330 331 af = nvlist_get_number(nvl, "af"); 332 333 switch (af) { 334 #ifdef INET 335 case AF_INET: { 336 struct sockaddr_in *in = (struct sockaddr_in *)sa; 337 size_t len; 338 const void *addr = nvlist_get_binary(nvl, "address", &len); 339 in->sin_family = af; 340 if (len != sizeof(in->sin_addr)) 341 return (EINVAL); 342 343 memcpy(&in->sin_addr, addr, sizeof(in->sin_addr)); 344 in->sin_port = nvlist_get_number(nvl, "port"); 345 break; 346 } 347 #endif 348 #ifdef INET6 349 case AF_INET6: { 350 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)sa; 351 size_t len; 352 const void *addr = nvlist_get_binary(nvl, "address", &len); 353 in6->sin6_family = af; 354 if (len != sizeof(in6->sin6_addr)) 355 return (EINVAL); 356 357 memcpy(&in6->sin6_addr, addr, sizeof(in6->sin6_addr)); 358 in6->sin6_port = nvlist_get_number(nvl, "port"); 359 break; 360 } 361 #endif 362 default: 363 return (EINVAL); 364 } 365 366 return (0); 367 } 368 369 static bool 370 ovpn_has_peers(struct ovpn_softc *sc) 371 { 372 OVPN_ASSERT(sc); 373 374 return (sc->peercount > 0); 375 } 376 377 static void 378 ovpn_rele_so(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 379 { 380 bool has_peers; 381 382 OVPN_WASSERT(sc); 383 384 if (sc->so == NULL) 385 return; 386 387 has_peers = ovpn_has_peers(sc); 388 389 /* Only remove the tunnel function if we're releasing the socket for 390 * the last peer. */ 391 if (! has_peers) 392 (void)udp_set_kernel_tunneling(sc->so, NULL, NULL, NULL); 393 394 sorele(sc->so); 395 396 if (! has_peers) 397 sc->so = NULL; 398 } 399 400 static void 401 ovpn_notify_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 402 { 403 struct ovpn_notification *n; 404 405 OVPN_WASSERT(sc); 406 407 n = malloc(sizeof(*n), M_OVPN, M_NOWAIT); 408 if (n == NULL) 409 return; 410 411 n->peerid = peer->peerid; 412 n->type = OVPN_NOTIF_DEL_PEER; 413 n->del_reason = peer->del_reason; 414 415 n->counters.pkt_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_in)); 416 n->counters.pkt_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_out)); 417 n->counters.bytes_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_in)); 418 n->counters.bytes_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_out)); 419 420 if (buf_ring_enqueue(sc->notifring, n) != 0) { 421 free(n, M_OVPN); 422 } else if (sc->so != NULL) { 423 /* Wake up userspace */ 424 sc->so->so_error = EAGAIN; 425 sorwakeup(sc->so); 426 sowwakeup(sc->so); 427 } 428 } 429 430 static void 431 ovpn_peer_release_ref(struct ovpn_kpeer *peer, bool locked) 432 { 433 struct ovpn_softc *sc; 434 435 CURVNET_ASSERT_SET(); 436 437 atomic_add_int(&peer->refcount, -1); 438 439 if (atomic_load_int(&peer->refcount) > 0) 440 return; 441 442 sc = peer->sc; 443 444 if (! locked) { 445 OVPN_WLOCK(sc); 446 447 /* Might have changed before we acquired the lock. */ 448 if (atomic_load_int(&peer->refcount) > 0) { 449 OVPN_WUNLOCK(sc); 450 return; 451 } 452 } 453 454 OVPN_ASSERT(sc); 455 456 /* The peer should have been removed from the list already. */ 457 MPASS(ovpn_find_peer(sc, peer->peerid) == NULL); 458 459 ovpn_notify_del_peer(sc, peer); 460 461 for (int i = 0; i < 2; i++) { 462 ovpn_free_kkey_dir(peer->keys[i].encrypt); 463 ovpn_free_kkey_dir(peer->keys[i].decrypt); 464 } 465 466 ovpn_rele_so(sc, peer); 467 468 callout_stop(&peer->ping_send); 469 callout_stop(&peer->ping_rcv); 470 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 471 free(peer, M_OVPN); 472 473 if (! locked) 474 OVPN_WUNLOCK(sc); 475 } 476 477 static int 478 ovpn_new_peer(struct ifnet *ifp, const nvlist_t *nvl) 479 { 480 #ifdef INET6 481 struct epoch_tracker et; 482 #endif 483 struct sockaddr_storage remote; 484 struct ovpn_kpeer *peer = NULL; 485 struct file *fp = NULL; 486 struct sockaddr *name = NULL; 487 struct ovpn_softc *sc = ifp->if_softc; 488 struct thread *td = curthread; 489 struct socket *so = NULL; 490 int fd; 491 uint32_t peerid; 492 int ret = 0; 493 494 if (nvl == NULL) 495 return (EINVAL); 496 497 if (! nvlist_exists_number(nvl, "peerid")) 498 return (EINVAL); 499 500 if (! nvlist_exists_number(nvl, "fd")) 501 return (EINVAL); 502 503 if (! nvlist_exists_nvlist(nvl, "remote")) 504 return (EINVAL); 505 506 peerid = nvlist_get_number(nvl, "peerid"); 507 508 ret = ovpn_nvlist_to_sockaddr(nvlist_get_nvlist(nvl, "remote"), 509 &remote); 510 if (ret != 0) 511 return (ret); 512 513 fd = nvlist_get_number(nvl, "fd"); 514 515 /* Look up the userspace process and use the fd to find the socket. */ 516 ret = getsock(td, fd, &cap_connect_rights, &fp); 517 if (ret != 0) 518 return (ret); 519 520 so = fp->f_data; 521 522 peer = malloc(sizeof(*peer), M_OVPN, M_WAITOK | M_ZERO); 523 peer->peerid = peerid; 524 peer->sc = sc; 525 peer->tx_seq = 1; 526 peer->refcount = 1; 527 peer->last_active = uma_zalloc_pcpu(pcpu_zone_4, M_WAITOK | M_ZERO); 528 COUNTER_ARRAY_ALLOC(peer->counters, OVPN_PEER_COUNTER_SIZE, M_WAITOK); 529 530 if (nvlist_exists_binary(nvl, "vpn_ipv4")) { 531 size_t len; 532 const void *addr = nvlist_get_binary(nvl, "vpn_ipv4", &len); 533 if (len != sizeof(peer->vpn4)) { 534 ret = EINVAL; 535 goto error; 536 } 537 memcpy(&peer->vpn4, addr, len); 538 } 539 540 if (nvlist_exists_binary(nvl, "vpn_ipv6")) { 541 size_t len; 542 const void *addr = nvlist_get_binary(nvl, "vpn_ipv6", &len); 543 if (len != sizeof(peer->vpn6)) { 544 ret = EINVAL; 545 goto error; 546 } 547 memcpy(&peer->vpn6, addr, len); 548 } 549 550 callout_init_rm(&peer->ping_send, &sc->lock, CALLOUT_SHAREDLOCK); 551 callout_init_rm(&peer->ping_rcv, &sc->lock, 0); 552 553 ret = so->so_proto->pr_sockaddr(so, &name); 554 if (ret) 555 goto error; 556 557 if (ovpn_get_port((struct sockaddr_storage *)name) == 0) { 558 ret = EINVAL; 559 goto error; 560 } 561 if (name->sa_family != remote.ss_family) { 562 ret = EINVAL; 563 goto error; 564 } 565 566 memcpy(&peer->local, name, name->sa_len); 567 memcpy(&peer->remote, &remote, sizeof(remote)); 568 free(name, M_SONAME); 569 name = NULL; 570 571 if (peer->local.ss_family == AF_INET6 && 572 IN6_IS_ADDR_V4MAPPED(&TO_IN6(&peer->remote)->sin6_addr)) { 573 /* V4 mapped address, so treat this as v4, not v6. */ 574 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->local); 575 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->remote); 576 } 577 578 #ifdef INET6 579 if (peer->local.ss_family == AF_INET6 && 580 IN6_IS_ADDR_UNSPECIFIED(&TO_IN6(&peer->local)->sin6_addr)) { 581 NET_EPOCH_ENTER(et); 582 ret = in6_selectsrc_addr(curthread->td_proc->p_fibnum, 583 &TO_IN6(&peer->remote)->sin6_addr, 584 0, NULL, &TO_IN6(&peer->local)->sin6_addr, NULL); 585 NET_EPOCH_EXIT(et); 586 if (ret != 0) { 587 goto error; 588 } 589 } 590 #endif 591 OVPN_WLOCK(sc); 592 593 /* Disallow peer id re-use. */ 594 if (ovpn_find_peer(sc, peerid) != NULL) { 595 ret = EEXIST; 596 goto error_locked; 597 } 598 599 /* Make sure this is really a UDP socket. */ 600 if (so->so_type != SOCK_DGRAM || so->so_proto->pr_type != SOCK_DGRAM) { 601 ret = EPROTOTYPE; 602 goto error_locked; 603 } 604 605 /* Must be the same socket as for other peers on this interface. */ 606 if (sc->so != NULL && so != sc->so) 607 goto error_locked; 608 609 if (sc->so == NULL) 610 sc->so = so; 611 612 /* Insert the peer into the list. */ 613 RB_INSERT(ovpn_kpeers, &sc->peers, peer); 614 sc->peercount++; 615 soref(sc->so); 616 617 ret = udp_set_kernel_tunneling(sc->so, ovpn_udp_input, NULL, sc); 618 if (ret == EBUSY) { 619 /* Fine, another peer already set the input function. */ 620 ret = 0; 621 } 622 if (ret != 0) { 623 RB_REMOVE(ovpn_kpeers, &sc->peers, peer); 624 sc->peercount--; 625 goto error_locked; 626 } 627 628 OVPN_WUNLOCK(sc); 629 630 goto done; 631 632 error_locked: 633 OVPN_WUNLOCK(sc); 634 error: 635 free(name, M_SONAME); 636 COUNTER_ARRAY_FREE(peer->counters, OVPN_PEER_COUNTER_SIZE); 637 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 638 free(peer, M_OVPN); 639 done: 640 if (fp != NULL) 641 fdrop(fp, td); 642 643 return (ret); 644 } 645 646 static int 647 _ovpn_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 648 { 649 struct ovpn_kpeer *tmp __diagused; 650 651 OVPN_WASSERT(sc); 652 CURVNET_ASSERT_SET(); 653 654 MPASS(RB_FIND(ovpn_kpeers, &sc->peers, peer) == peer); 655 656 tmp = RB_REMOVE(ovpn_kpeers, &sc->peers, peer); 657 MPASS(tmp != NULL); 658 659 sc->peercount--; 660 661 ovpn_peer_release_ref(peer, true); 662 663 return (0); 664 } 665 666 static int 667 ovpn_del_peer(struct ifnet *ifp, nvlist_t *nvl) 668 { 669 struct ovpn_softc *sc = ifp->if_softc; 670 struct ovpn_kpeer *peer; 671 uint32_t peerid; 672 int ret; 673 674 OVPN_WASSERT(sc); 675 676 if (nvl == NULL) 677 return (EINVAL); 678 679 if (! nvlist_exists_number(nvl, "peerid")) 680 return (EINVAL); 681 682 peerid = nvlist_get_number(nvl, "peerid"); 683 684 peer = ovpn_find_peer(sc, peerid); 685 if (peer == NULL) 686 return (ENOENT); 687 688 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 689 ret = _ovpn_del_peer(sc, peer); 690 691 return (ret); 692 } 693 694 static int 695 ovpn_create_kkey_dir(struct ovpn_kkey_dir **kdirp, 696 const nvlist_t *nvl) 697 { 698 struct crypto_session_params csp; 699 struct ovpn_kkey_dir *kdir; 700 const char *ciphername; 701 enum ovpn_key_cipher cipher; 702 const void *key, *iv; 703 size_t keylen = 0, ivlen = 0; 704 int error; 705 706 if (! nvlist_exists_string(nvl, "cipher")) 707 return (EINVAL); 708 ciphername = nvlist_get_string(nvl, "cipher"); 709 710 if (strcmp(ciphername, "none") == 0) 711 cipher = OVPN_CIPHER_ALG_NONE; 712 else if (strcmp(ciphername, "AES-256-GCM") == 0 || 713 strcmp(ciphername, "AES-192-GCM") == 0 || 714 strcmp(ciphername, "AES-128-GCM") == 0) 715 cipher = OVPN_CIPHER_ALG_AES_GCM; 716 else if (strcmp(ciphername, "CHACHA20-POLY1305") == 0) 717 cipher = OVPN_CIPHER_ALG_CHACHA20_POLY1305; 718 else 719 return (EINVAL); 720 721 if (cipher != OVPN_CIPHER_ALG_NONE) { 722 if (! nvlist_exists_binary(nvl, "key")) 723 return (EINVAL); 724 key = nvlist_get_binary(nvl, "key", &keylen); 725 if (keylen > sizeof(kdir->key)) 726 return (E2BIG); 727 728 if (! nvlist_exists_binary(nvl, "iv")) 729 return (EINVAL); 730 iv = nvlist_get_binary(nvl, "iv", &ivlen); 731 if (ivlen != 8) 732 return (E2BIG); 733 } 734 735 kdir = malloc(sizeof(struct ovpn_kkey_dir), M_OVPN, 736 M_WAITOK | M_ZERO); 737 738 kdir->cipher = cipher; 739 kdir->keylen = keylen; 740 memcpy(kdir->key, key, keylen); 741 kdir->noncelen = ivlen; 742 memcpy(kdir->nonce, iv, ivlen); 743 744 if (kdir->cipher != OVPN_CIPHER_ALG_NONE) { 745 /* Crypto init */ 746 bzero(&csp, sizeof(csp)); 747 csp.csp_mode = CSP_MODE_AEAD; 748 749 if (kdir->cipher == OVPN_CIPHER_ALG_CHACHA20_POLY1305) 750 csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305; 751 else 752 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 753 754 csp.csp_flags |= CSP_F_SEPARATE_AAD; 755 756 csp.csp_cipher_klen = kdir->keylen; 757 csp.csp_cipher_key = kdir->key; 758 csp.csp_ivlen = 96 / 8; 759 760 error = crypto_newsession(&kdir->cryptoid, &csp, 761 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE); 762 if (error) { 763 free(kdir, M_OVPN); 764 return (error); 765 } 766 } 767 768 mtx_init(&kdir->replay_mtx, "if_ovpn rx replay", NULL, MTX_DEF); 769 *kdirp = kdir; 770 771 return (0); 772 } 773 774 static void 775 ovpn_free_kkey_dir(struct ovpn_kkey_dir *kdir) 776 { 777 if (kdir == NULL) 778 return; 779 780 mtx_destroy(&kdir->replay_mtx); 781 782 crypto_freesession(kdir->cryptoid); 783 free(kdir, M_OVPN); 784 } 785 786 static int 787 ovpn_set_key(struct ifnet *ifp, const nvlist_t *nvl) 788 { 789 struct ovpn_softc *sc = ifp->if_softc; 790 struct ovpn_kkey_dir *enc, *dec; 791 struct ovpn_kpeer *peer; 792 int slot, keyid, peerid; 793 int error; 794 795 if (nvl == NULL) 796 return (EINVAL); 797 798 if (! nvlist_exists_number(nvl, "slot")) 799 return (EINVAL); 800 slot = nvlist_get_number(nvl, "slot"); 801 802 if (! nvlist_exists_number(nvl, "keyid")) 803 return (EINVAL); 804 keyid = nvlist_get_number(nvl, "keyid"); 805 806 if (! nvlist_exists_number(nvl, "peerid")) 807 return (EINVAL); 808 peerid = nvlist_get_number(nvl, "peerid"); 809 810 if (slot != OVPN_KEY_SLOT_PRIMARY && 811 slot != OVPN_KEY_SLOT_SECONDARY) 812 return (EINVAL); 813 814 if (! nvlist_exists_nvlist(nvl, "encrypt") || 815 ! nvlist_exists_nvlist(nvl, "decrypt")) 816 return (EINVAL); 817 818 error = ovpn_create_kkey_dir(&enc, nvlist_get_nvlist(nvl, "encrypt")); 819 if (error) 820 return (error); 821 822 error = ovpn_create_kkey_dir(&dec, nvlist_get_nvlist(nvl, "decrypt")); 823 if (error) { 824 ovpn_free_kkey_dir(enc); 825 return (error); 826 } 827 828 OVPN_WLOCK(sc); 829 830 peer = ovpn_find_peer(sc, peerid); 831 if (peer == NULL) { 832 ovpn_free_kkey_dir(dec); 833 ovpn_free_kkey_dir(enc); 834 OVPN_WUNLOCK(sc); 835 return (ENOENT); 836 } 837 838 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 839 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 840 841 peer->keys[slot].encrypt = enc; 842 peer->keys[slot].decrypt = dec; 843 844 peer->keys[slot].keyid = keyid; 845 peer->keys[slot].peerid = peerid; 846 847 OVPN_WUNLOCK(sc); 848 849 return (0); 850 } 851 852 static int 853 ovpn_check_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, enum ovpn_key_slot slot) 854 { 855 OVPN_ASSERT(sc); 856 857 if (peer->keys[slot].encrypt == NULL) 858 return (ENOLINK); 859 860 if (peer->keys[slot].decrypt == NULL) 861 return (ENOLINK); 862 863 return (0); 864 } 865 866 static int 867 ovpn_start(struct ifnet *ifp) 868 { 869 struct ovpn_softc *sc = ifp->if_softc; 870 871 OVPN_WLOCK(sc); 872 873 ifp->if_flags |= IFF_UP; 874 ifp->if_drv_flags |= IFF_DRV_RUNNING; 875 if_link_state_change(ifp, LINK_STATE_UP); 876 877 OVPN_WUNLOCK(sc); 878 879 return (0); 880 } 881 882 static int 883 ovpn_swap_keys(struct ifnet *ifp, nvlist_t *nvl) 884 { 885 struct ovpn_softc *sc = ifp->if_softc; 886 struct ovpn_kpeer *peer; 887 struct ovpn_kkey tmpkey; 888 int error; 889 890 if (nvl == NULL) 891 return (EINVAL); 892 893 if (! nvlist_exists_number(nvl, "peerid")) 894 return (EINVAL); 895 896 OVPN_WLOCK(sc); 897 898 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 899 if (peer == NULL) { 900 OVPN_WUNLOCK(sc); 901 return (ENOENT); 902 } 903 904 /* Check that we have a second key to swap to. */ 905 error = ovpn_check_key(sc, peer, OVPN_KEY_SLOT_SECONDARY); 906 if (error) { 907 OVPN_WUNLOCK(sc); 908 return (error); 909 } 910 911 tmpkey = peer->keys[0]; 912 peer->keys[0] = peer->keys[1]; 913 peer->keys[1] = tmpkey; 914 915 OVPN_WUNLOCK(sc); 916 917 return (0); 918 } 919 920 static int 921 ovpn_del_key(struct ifnet *ifp, const nvlist_t *nvl) 922 { 923 enum ovpn_key_slot slot; 924 struct ovpn_kpeer *peer; 925 struct ovpn_softc *sc = ifp->if_softc; 926 927 if (nvl == NULL) 928 return (EINVAL); 929 930 if (! nvlist_exists_number(nvl, "peerid")) 931 return (EINVAL); 932 933 if (! nvlist_exists_number(nvl, "slot")) 934 return (EINVAL); 935 slot = nvlist_get_number(nvl, "slot"); 936 937 if (slot != OVPN_KEY_SLOT_PRIMARY && 938 slot != OVPN_KEY_SLOT_SECONDARY) 939 return (EINVAL); 940 941 OVPN_WLOCK(sc); 942 943 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 944 if (peer == NULL) { 945 OVPN_WUNLOCK(sc); 946 return (ENOENT); 947 } 948 949 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 950 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 951 952 peer->keys[slot].encrypt = NULL; 953 peer->keys[slot].decrypt = NULL; 954 955 peer->keys[slot].keyid = 0; 956 peer->keys[slot].peerid = 0; 957 958 OVPN_WUNLOCK(sc); 959 960 return (0); 961 } 962 963 static void 964 ovpn_send_ping(void *arg) 965 { 966 static const uint8_t ping_str[] = { 967 0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb, 968 0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48 969 }; 970 971 struct epoch_tracker et; 972 struct ovpn_kpeer *peer = arg; 973 struct ovpn_softc *sc = peer->sc; 974 struct mbuf *m; 975 976 OVPN_RASSERT(sc); 977 978 /* Ensure we repeat! */ 979 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 980 ovpn_send_ping, peer); 981 982 m = m_get2(sizeof(ping_str), M_NOWAIT, MT_DATA, M_PKTHDR); 983 if (m == NULL) 984 return; 985 986 m_copyback(m, 0, sizeof(ping_str), ping_str); 987 m->m_len = m->m_pkthdr.len = sizeof(ping_str); 988 989 CURVNET_SET(sc->ifp->if_vnet); 990 NET_EPOCH_ENTER(et); 991 (void)ovpn_transmit_to_peer(sc->ifp, m, peer, NULL); 992 NET_EPOCH_EXIT(et); 993 CURVNET_RESTORE(); 994 } 995 996 static void 997 ovpn_timeout(void *arg) 998 { 999 struct ovpn_kpeer *peer = arg; 1000 struct ovpn_softc *sc = peer->sc; 1001 uint32_t last, _last_active; 1002 int ret __diagused; 1003 int cpu; 1004 1005 OVPN_WASSERT(sc); 1006 1007 last = 0; 1008 CPU_FOREACH(cpu) { 1009 _last_active = *zpcpu_get_cpu(peer->last_active, cpu); 1010 if (_last_active > last) 1011 last = _last_active; 1012 } 1013 1014 if (last + peer->keepalive.timeout > time_uptime) { 1015 callout_reset(&peer->ping_rcv, 1016 (peer->keepalive.timeout - (time_uptime - last)) * hz, 1017 ovpn_timeout, peer); 1018 return; 1019 } 1020 1021 CURVNET_SET(sc->ifp->if_vnet); 1022 peer->del_reason = OVPN_DEL_REASON_TIMEOUT; 1023 ret = _ovpn_del_peer(sc, peer); 1024 MPASS(ret == 0); 1025 CURVNET_RESTORE(); 1026 } 1027 1028 static int 1029 ovpn_set_peer(struct ifnet *ifp, const nvlist_t *nvl) 1030 { 1031 struct ovpn_softc *sc = ifp->if_softc; 1032 struct ovpn_kpeer *peer; 1033 1034 if (nvl == NULL) 1035 return (EINVAL); 1036 1037 if (! nvlist_exists_number(nvl, "interval") || 1038 ! nvlist_exists_number(nvl, "timeout") || 1039 ! nvlist_exists_number(nvl, "peerid")) 1040 return (EINVAL); 1041 1042 OVPN_WLOCK(sc); 1043 1044 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 1045 if (peer == NULL) { 1046 OVPN_WUNLOCK(sc); 1047 return (ENOENT); 1048 } 1049 1050 peer->keepalive.interval = nvlist_get_number(nvl, "interval"); 1051 peer->keepalive.timeout = nvlist_get_number(nvl, "timeout"); 1052 1053 if (peer->keepalive.interval > 0) 1054 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 1055 ovpn_send_ping, peer); 1056 if (peer->keepalive.timeout > 0) 1057 callout_reset(&peer->ping_rcv, peer->keepalive.timeout * hz, 1058 ovpn_timeout, peer); 1059 1060 OVPN_WUNLOCK(sc); 1061 1062 return (0); 1063 } 1064 1065 static int 1066 ovpn_set_ifmode(struct ifnet *ifp, const nvlist_t *nvl) 1067 { 1068 struct ovpn_softc *sc = ifp->if_softc; 1069 int ifmode; 1070 1071 if (nvl == NULL) 1072 return (EINVAL); 1073 1074 if (! nvlist_exists_number(nvl, "ifmode") ) 1075 return (EINVAL); 1076 1077 ifmode = nvlist_get_number(nvl, "ifmode"); 1078 1079 OVPN_WLOCK(sc); 1080 1081 /* deny this if UP */ 1082 if (ifp->if_flags & IFF_UP) { 1083 OVPN_WUNLOCK(sc); 1084 return (EBUSY); 1085 } 1086 1087 switch (ifmode & ~IFF_MULTICAST) { 1088 case IFF_POINTOPOINT: 1089 case IFF_BROADCAST: 1090 ifp->if_flags &= 1091 ~(IFF_BROADCAST|IFF_POINTOPOINT|IFF_MULTICAST); 1092 ifp->if_flags |= ifmode; 1093 break; 1094 default: 1095 OVPN_WUNLOCK(sc); 1096 return (EINVAL); 1097 } 1098 1099 OVPN_WUNLOCK(sc); 1100 1101 return (0); 1102 } 1103 1104 static int 1105 ovpn_ioctl_set(struct ifnet *ifp, struct ifdrv *ifd) 1106 { 1107 struct ovpn_softc *sc = ifp->if_softc; 1108 uint8_t *buf = NULL; 1109 nvlist_t *nvl = NULL; 1110 int ret; 1111 1112 if (ifd->ifd_len != 0) { 1113 if (ifd->ifd_len > OVPN_MAX_REQUEST_SIZE) 1114 return (E2BIG); 1115 1116 buf = malloc(ifd->ifd_len, M_OVPN, M_WAITOK); 1117 1118 ret = copyin(ifd->ifd_data, buf, ifd->ifd_len); 1119 if (ret != 0) { 1120 free(buf, M_OVPN); 1121 return (ret); 1122 } 1123 1124 nvl = nvlist_unpack(buf, ifd->ifd_len, 0); 1125 free(buf, M_OVPN); 1126 if (nvl == NULL) { 1127 return (EINVAL); 1128 } 1129 } 1130 1131 switch (ifd->ifd_cmd) { 1132 case OVPN_NEW_PEER: 1133 ret = ovpn_new_peer(ifp, nvl); 1134 break; 1135 case OVPN_DEL_PEER: 1136 OVPN_WLOCK(sc); 1137 ret = ovpn_del_peer(ifp, nvl); 1138 OVPN_WUNLOCK(sc); 1139 break; 1140 case OVPN_NEW_KEY: 1141 ret = ovpn_set_key(ifp, nvl); 1142 break; 1143 case OVPN_START_VPN: 1144 ret = ovpn_start(ifp); 1145 break; 1146 case OVPN_SWAP_KEYS: 1147 ret = ovpn_swap_keys(ifp, nvl); 1148 break; 1149 case OVPN_DEL_KEY: 1150 ret = ovpn_del_key(ifp, nvl); 1151 break; 1152 case OVPN_SET_PEER: 1153 ret = ovpn_set_peer(ifp, nvl); 1154 break; 1155 case OVPN_SET_IFMODE: 1156 ret = ovpn_set_ifmode(ifp, nvl); 1157 break; 1158 default: 1159 ret = ENOTSUP; 1160 } 1161 1162 nvlist_destroy(nvl); 1163 return (ret); 1164 } 1165 1166 static int 1167 ovpn_add_counters(nvlist_t *parent, const char *name, counter_u64_t in, 1168 counter_u64_t out) 1169 { 1170 nvlist_t *nvl; 1171 1172 nvl = nvlist_create(0); 1173 if (nvl == NULL) 1174 return (ENOMEM); 1175 1176 nvlist_add_number(nvl, "in", counter_u64_fetch(in)); 1177 nvlist_add_number(nvl, "out", counter_u64_fetch(out)); 1178 1179 nvlist_add_nvlist(parent, name, nvl); 1180 1181 nvlist_destroy(nvl); 1182 1183 return (0); 1184 } 1185 1186 static int 1187 ovpn_get_stats(struct ovpn_softc *sc, nvlist_t **onvl) 1188 { 1189 nvlist_t *nvl; 1190 int ret; 1191 1192 nvl = nvlist_create(0); 1193 if (nvl == NULL) 1194 return (ENOMEM); 1195 1196 #define OVPN_COUNTER_OUT(name, in, out) \ 1197 do { \ 1198 ret = ovpn_add_counters(nvl, name, OVPN_COUNTER(sc, in), \ 1199 OVPN_COUNTER(sc, out)); \ 1200 if (ret != 0) \ 1201 goto error; \ 1202 } while(0) 1203 1204 OVPN_COUNTER_OUT("lost_ctrl", lost_ctrl_pkts_in, lost_ctrl_pkts_out); 1205 OVPN_COUNTER_OUT("lost_data", lost_data_pkts_in, lost_data_pkts_out); 1206 OVPN_COUNTER_OUT("nomem_data", nomem_data_pkts_in, 1207 nomem_data_pkts_out); 1208 OVPN_COUNTER_OUT("data", received_data_pkts, sent_data_pkts); 1209 OVPN_COUNTER_OUT("ctrl", received_ctrl_pkts, sent_ctrl_pkts); 1210 OVPN_COUNTER_OUT("tunnel", tunnel_bytes_received, 1211 tunnel_bytes_received); 1212 OVPN_COUNTER_OUT("transport", transport_bytes_received, 1213 transport_bytes_received); 1214 #undef OVPN_COUNTER_OUT 1215 1216 *onvl = nvl; 1217 1218 return (0); 1219 1220 error: 1221 nvlist_destroy(nvl); 1222 return (ret); 1223 } 1224 1225 static int 1226 ovpn_get_peer_stats(struct ovpn_softc *sc, nvlist_t **nvl) 1227 { 1228 struct ovpn_kpeer *peer; 1229 nvlist_t *nvpeer = NULL; 1230 int ret; 1231 1232 OVPN_RLOCK_TRACKER; 1233 1234 *nvl = nvlist_create(0); 1235 if (*nvl == NULL) 1236 return (ENOMEM); 1237 1238 #define OVPN_PEER_COUNTER_OUT(name, in, out) \ 1239 do { \ 1240 ret = ovpn_add_counters(nvpeer, name, \ 1241 OVPN_PEER_COUNTER(peer, in), OVPN_PEER_COUNTER(peer, out)); \ 1242 if (ret != 0) \ 1243 goto error; \ 1244 } while(0) 1245 1246 OVPN_RLOCK(sc); 1247 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1248 nvpeer = nvlist_create(0); 1249 if (nvpeer == NULL) { 1250 OVPN_RUNLOCK(sc); 1251 nvlist_destroy(*nvl); 1252 *nvl = NULL; 1253 return (ENOMEM); 1254 } 1255 1256 nvlist_add_number(nvpeer, "peerid", peer->peerid); 1257 1258 OVPN_PEER_COUNTER_OUT("packets", pkt_in, pkt_out); 1259 OVPN_PEER_COUNTER_OUT("bytes", bytes_in, bytes_out); 1260 1261 nvlist_append_nvlist_array(*nvl, "peers", nvpeer); 1262 nvlist_destroy(nvpeer); 1263 } 1264 #undef OVPN_PEER_COUNTER_OUT 1265 OVPN_RUNLOCK(sc); 1266 1267 return (0); 1268 1269 error: 1270 nvlist_destroy(nvpeer); 1271 nvlist_destroy(*nvl); 1272 *nvl = NULL; 1273 return (ret); 1274 } 1275 1276 static int 1277 ovpn_poll_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1278 { 1279 nvlist_t *nvl; 1280 1281 nvl = nvlist_create(0); 1282 if (nvl == NULL) 1283 return (ENOMEM); 1284 1285 nvlist_add_number(nvl, "pending", buf_ring_count(sc->notifring)); 1286 1287 *onvl = nvl; 1288 1289 return (0); 1290 } 1291 1292 static void 1293 ovpn_notif_add_counters(nvlist_t *parent, struct ovpn_notification *n) 1294 { 1295 nvlist_t *nvl; 1296 1297 nvl = nvlist_create(0); 1298 if (nvl == NULL) 1299 return; 1300 1301 nvlist_add_number(nvl, "in", n->counters.pkt_in); 1302 nvlist_add_number(nvl, "out", n->counters.pkt_out); 1303 1304 nvlist_add_nvlist(parent, "packets", nvl); 1305 nvlist_destroy(nvl); 1306 1307 nvl = nvlist_create(0); 1308 if (nvl == NULL) 1309 return; 1310 1311 nvlist_add_number(nvl, "in", n->counters.bytes_in); 1312 nvlist_add_number(nvl, "out", n->counters.bytes_out); 1313 1314 nvlist_add_nvlist(parent, "bytes", nvl); 1315 nvlist_destroy(nvl); 1316 } 1317 1318 static int 1319 opvn_get_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1320 { 1321 struct ovpn_notification *n; 1322 nvlist_t *nvl; 1323 1324 /* Check if we have notifications pending. */ 1325 n = buf_ring_dequeue_mc(sc->notifring); 1326 if (n == NULL) 1327 return (ENOENT); 1328 1329 nvl = nvlist_create(0); 1330 if (nvl == NULL) { 1331 free(n, M_OVPN); 1332 return (ENOMEM); 1333 } 1334 nvlist_add_number(nvl, "peerid", n->peerid); 1335 nvlist_add_number(nvl, "notification", n->type); 1336 if (n->type == OVPN_NOTIF_DEL_PEER) { 1337 nvlist_add_number(nvl, "del_reason", n->del_reason); 1338 1339 /* No error handling, because we want to send the notification 1340 * even if we can't attach the counters. */ 1341 ovpn_notif_add_counters(nvl, n); 1342 } 1343 free(n, M_OVPN); 1344 1345 *onvl = nvl; 1346 1347 return (0); 1348 } 1349 1350 static int 1351 ovpn_ioctl_get(struct ifnet *ifp, struct ifdrv *ifd) 1352 { 1353 struct ovpn_softc *sc = ifp->if_softc; 1354 nvlist_t *nvl = NULL; 1355 int error; 1356 1357 switch (ifd->ifd_cmd) { 1358 case OVPN_GET_STATS: 1359 error = ovpn_get_stats(sc, &nvl); 1360 break; 1361 case OVPN_GET_PEER_STATS: 1362 error = ovpn_get_peer_stats(sc, &nvl); 1363 break; 1364 case OVPN_POLL_PKT: 1365 error = ovpn_poll_pkt(sc, &nvl); 1366 break; 1367 case OVPN_GET_PKT: 1368 error = opvn_get_pkt(sc, &nvl); 1369 break; 1370 default: 1371 error = ENOTSUP; 1372 break; 1373 } 1374 1375 if (error == 0) { 1376 void *packed = NULL; 1377 size_t len; 1378 1379 MPASS(nvl != NULL); 1380 1381 packed = nvlist_pack(nvl, &len); 1382 if (! packed) { 1383 nvlist_destroy(nvl); 1384 return (ENOMEM); 1385 } 1386 1387 if (len > ifd->ifd_len) { 1388 free(packed, M_NVLIST); 1389 nvlist_destroy(nvl); 1390 return (ENOSPC); 1391 } 1392 1393 error = copyout(packed, ifd->ifd_data, len); 1394 ifd->ifd_len = len; 1395 1396 free(packed, M_NVLIST); 1397 nvlist_destroy(nvl); 1398 } 1399 1400 return (error); 1401 } 1402 1403 static int 1404 ovpn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 1405 { 1406 struct ifdrv *ifd; 1407 int error; 1408 1409 CURVNET_ASSERT_SET(); 1410 1411 switch (cmd) { 1412 case SIOCSDRVSPEC: 1413 case SIOCGDRVSPEC: 1414 error = priv_check(curthread, PRIV_NET_OVPN); 1415 if (error) 1416 return (error); 1417 break; 1418 } 1419 1420 switch (cmd) { 1421 case SIOCSDRVSPEC: 1422 ifd = (struct ifdrv *)data; 1423 error = ovpn_ioctl_set(ifp, ifd); 1424 break; 1425 case SIOCGDRVSPEC: 1426 ifd = (struct ifdrv *)data; 1427 error = ovpn_ioctl_get(ifp, ifd); 1428 break; 1429 case SIOCSIFMTU: { 1430 struct ifreq *ifr = (struct ifreq *)data; 1431 if (ifr->ifr_mtu < OVPN_MTU_MIN || ifr->ifr_mtu > OVPN_MTU_MAX) 1432 return (EINVAL); 1433 1434 ifp->if_mtu = ifr->ifr_mtu; 1435 return (0); 1436 } 1437 case SIOCSIFADDR: 1438 case SIOCADDMULTI: 1439 case SIOCDELMULTI: 1440 case SIOCGIFMTU: 1441 case SIOCSIFFLAGS: 1442 return (0); 1443 default: 1444 error = EINVAL; 1445 } 1446 1447 return (error); 1448 } 1449 1450 static int 1451 ovpn_encrypt_tx_cb(struct cryptop *crp) 1452 { 1453 struct epoch_tracker et; 1454 struct ovpn_kpeer *peer = crp->crp_opaque; 1455 struct ovpn_softc *sc = peer->sc; 1456 struct mbuf *m = crp->crp_buf.cb_mbuf; 1457 int tunnel_len; 1458 int ret; 1459 1460 CURVNET_SET(sc->ifp->if_vnet); 1461 NET_EPOCH_ENTER(et); 1462 1463 if (crp->crp_etype != 0) { 1464 crypto_freereq(crp); 1465 ovpn_peer_release_ref(peer, false); 1466 NET_EPOCH_EXIT(et); 1467 CURVNET_RESTORE(); 1468 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1469 m_freem(m); 1470 return (0); 1471 } 1472 1473 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1474 1475 tunnel_len = m->m_pkthdr.len - sizeof(struct ovpn_wire_header); 1476 ret = ovpn_encap(sc, peer->peerid, m); 1477 if (ret == 0) { 1478 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1479 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 1480 } 1481 1482 crypto_freereq(crp); 1483 ovpn_peer_release_ref(peer, false); 1484 1485 NET_EPOCH_EXIT(et); 1486 CURVNET_RESTORE(); 1487 1488 return (0); 1489 } 1490 1491 static void 1492 ovpn_finish_rx(struct ovpn_softc *sc, struct mbuf *m, 1493 struct ovpn_kpeer *peer, struct ovpn_kkey *key, uint32_t seq, 1494 struct rm_priotracker *_ovpn_lock_trackerp) 1495 { 1496 uint32_t af; 1497 1498 OVPN_RASSERT(sc); 1499 NET_EPOCH_ASSERT(); 1500 1501 /* Replay protection. */ 1502 if (V_replay_protection && ! ovpn_check_replay(key->decrypt, seq)) { 1503 OVPN_RUNLOCK(sc); 1504 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1505 m_freem(m); 1506 return; 1507 } 1508 1509 critical_enter(); 1510 *zpcpu_get(peer->last_active) = time_uptime; 1511 critical_exit(); 1512 1513 OVPN_RUNLOCK(sc); 1514 1515 OVPN_COUNTER_ADD(sc, received_data_pkts, 1); 1516 OVPN_COUNTER_ADD(sc, tunnel_bytes_received, m->m_pkthdr.len); 1517 OVPN_PEER_COUNTER_ADD(peer, pkt_in, 1); 1518 OVPN_PEER_COUNTER_ADD(peer, bytes_in, m->m_pkthdr.len); 1519 1520 /* Receive the packet on our interface. */ 1521 m->m_pkthdr.rcvif = sc->ifp; 1522 1523 /* Clear checksum flags in case the real hardware set them. */ 1524 m->m_pkthdr.csum_flags = 0; 1525 1526 /* Ensure we can read the first byte. */ 1527 m = m_pullup(m, 1); 1528 if (m == NULL) { 1529 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 1530 return; 1531 } 1532 1533 /* 1534 * Check for address family, and disregard any control packets (e.g. 1535 * keepalive). 1536 */ 1537 af = ovpn_get_af(m); 1538 if (af != 0) { 1539 BPF_MTAP2(sc->ifp, &af, sizeof(af), m); 1540 if (V_async_netisr_queue) 1541 netisr_queue(af == AF_INET ? NETISR_IP : NETISR_IPV6, m); 1542 else 1543 netisr_dispatch(af == AF_INET ? NETISR_IP : NETISR_IPV6, m); 1544 } else { 1545 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1546 m_freem(m); 1547 } 1548 } 1549 1550 static struct ovpn_kkey * 1551 ovpn_find_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, 1552 const struct ovpn_wire_header *ohdr) 1553 { 1554 struct ovpn_kkey *key = NULL; 1555 uint8_t keyid; 1556 1557 OVPN_RASSERT(sc); 1558 1559 keyid = (ntohl(ohdr->opcode) >> 24) & 0x07; 1560 1561 if (peer->keys[0].keyid == keyid) 1562 key = &peer->keys[0]; 1563 else if (peer->keys[1].keyid == keyid) 1564 key = &peer->keys[1]; 1565 1566 return (key); 1567 } 1568 1569 static int 1570 ovpn_decrypt_rx_cb(struct cryptop *crp) 1571 { 1572 struct epoch_tracker et; 1573 struct ovpn_softc *sc = crp->crp_opaque; 1574 struct mbuf *m = crp->crp_buf.cb_mbuf; 1575 struct ovpn_kkey *key; 1576 struct ovpn_kpeer *peer; 1577 struct ovpn_wire_header *ohdr; 1578 uint32_t peerid; 1579 1580 OVPN_RLOCK_TRACKER; 1581 1582 OVPN_RLOCK(sc); 1583 1584 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1585 1586 if (crp->crp_etype != 0) { 1587 crypto_freereq(crp); 1588 atomic_add_int(&sc->refcount, -1); 1589 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1590 OVPN_RUNLOCK(sc); 1591 m_freem(m); 1592 return (0); 1593 } 1594 1595 CURVNET_SET(sc->ifp->if_vnet); 1596 1597 ohdr = mtodo(m, sizeof(struct udphdr)); 1598 1599 peerid = ntohl(ohdr->opcode) & 0x00ffffff; 1600 peer = ovpn_find_peer(sc, peerid); 1601 if (peer == NULL) { 1602 /* No such peer. Drop packet. */ 1603 crypto_freereq(crp); 1604 atomic_add_int(&sc->refcount, -1); 1605 OVPN_RUNLOCK(sc); 1606 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1607 m_freem(m); 1608 CURVNET_RESTORE(); 1609 return (0); 1610 } 1611 1612 key = ovpn_find_key(sc, peer, ohdr); 1613 if (key == NULL) { 1614 crypto_freereq(crp); 1615 atomic_add_int(&sc->refcount, -1); 1616 /* 1617 * Has this key been removed between us starting the decrypt 1618 * and finishing it? 1619 */ 1620 OVPN_RUNLOCK(sc); 1621 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1622 m_freem(m); 1623 CURVNET_RESTORE(); 1624 return (0); 1625 } 1626 1627 /* Now remove the outer headers */ 1628 m_adj_decap(m, sizeof(struct udphdr) + 1629 sizeof(struct ovpn_wire_header)); 1630 1631 NET_EPOCH_ENTER(et); 1632 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), _ovpn_lock_trackerp); 1633 NET_EPOCH_EXIT(et); 1634 OVPN_UNLOCK_ASSERT(sc); 1635 1636 CURVNET_RESTORE(); 1637 1638 crypto_freereq(crp); 1639 atomic_add_int(&sc->refcount, -1); 1640 1641 return (0); 1642 } 1643 1644 static int 1645 ovpn_get_af(struct mbuf *m) 1646 { 1647 struct ip *ip; 1648 struct ip6_hdr *ip6; 1649 1650 /* 1651 * We should pullup, but we're only interested in the first byte, so 1652 * that'll always be contiguous. 1653 */ 1654 ip = mtod(m, struct ip *); 1655 if (ip->ip_v == IPVERSION) 1656 return (AF_INET); 1657 1658 ip6 = mtod(m, struct ip6_hdr *); 1659 if ((ip6->ip6_vfc & IPV6_VERSION_MASK) == IPV6_VERSION) 1660 return (AF_INET6); 1661 1662 return (0); 1663 } 1664 1665 static struct ovpn_kpeer * 1666 ovpn_find_peer_by_ip(struct ovpn_softc *sc, const struct in_addr addr) 1667 { 1668 struct ovpn_kpeer *peer = NULL; 1669 1670 OVPN_ASSERT(sc); 1671 1672 /* TODO: Add a second RB so we can look up by IP. */ 1673 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1674 if (addr.s_addr == peer->vpn4.s_addr) 1675 return (peer); 1676 } 1677 1678 return (peer); 1679 } 1680 1681 static struct ovpn_kpeer * 1682 ovpn_find_peer_by_ip6(struct ovpn_softc *sc, const struct in6_addr *addr) 1683 { 1684 struct ovpn_kpeer *peer = NULL; 1685 1686 OVPN_ASSERT(sc); 1687 1688 /* TODO: Add a third RB so we can look up by IPv6 address. */ 1689 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1690 if (memcmp(addr, &peer->vpn6, sizeof(*addr)) == 0) 1691 return (peer); 1692 } 1693 1694 return (peer); 1695 } 1696 1697 static struct ovpn_kpeer * 1698 ovpn_route_peer(struct ovpn_softc *sc, struct mbuf **m0, 1699 const struct sockaddr *dst) 1700 { 1701 struct ovpn_kpeer *peer = NULL; 1702 int af; 1703 1704 NET_EPOCH_ASSERT(); 1705 OVPN_ASSERT(sc); 1706 1707 /* Shortcut if we're a client (or are a server and have only one client). */ 1708 if (sc->peercount == 1) 1709 return (ovpn_find_only_peer(sc)); 1710 1711 if (dst != NULL) 1712 af = dst->sa_family; 1713 else 1714 af = ovpn_get_af(*m0); 1715 1716 switch (af) { 1717 #ifdef INET 1718 case AF_INET: { 1719 const struct sockaddr_in *sa = (const struct sockaddr_in *)dst; 1720 struct nhop_object *nh; 1721 const struct in_addr *ip_dst; 1722 1723 if (sa != NULL) { 1724 ip_dst = &sa->sin_addr; 1725 } else { 1726 struct ip *ip; 1727 1728 *m0 = m_pullup(*m0, sizeof(struct ip)); 1729 if (*m0 == NULL) 1730 return (NULL); 1731 ip = mtod(*m0, struct ip *); 1732 ip_dst = &ip->ip_dst; 1733 } 1734 1735 peer = ovpn_find_peer_by_ip(sc, *ip_dst); 1736 SDT_PROBE2(if_ovpn, tx, route, ip4, ip_dst, peer); 1737 if (peer == NULL) { 1738 nh = fib4_lookup(M_GETFIB(*m0), *ip_dst, 0, 1739 NHR_NONE, 0); 1740 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1741 peer = ovpn_find_peer_by_ip(sc, 1742 nh->gw4_sa.sin_addr); 1743 SDT_PROBE2(if_ovpn, tx, route, ip4, 1744 &nh->gw4_sa.sin_addr, peer); 1745 } 1746 } 1747 break; 1748 } 1749 #endif 1750 #ifdef INET6 1751 case AF_INET6: { 1752 const struct sockaddr_in6 *sa6 = 1753 (const struct sockaddr_in6 *)dst; 1754 struct nhop_object *nh; 1755 const struct in6_addr *ip6_dst; 1756 1757 if (sa6 != NULL) { 1758 ip6_dst = &sa6->sin6_addr; 1759 } else { 1760 struct ip6_hdr *ip6; 1761 1762 *m0 = m_pullup(*m0, sizeof(struct ip6_hdr)); 1763 if (*m0 == NULL) 1764 return (NULL); 1765 ip6 = mtod(*m0, struct ip6_hdr *); 1766 ip6_dst = &ip6->ip6_dst; 1767 } 1768 1769 peer = ovpn_find_peer_by_ip6(sc, ip6_dst); 1770 SDT_PROBE2(if_ovpn, tx, route, ip6, ip6_dst, peer); 1771 if (peer == NULL) { 1772 nh = fib6_lookup(M_GETFIB(*m0), ip6_dst, 0, 1773 NHR_NONE, 0); 1774 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1775 peer = ovpn_find_peer_by_ip6(sc, 1776 &nh->gw6_sa.sin6_addr); 1777 SDT_PROBE2(if_ovpn, tx, route, ip6, 1778 &nh->gw6_sa.sin6_addr, peer); 1779 } 1780 } 1781 break; 1782 } 1783 #endif 1784 } 1785 1786 return (peer); 1787 } 1788 1789 static int 1790 ovpn_transmit(struct ifnet *ifp, struct mbuf *m) 1791 { 1792 return (ifp->if_output(ifp, m, NULL, NULL)); 1793 } 1794 1795 static int 1796 ovpn_transmit_to_peer(struct ifnet *ifp, struct mbuf *m, 1797 struct ovpn_kpeer *peer, struct rm_priotracker *_ovpn_lock_trackerp) 1798 { 1799 struct ovpn_wire_header *ohdr; 1800 struct ovpn_kkey *key; 1801 struct ovpn_softc *sc; 1802 struct cryptop *crp; 1803 uint32_t af, seq; 1804 size_t len, ovpn_hdr_len; 1805 int tunnel_len; 1806 int ret; 1807 1808 sc = ifp->if_softc; 1809 1810 OVPN_RASSERT(sc); 1811 1812 tunnel_len = m->m_pkthdr.len; 1813 1814 key = &peer->keys[OVPN_KEY_SLOT_PRIMARY]; 1815 if (key->encrypt == NULL) { 1816 if (_ovpn_lock_trackerp != NULL) 1817 OVPN_RUNLOCK(sc); 1818 m_freem(m); 1819 return (ENOLINK); 1820 } 1821 1822 af = ovpn_get_af(m); 1823 /* Don't capture control packets. */ 1824 if (af != 0) 1825 BPF_MTAP2(ifp, &af, sizeof(af), m); 1826 1827 len = m->m_pkthdr.len; 1828 MPASS(len <= ifp->if_mtu); 1829 1830 ovpn_hdr_len = sizeof(struct ovpn_wire_header); 1831 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) 1832 ovpn_hdr_len -= 16; /* No auth tag. */ 1833 1834 M_PREPEND(m, ovpn_hdr_len, M_NOWAIT); 1835 if (m == NULL) { 1836 if (_ovpn_lock_trackerp != NULL) 1837 OVPN_RUNLOCK(sc); 1838 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1839 return (ENOBUFS); 1840 } 1841 ohdr = mtod(m, struct ovpn_wire_header *); 1842 ohdr->opcode = (OVPN_OP_DATA_V2 << OVPN_OP_SHIFT) | key->keyid; 1843 ohdr->opcode <<= 24; 1844 ohdr->opcode |= key->peerid; 1845 ohdr->opcode = htonl(ohdr->opcode); 1846 1847 seq = atomic_fetchadd_32(&peer->tx_seq, 1); 1848 seq = htonl(seq); 1849 ohdr->seq = seq; 1850 1851 OVPN_PEER_COUNTER_ADD(peer, pkt_out, 1); 1852 OVPN_PEER_COUNTER_ADD(peer, bytes_out, len); 1853 1854 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) { 1855 ret = ovpn_encap(sc, peer->peerid, m); 1856 if (_ovpn_lock_trackerp != NULL) 1857 OVPN_RUNLOCK(sc); 1858 if (ret == 0) { 1859 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1860 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 1861 } 1862 return (ret); 1863 } 1864 1865 crp = crypto_getreq(key->encrypt->cryptoid, M_NOWAIT); 1866 if (crp == NULL) { 1867 if (_ovpn_lock_trackerp != NULL) 1868 OVPN_RUNLOCK(sc); 1869 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1870 m_freem(m); 1871 return (ENOBUFS); 1872 } 1873 1874 /* Encryption covers only the payload, not the header. */ 1875 crp->crp_payload_start = sizeof(*ohdr); 1876 crp->crp_payload_length = len; 1877 crp->crp_op = CRYPTO_OP_ENCRYPT; 1878 1879 /* 1880 * AAD data covers the ovpn_wire_header minus the auth 1881 * tag. 1882 */ 1883 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 1884 crp->crp_aad = ohdr; 1885 crp->crp_aad_start = 0; 1886 crp->crp_op |= CRYPTO_OP_COMPUTE_DIGEST; 1887 crp->crp_digest_start = offsetof(struct ovpn_wire_header, auth_tag); 1888 1889 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 1890 memcpy(crp->crp_iv, &seq, sizeof(seq)); 1891 memcpy(crp->crp_iv + sizeof(seq), key->encrypt->nonce, 1892 key->encrypt->noncelen); 1893 1894 crypto_use_mbuf(crp, m); 1895 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 1896 crp->crp_callback = ovpn_encrypt_tx_cb; 1897 crp->crp_opaque = peer; 1898 1899 atomic_add_int(&peer->refcount, 1); 1900 if (_ovpn_lock_trackerp != NULL) 1901 OVPN_RUNLOCK(sc); 1902 if (V_async_crypto) 1903 ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); 1904 else 1905 ret = crypto_dispatch(crp); 1906 if (ret) { 1907 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1908 } 1909 1910 return (ret); 1911 } 1912 1913 /* 1914 * Note: Expects to hold the read lock on entry, and will release it itself. 1915 */ 1916 static int 1917 ovpn_encap(struct ovpn_softc *sc, uint32_t peerid, struct mbuf *m) 1918 { 1919 struct udphdr *udp; 1920 struct ovpn_kpeer *peer; 1921 int len; 1922 1923 OVPN_RLOCK_TRACKER; 1924 1925 OVPN_RLOCK(sc); 1926 NET_EPOCH_ASSERT(); 1927 1928 peer = ovpn_find_peer(sc, peerid); 1929 if (peer == NULL || sc->ifp->if_link_state != LINK_STATE_UP) { 1930 OVPN_RUNLOCK(sc); 1931 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1932 m_freem(m); 1933 return (ENETDOWN); 1934 } 1935 1936 len = m->m_pkthdr.len; 1937 1938 M_PREPEND(m, sizeof(struct udphdr), M_NOWAIT); 1939 if (m == NULL) { 1940 OVPN_RUNLOCK(sc); 1941 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1942 m_freem(m); 1943 return (ENOBUFS); 1944 } 1945 udp = mtod(m, struct udphdr *); 1946 1947 MPASS(peer->local.ss_family == peer->remote.ss_family); 1948 1949 udp->uh_sport = ovpn_get_port(&peer->local); 1950 udp->uh_dport = ovpn_get_port(&peer->remote); 1951 udp->uh_ulen = htons(sizeof(struct udphdr) + len); 1952 1953 switch (peer->remote.ss_family) { 1954 #ifdef INET 1955 case AF_INET: { 1956 struct sockaddr_in *in_local = TO_IN(&peer->local); 1957 struct sockaddr_in *in_remote = TO_IN(&peer->remote); 1958 struct ip *ip; 1959 1960 /* 1961 * This requires knowing the source IP, which we don't. Happily 1962 * we're allowed to keep this at 0, and the checksum won't do 1963 * anything the crypto won't already do. 1964 */ 1965 udp->uh_sum = 0; 1966 1967 /* Set the checksum flags so we recalculate checksums. */ 1968 m->m_pkthdr.csum_flags |= CSUM_IP; 1969 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 1970 1971 M_PREPEND(m, sizeof(struct ip), M_NOWAIT); 1972 if (m == NULL) { 1973 OVPN_RUNLOCK(sc); 1974 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1975 return (ENOBUFS); 1976 } 1977 ip = mtod(m, struct ip *); 1978 1979 ip->ip_tos = 0; 1980 ip->ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) + 1981 len); 1982 ip->ip_off = 0; 1983 ip->ip_ttl = V_ip_defttl; 1984 ip->ip_p = IPPROTO_UDP; 1985 ip->ip_sum = 0; 1986 if (in_local->sin_port != 0) 1987 ip->ip_src = in_local->sin_addr; 1988 else 1989 ip->ip_src.s_addr = INADDR_ANY; 1990 ip->ip_dst = in_remote->sin_addr; 1991 1992 OVPN_RUNLOCK(sc); 1993 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 1994 1995 return (ip_output(m, NULL, NULL, 0, NULL, NULL)); 1996 } 1997 #endif 1998 #ifdef INET6 1999 case AF_INET6: { 2000 struct sockaddr_in6 *in6_local = TO_IN6(&peer->local); 2001 struct sockaddr_in6 *in6_remote = TO_IN6(&peer->remote); 2002 struct ip6_hdr *ip6; 2003 2004 M_PREPEND(m, sizeof(struct ip6_hdr), M_NOWAIT); 2005 if (m == NULL) { 2006 OVPN_RUNLOCK(sc); 2007 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2008 return (ENOBUFS); 2009 } 2010 m = m_pullup(m, sizeof(*ip6) + sizeof(*udp)); 2011 if (m == NULL) { 2012 OVPN_RUNLOCK(sc); 2013 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2014 return (ENOBUFS); 2015 } 2016 2017 ip6 = mtod(m, struct ip6_hdr *); 2018 2019 ip6->ip6_vfc = IPV6_VERSION; 2020 ip6->ip6_flow &= ~IPV6_FLOWINFO_MASK; 2021 ip6->ip6_plen = htons(sizeof(*ip6) + sizeof(struct udphdr) + 2022 len); 2023 ip6->ip6_nxt = IPPROTO_UDP; 2024 ip6->ip6_hlim = V_ip6_defhlim; 2025 2026 memcpy(&ip6->ip6_src, &in6_local->sin6_addr, 2027 sizeof(ip6->ip6_src)); 2028 memcpy(&ip6->ip6_dst, &in6_remote->sin6_addr, 2029 sizeof(ip6->ip6_dst)); 2030 2031 udp = mtodo(m, sizeof(*ip6)); 2032 udp->uh_sum = in6_cksum_pseudo(ip6, 2033 m->m_pkthdr.len - sizeof(struct ip6_hdr), 2034 IPPROTO_UDP, 0); 2035 2036 m->m_pkthdr.csum_flags |= CSUM_UDP_IPV6; 2037 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 2038 2039 OVPN_RUNLOCK(sc); 2040 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 2041 2042 return (ip6_output(m, NULL, NULL, IPV6_UNSPECSRC, NULL, NULL, 2043 NULL)); 2044 } 2045 #endif 2046 default: 2047 panic("Unsupported address family %d", 2048 peer->remote.ss_family); 2049 } 2050 } 2051 2052 static int 2053 ovpn_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 2054 struct route *ro) 2055 { 2056 struct ovpn_softc *sc; 2057 struct ovpn_kpeer *peer; 2058 2059 OVPN_RLOCK_TRACKER; 2060 2061 sc = ifp->if_softc; 2062 2063 OVPN_RLOCK(sc); 2064 2065 SDT_PROBE1(if_ovpn, tx, transmit, start, m); 2066 2067 if (__predict_false(ifp->if_link_state != LINK_STATE_UP)) { 2068 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2069 OVPN_RUNLOCK(sc); 2070 m_freem(m); 2071 return (ENETDOWN); 2072 } 2073 2074 /** 2075 * Only obey 'dst' (i.e. the gateway) if no route is supplied. 2076 * That's our indication that we're being called through pf's route-to, 2077 * and we should route according to 'dst' instead. We can't do so 2078 * consistently, because the usual openvpn configuration sets the first 2079 * non-server IP in the subnet as the gateway. If we always use that 2080 * one we'd end up routing all traffic to the first client. 2081 * tl;dr: 'ro == NULL' tells us pf is doing a route-to, and then but 2082 * only then, we should treat 'dst' as the destination. */ 2083 peer = ovpn_route_peer(sc, &m, ro == NULL ? dst : NULL); 2084 if (peer == NULL) { 2085 /* No destination. */ 2086 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2087 OVPN_RUNLOCK(sc); 2088 m_freem(m); 2089 return (ENETDOWN); 2090 } 2091 2092 return (ovpn_transmit_to_peer(ifp, m, peer, _ovpn_lock_trackerp)); 2093 } 2094 2095 static bool 2096 ovpn_check_replay(struct ovpn_kkey_dir *key, uint32_t seq) 2097 { 2098 uint32_t d; 2099 2100 mtx_lock(&key->replay_mtx); 2101 2102 /* Sequence number must be strictly greater than rx_seq */ 2103 if (seq <= key->rx_seq) { 2104 mtx_unlock(&key->replay_mtx); 2105 return (false); 2106 } 2107 2108 /* Large jump. The packet authenticated okay, so just accept that. */ 2109 if (seq > (key->rx_seq + (sizeof(key->rx_window) * 8))) { 2110 key->rx_seq = seq; 2111 key->rx_window = 0; 2112 mtx_unlock(&key->replay_mtx); 2113 return (true); 2114 } 2115 2116 /* Happy case. */ 2117 if ((seq == key->rx_seq + 1) && key->rx_window == 0) { 2118 key->rx_seq++; 2119 mtx_unlock(&key->replay_mtx); 2120 return (true); 2121 } 2122 2123 d = seq - key->rx_seq - 1; 2124 2125 if (key->rx_window & ((uint64_t)1 << d)) { 2126 /* Dupe! */ 2127 mtx_unlock(&key->replay_mtx); 2128 return (false); 2129 } 2130 2131 key->rx_window |= (uint64_t)1 << d; 2132 2133 while (key->rx_window & 1) { 2134 key->rx_seq++; 2135 key->rx_window >>= 1; 2136 } 2137 2138 mtx_unlock(&key->replay_mtx); 2139 2140 return (true); 2141 } 2142 2143 static struct ovpn_kpeer * 2144 ovpn_peer_from_mbuf(struct ovpn_softc *sc, struct mbuf *m, int off) 2145 { 2146 struct ovpn_wire_header ohdr; 2147 uint32_t peerid; 2148 const size_t hdrlen = sizeof(ohdr) - sizeof(ohdr.auth_tag); 2149 2150 OVPN_RASSERT(sc); 2151 2152 if (m_length(m, NULL) < (off + sizeof(struct udphdr) + hdrlen)) 2153 return (NULL); 2154 2155 m_copydata(m, off + sizeof(struct udphdr), hdrlen, (caddr_t)&ohdr); 2156 2157 peerid = ntohl(ohdr.opcode) & 0x00ffffff; 2158 2159 return (ovpn_find_peer(sc, peerid)); 2160 } 2161 2162 static bool 2163 ovpn_udp_input(struct mbuf *m, int off, struct inpcb *inp, 2164 const struct sockaddr *sa, void *ctx) 2165 { 2166 struct ovpn_softc *sc = ctx; 2167 struct ovpn_wire_header tmphdr; 2168 struct ovpn_wire_header *ohdr; 2169 struct udphdr *uhdr; 2170 struct ovpn_kkey *key; 2171 struct cryptop *crp; 2172 struct ovpn_kpeer *peer; 2173 size_t ohdrlen; 2174 int ret; 2175 uint8_t op; 2176 2177 OVPN_RLOCK_TRACKER; 2178 2179 M_ASSERTPKTHDR(m); 2180 2181 OVPN_COUNTER_ADD(sc, transport_bytes_received, m->m_pkthdr.len - off); 2182 2183 ohdrlen = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2184 2185 OVPN_RLOCK(sc); 2186 2187 peer = ovpn_peer_from_mbuf(sc, m, off); 2188 if (peer == NULL) { 2189 OVPN_RUNLOCK(sc); 2190 return (false); 2191 } 2192 2193 if (m_length(m, NULL) < (off + sizeof(*uhdr) + ohdrlen)) { 2194 /* Short packet. */ 2195 OVPN_RUNLOCK(sc); 2196 return (false); 2197 } 2198 2199 m_copydata(m, off + sizeof(*uhdr), ohdrlen, (caddr_t)&tmphdr); 2200 2201 op = ntohl(tmphdr.opcode) >> 24 >> OVPN_OP_SHIFT; 2202 if (op != OVPN_OP_DATA_V2) { 2203 /* Control packet? */ 2204 OVPN_RUNLOCK(sc); 2205 return (false); 2206 } 2207 2208 m = m_pullup(m, off + sizeof(*uhdr) + ohdrlen); 2209 if (m == NULL) { 2210 OVPN_RUNLOCK(sc); 2211 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2212 return (true); 2213 } 2214 2215 /* 2216 * Simplify things by getting rid of the preceding headers, we don't 2217 * care about them. 2218 */ 2219 m_adj_decap(m, off); 2220 2221 uhdr = mtodo(m, 0); 2222 ohdr = mtodo(m, sizeof(*uhdr)); 2223 2224 key = ovpn_find_key(sc, peer, ohdr); 2225 if (key == NULL || key->decrypt == NULL) { 2226 OVPN_RUNLOCK(sc); 2227 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2228 m_freem(m); 2229 return (true); 2230 } 2231 2232 if (key->decrypt->cipher == OVPN_CIPHER_ALG_NONE) { 2233 /* Now remove the outer headers */ 2234 m_adj_decap(m, sizeof(struct udphdr) + ohdrlen); 2235 2236 ohdr = mtodo(m, sizeof(*uhdr)); 2237 2238 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), 2239 _ovpn_lock_trackerp); 2240 OVPN_UNLOCK_ASSERT(sc); 2241 return (true); 2242 } 2243 2244 ohdrlen += sizeof(ohdr->auth_tag); 2245 2246 m = m_pullup(m, sizeof(*uhdr) + ohdrlen); 2247 if (m == NULL) { 2248 OVPN_RUNLOCK(sc); 2249 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2250 return (true); 2251 } 2252 uhdr = mtodo(m, 0); 2253 ohdr = mtodo(m, sizeof(*uhdr)); 2254 2255 /* Decrypt */ 2256 crp = crypto_getreq(key->decrypt->cryptoid, M_NOWAIT); 2257 if (crp == NULL) { 2258 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2259 OVPN_RUNLOCK(sc); 2260 m_freem(m); 2261 return (true); 2262 } 2263 2264 crp->crp_payload_start = sizeof(struct udphdr) + sizeof(*ohdr); 2265 crp->crp_payload_length = ntohs(uhdr->uh_ulen) - 2266 sizeof(*uhdr) - sizeof(*ohdr); 2267 crp->crp_op = CRYPTO_OP_DECRYPT; 2268 2269 /* AAD validation. */ 2270 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2271 crp->crp_aad = ohdr; 2272 crp->crp_aad_start = 0; 2273 crp->crp_op |= CRYPTO_OP_VERIFY_DIGEST; 2274 crp->crp_digest_start = sizeof(struct udphdr) + 2275 offsetof(struct ovpn_wire_header, auth_tag); 2276 2277 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 2278 memcpy(crp->crp_iv, &ohdr->seq, sizeof(ohdr->seq)); 2279 memcpy(crp->crp_iv + sizeof(ohdr->seq), key->decrypt->nonce, 2280 key->decrypt->noncelen); 2281 2282 crypto_use_mbuf(crp, m); 2283 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 2284 crp->crp_callback = ovpn_decrypt_rx_cb; 2285 crp->crp_opaque = sc; 2286 2287 atomic_add_int(&sc->refcount, 1); 2288 OVPN_RUNLOCK(sc); 2289 if (V_async_crypto) 2290 ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); 2291 else 2292 ret = crypto_dispatch(crp); 2293 if (ret != 0) { 2294 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2295 } 2296 2297 return (true); 2298 } 2299 2300 static void 2301 ovpn_qflush(struct ifnet *ifp __unused) 2302 { 2303 2304 } 2305 2306 static void 2307 ovpn_flush_rxring(struct ovpn_softc *sc) 2308 { 2309 struct ovpn_notification *n; 2310 2311 OVPN_WASSERT(sc); 2312 2313 while (! buf_ring_empty(sc->notifring)) { 2314 n = buf_ring_dequeue_sc(sc->notifring); 2315 free(n, M_OVPN); 2316 } 2317 } 2318 2319 #ifdef VIMAGE 2320 static void 2321 ovpn_reassign(struct ifnet *ifp, struct vnet *new_vnet __unused, 2322 char *unused __unused) 2323 { 2324 struct ovpn_softc *sc = ifp->if_softc; 2325 struct ovpn_kpeer *peer, *tmppeer; 2326 int ret __diagused; 2327 2328 OVPN_WLOCK(sc); 2329 2330 /* Flush keys & configuration. */ 2331 RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) { 2332 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 2333 ret = _ovpn_del_peer(sc, peer); 2334 MPASS(ret == 0); 2335 } 2336 2337 ovpn_flush_rxring(sc); 2338 2339 OVPN_WUNLOCK(sc); 2340 } 2341 #endif 2342 2343 static int 2344 ovpn_clone_match(struct if_clone *ifc, const char *name) 2345 { 2346 /* 2347 * Allow all names that start with 'ovpn', specifically because pfSense 2348 * uses ovpnc1 / ovpns2 2349 */ 2350 return (strncmp(ovpnname, name, strlen(ovpnname)) == 0); 2351 } 2352 2353 static int 2354 ovpn_clone_create(struct if_clone *ifc, char *name, size_t len, 2355 struct ifc_data *ifd, struct ifnet **ifpp) 2356 { 2357 struct ovpn_softc *sc; 2358 struct ifnet *ifp; 2359 char *dp; 2360 int error, unit, wildcard; 2361 2362 /* Try to see if a special unit was requested. */ 2363 error = ifc_name2unit(name, &unit); 2364 if (error != 0) 2365 return (error); 2366 wildcard = (unit < 0); 2367 2368 error = ifc_alloc_unit(ifc, &unit); 2369 if (error != 0) 2370 return (error); 2371 2372 /* 2373 * If no unit had been given, we need to adjust the ifName. 2374 */ 2375 for (dp = name; *dp != '\0'; dp++); 2376 if (wildcard) { 2377 error = snprintf(dp, len - (dp - name), "%d", unit); 2378 if (error > len - (dp - name)) { 2379 /* ifName too long. */ 2380 ifc_free_unit(ifc, unit); 2381 return (ENOSPC); 2382 } 2383 dp += error; 2384 } 2385 2386 /* Make sure it doesn't already exist. */ 2387 if (ifunit(name) != NULL) 2388 return (EEXIST); 2389 2390 sc = malloc(sizeof(struct ovpn_softc), M_OVPN, M_WAITOK | M_ZERO); 2391 sc->ifp = if_alloc(IFT_ENC); 2392 rm_init_flags(&sc->lock, "if_ovpn_lock", RM_RECURSE); 2393 sc->refcount = 0; 2394 2395 sc->notifring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL); 2396 2397 COUNTER_ARRAY_ALLOC(sc->counters, OVPN_COUNTER_SIZE, M_WAITOK); 2398 2399 ifp = sc->ifp; 2400 ifp->if_softc = sc; 2401 strlcpy(ifp->if_xname, name, IFNAMSIZ); 2402 ifp->if_dname = ovpngroupname; 2403 ifp->if_dunit = unit; 2404 2405 ifp->if_addrlen = 0; 2406 ifp->if_mtu = 1428; 2407 ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST; 2408 ifp->if_ioctl = ovpn_ioctl; 2409 ifp->if_transmit = ovpn_transmit; 2410 ifp->if_output = ovpn_output; 2411 ifp->if_qflush = ovpn_qflush; 2412 #ifdef VIMAGE 2413 ifp->if_reassign = ovpn_reassign; 2414 #endif 2415 ifp->if_capabilities |= IFCAP_LINKSTATE; 2416 ifp->if_capenable |= IFCAP_LINKSTATE; 2417 2418 if_attach(ifp); 2419 bpfattach(ifp, DLT_NULL, sizeof(uint32_t)); 2420 *ifpp = ifp; 2421 2422 return (0); 2423 } 2424 2425 static void 2426 ovpn_clone_destroy_cb(struct epoch_context *ctx) 2427 { 2428 struct ovpn_softc *sc; 2429 2430 sc = __containerof(ctx, struct ovpn_softc, epoch_ctx); 2431 2432 MPASS(sc->peercount == 0); 2433 MPASS(RB_EMPTY(&sc->peers)); 2434 2435 COUNTER_ARRAY_FREE(sc->counters, OVPN_COUNTER_SIZE); 2436 2437 if_free(sc->ifp); 2438 free(sc, M_OVPN); 2439 } 2440 2441 static int 2442 ovpn_clone_destroy(struct if_clone *ifc, struct ifnet *ifp, uint32_t flags) 2443 { 2444 struct ovpn_softc *sc; 2445 struct ovpn_kpeer *peer, *tmppeer; 2446 int unit; 2447 int ret __diagused; 2448 2449 sc = ifp->if_softc; 2450 unit = ifp->if_dunit; 2451 2452 OVPN_WLOCK(sc); 2453 2454 if (atomic_load_int(&sc->refcount) > 0) { 2455 OVPN_WUNLOCK(sc); 2456 return (EBUSY); 2457 } 2458 2459 RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) { 2460 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 2461 ret = _ovpn_del_peer(sc, peer); 2462 MPASS(ret == 0); 2463 } 2464 2465 ovpn_flush_rxring(sc); 2466 buf_ring_free(sc->notifring, M_OVPN); 2467 2468 OVPN_WUNLOCK(sc); 2469 2470 bpfdetach(ifp); 2471 if_detach(ifp); 2472 ifp->if_softc = NULL; 2473 2474 NET_EPOCH_CALL(ovpn_clone_destroy_cb, &sc->epoch_ctx); 2475 2476 if (unit != IF_DUNIT_NONE) 2477 ifc_free_unit(ifc, unit); 2478 2479 NET_EPOCH_DRAIN_CALLBACKS(); 2480 2481 return (0); 2482 } 2483 2484 static void 2485 vnet_ovpn_init(const void *unused __unused) 2486 { 2487 struct if_clone_addreq req = { 2488 .match_f = ovpn_clone_match, 2489 .create_f = ovpn_clone_create, 2490 .destroy_f = ovpn_clone_destroy, 2491 }; 2492 V_ovpn_cloner = ifc_attach_cloner(ovpngroupname, &req); 2493 } 2494 VNET_SYSINIT(vnet_ovpn_init, SI_SUB_PSEUDO, SI_ORDER_ANY, 2495 vnet_ovpn_init, NULL); 2496 2497 static void 2498 vnet_ovpn_uninit(const void *unused __unused) 2499 { 2500 if_clone_detach(V_ovpn_cloner); 2501 } 2502 VNET_SYSUNINIT(vnet_ovpn_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY, 2503 vnet_ovpn_uninit, NULL); 2504 2505 static int 2506 ovpnmodevent(module_t mod, int type, void *data) 2507 { 2508 switch (type) { 2509 case MOD_LOAD: 2510 /* Done in vnet_ovpn_init() */ 2511 break; 2512 case MOD_UNLOAD: 2513 /* Done in vnet_ovpn_uninit() */ 2514 break; 2515 default: 2516 return (EOPNOTSUPP); 2517 } 2518 2519 return (0); 2520 } 2521 2522 static moduledata_t ovpn_mod = { 2523 "if_ovpn", 2524 ovpnmodevent, 2525 0 2526 }; 2527 2528 DECLARE_MODULE(if_ovpn, ovpn_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 2529 MODULE_VERSION(if_ovpn, 1); 2530