1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2021-2022 Rubicon Communications, LLC (Netgate) 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 */ 28 #include "opt_inet.h" 29 #include "opt_inet6.h" 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/buf_ring.h> 34 #include <sys/epoch.h> 35 #include <sys/file.h> 36 #include <sys/filedesc.h> 37 #include <sys/kernel.h> 38 #include <sys/malloc.h> 39 #include <sys/mbuf.h> 40 #include <sys/module.h> 41 #include <sys/nv.h> 42 #include <sys/priv.h> 43 #include <sys/protosw.h> 44 #include <sys/rmlock.h> 45 #include <sys/sdt.h> 46 #include <sys/smp.h> 47 #include <sys/socket.h> 48 #include <sys/socketvar.h> 49 #include <sys/sockio.h> 50 #include <sys/sysctl.h> 51 #include <sys/time.h> 52 53 #include <machine/atomic.h> 54 55 #include <net/bpf.h> 56 #include <net/if.h> 57 #include <net/if_clone.h> 58 #include <net/if_types.h> 59 #include <net/if_var.h> 60 #include <net/if_private.h> 61 #include <net/netisr.h> 62 #include <net/route/nhop.h> 63 64 #include <netinet/in.h> 65 #include <netinet/in_fib.h> 66 #include <netinet/ip.h> 67 #include <netinet/ip6.h> 68 #include <netinet/ip_var.h> 69 #include <netinet/udp.h> 70 #include <netinet/udp_var.h> 71 72 #include <netinet6/ip6_var.h> 73 #include <netinet6/in6_fib.h> 74 75 #include <machine/in_cksum.h> 76 77 #include <opencrypto/cryptodev.h> 78 79 #include "if_ovpn.h" 80 81 struct ovpn_kkey_dir { 82 int refcount; 83 uint8_t key[32]; 84 uint8_t keylen; 85 uint8_t nonce[8]; 86 uint8_t noncelen; 87 enum ovpn_key_cipher cipher; 88 crypto_session_t cryptoid; 89 90 struct mtx replay_mtx; 91 /* 92 * Last seen gapless sequence number. New rx seq numbers must be 93 * strictly higher than this. 94 */ 95 uint32_t rx_seq; 96 /* Seen packets, relative to rx_seq. bit(0) will always be 0. */ 97 uint64_t rx_window; 98 }; 99 100 struct ovpn_kkey { 101 struct ovpn_kkey_dir *encrypt; 102 struct ovpn_kkey_dir *decrypt; 103 uint8_t keyid; 104 uint32_t peerid; 105 }; 106 107 struct ovpn_keepalive { 108 uint32_t interval; 109 uint32_t timeout; 110 }; 111 112 struct ovpn_wire_header { 113 uint32_t opcode; /* opcode, key id, peer id */ 114 uint32_t seq; 115 uint8_t auth_tag[16]; 116 }; 117 118 struct ovpn_peer_counters { 119 uint64_t pkt_in; 120 uint64_t pkt_out; 121 uint64_t bytes_in; 122 uint64_t bytes_out; 123 }; 124 #define OVPN_PEER_COUNTER_SIZE (sizeof(struct ovpn_peer_counters)/sizeof(uint64_t)) 125 126 struct ovpn_notification { 127 enum ovpn_notif_type type; 128 uint32_t peerid; 129 130 /* Delete notification */ 131 enum ovpn_del_reason del_reason; 132 struct ovpn_peer_counters counters; 133 }; 134 135 struct ovpn_softc; 136 137 struct ovpn_kpeer { 138 RB_ENTRY(ovpn_kpeer) tree; 139 int refcount; 140 uint32_t peerid; 141 142 struct ovpn_softc *sc; 143 struct sockaddr_storage local; 144 struct sockaddr_storage remote; 145 146 struct in_addr vpn4; 147 struct in6_addr vpn6; 148 149 struct ovpn_kkey keys[2]; 150 uint32_t tx_seq; 151 152 enum ovpn_del_reason del_reason; 153 struct ovpn_keepalive keepalive; 154 uint32_t *last_active; 155 struct callout ping_send; 156 struct callout ping_rcv; 157 158 counter_u64_t counters[OVPN_PEER_COUNTER_SIZE]; 159 }; 160 161 struct ovpn_counters { 162 uint64_t lost_ctrl_pkts_in; 163 uint64_t lost_ctrl_pkts_out; 164 uint64_t lost_data_pkts_in; 165 uint64_t lost_data_pkts_out; 166 uint64_t nomem_data_pkts_in; 167 uint64_t nomem_data_pkts_out; 168 uint64_t received_ctrl_pkts; 169 uint64_t received_data_pkts; 170 uint64_t sent_ctrl_pkts; 171 uint64_t sent_data_pkts; 172 173 uint64_t transport_bytes_sent; 174 uint64_t transport_bytes_received; 175 uint64_t tunnel_bytes_sent; 176 uint64_t tunnel_bytes_received; 177 }; 178 #define OVPN_COUNTER_SIZE (sizeof(struct ovpn_counters)/sizeof(uint64_t)) 179 180 RB_HEAD(ovpn_kpeers, ovpn_kpeer); 181 182 struct ovpn_softc { 183 int refcount; 184 struct rmlock lock; 185 struct ifnet *ifp; 186 struct socket *so; 187 int peercount; 188 struct ovpn_kpeers peers; 189 190 /* Pending notification */ 191 struct buf_ring *notifring; 192 193 counter_u64_t counters[OVPN_COUNTER_SIZE]; 194 195 struct epoch_context epoch_ctx; 196 }; 197 198 static struct ovpn_kpeer *ovpn_find_peer(struct ovpn_softc *, uint32_t); 199 static bool ovpn_udp_input(struct mbuf *, int, struct inpcb *, 200 const struct sockaddr *, void *); 201 static int ovpn_transmit_to_peer(struct ifnet *, struct mbuf *, 202 struct ovpn_kpeer *, struct rm_priotracker *); 203 static int ovpn_encap(struct ovpn_softc *, uint32_t, struct mbuf *); 204 static int ovpn_get_af(struct mbuf *); 205 static void ovpn_free_kkey_dir(struct ovpn_kkey_dir *); 206 static bool ovpn_check_replay(struct ovpn_kkey_dir *, uint32_t); 207 static int ovpn_peer_compare(struct ovpn_kpeer *, struct ovpn_kpeer *); 208 209 static RB_PROTOTYPE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare); 210 static RB_GENERATE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare); 211 212 #define OVPN_MTU_MIN 576 213 #define OVPN_MTU_MAX (IP_MAXPACKET - sizeof(struct ip) - \ 214 sizeof(struct udphdr) - sizeof(struct ovpn_wire_header)) 215 216 #define OVPN_OP_DATA_V2 0x09 217 #define OVPN_OP_SHIFT 3 218 219 VNET_DEFINE_STATIC(struct if_clone *, ovpn_cloner); 220 #define V_ovpn_cloner VNET(ovpn_cloner) 221 222 #define OVPN_RLOCK_TRACKER struct rm_priotracker _ovpn_lock_tracker; \ 223 struct rm_priotracker *_ovpn_lock_trackerp = &_ovpn_lock_tracker 224 #define OVPN_RLOCK(sc) rm_rlock(&(sc)->lock, _ovpn_lock_trackerp) 225 #define OVPN_RUNLOCK(sc) rm_runlock(&(sc)->lock, _ovpn_lock_trackerp) 226 #define OVPN_WLOCK(sc) rm_wlock(&(sc)->lock) 227 #define OVPN_WUNLOCK(sc) rm_wunlock(&(sc)->lock) 228 #define OVPN_ASSERT(sc) rm_assert(&(sc)->lock, RA_LOCKED) 229 #define OVPN_RASSERT(sc) rm_assert(&(sc)->lock, RA_RLOCKED) 230 #define OVPN_WASSERT(sc) rm_assert(&(sc)->lock, RA_WLOCKED) 231 #define OVPN_UNLOCK_ASSERT(sc) rm_assert(&(sc)->lock, RA_UNLOCKED) 232 233 #define OVPN_COUNTER(sc, name) \ 234 ((sc)->counters[offsetof(struct ovpn_counters, name)/sizeof(uint64_t)]) 235 #define OVPN_PEER_COUNTER(peer, name) \ 236 ((peer)->counters[offsetof(struct ovpn_peer_counters, name) / \ 237 sizeof(uint64_t)]) 238 239 #define OVPN_COUNTER_ADD(sc, name, val) \ 240 counter_u64_add(OVPN_COUNTER(sc, name), val) 241 #define OVPN_PEER_COUNTER_ADD(p, name, val) \ 242 counter_u64_add(OVPN_PEER_COUNTER(p, name), val) 243 244 #define TO_IN(x) ((struct sockaddr_in *)(x)) 245 #define TO_IN6(x) ((struct sockaddr_in6 *)(x)) 246 247 SDT_PROVIDER_DEFINE(if_ovpn); 248 SDT_PROBE_DEFINE1(if_ovpn, tx, transmit, start, "struct mbuf *"); 249 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip4, "struct in_addr *", "struct ovpn_kpeer *"); 250 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip6, "struct in6_addr *", "struct ovpn_kpeer *"); 251 252 static const char ovpnname[] = "ovpn"; 253 static const char ovpngroupname[] = "openvpn"; 254 255 static MALLOC_DEFINE(M_OVPN, ovpnname, "OpenVPN DCO Interface"); 256 257 SYSCTL_DECL(_net_link); 258 static SYSCTL_NODE(_net_link, IFT_OTHER, openvpn, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 259 "OpenVPN DCO Interface"); 260 VNET_DEFINE_STATIC(int, replay_protection) = 0; 261 #define V_replay_protection VNET(replay_protection) 262 SYSCTL_INT(_net_link_openvpn, OID_AUTO, replay_protection, CTLFLAG_VNET | CTLFLAG_RW, 263 &VNET_NAME(replay_protection), 0, "Validate sequence numbers"); 264 265 VNET_DEFINE_STATIC(int, async_crypto); 266 #define V_async_crypto VNET(async_crypto) 267 SYSCTL_INT(_net_link_openvpn, OID_AUTO, async_crypto, 268 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_crypto), 0, 269 "Use asynchronous mode to parallelize crypto jobs."); 270 271 VNET_DEFINE_STATIC(int, async_netisr_queue); 272 #define V_async_netisr_queue VNET(async_netisr_queue) 273 SYSCTL_INT(_net_link_openvpn, OID_AUTO, netisr_queue, 274 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_netisr_queue), 0, 275 "Use netisr_queue() rather than netisr_dispatch()."); 276 277 static int 278 ovpn_peer_compare(struct ovpn_kpeer *a, struct ovpn_kpeer *b) 279 { 280 return (a->peerid - b->peerid); 281 } 282 283 static struct ovpn_kpeer * 284 ovpn_find_peer(struct ovpn_softc *sc, uint32_t peerid) 285 { 286 struct ovpn_kpeer p; 287 288 OVPN_ASSERT(sc); 289 290 p.peerid = peerid; 291 292 return (RB_FIND(ovpn_kpeers, &sc->peers, &p)); 293 } 294 295 static struct ovpn_kpeer * 296 ovpn_find_only_peer(struct ovpn_softc *sc) 297 { 298 OVPN_ASSERT(sc); 299 300 return (RB_ROOT(&sc->peers)); 301 } 302 303 static uint16_t 304 ovpn_get_port(struct sockaddr_storage *s) 305 { 306 switch (s->ss_family) { 307 case AF_INET: { 308 struct sockaddr_in *in = (struct sockaddr_in *)s; 309 return (in->sin_port); 310 } 311 case AF_INET6: { 312 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)s; 313 return (in6->sin6_port); 314 } 315 default: 316 panic("Unsupported address family %d", s->ss_family); 317 } 318 } 319 320 static int 321 ovpn_nvlist_to_sockaddr(const nvlist_t *nvl, struct sockaddr_storage *sa) 322 { 323 int af; 324 325 if (! nvlist_exists_number(nvl, "af")) 326 return (EINVAL); 327 if (! nvlist_exists_binary(nvl, "address")) 328 return (EINVAL); 329 if (! nvlist_exists_number(nvl, "port")) 330 return (EINVAL); 331 332 af = nvlist_get_number(nvl, "af"); 333 334 switch (af) { 335 #ifdef INET 336 case AF_INET: { 337 struct sockaddr_in *in = (struct sockaddr_in *)sa; 338 size_t len; 339 const void *addr = nvlist_get_binary(nvl, "address", &len); 340 in->sin_family = af; 341 if (len != sizeof(in->sin_addr)) 342 return (EINVAL); 343 344 memcpy(&in->sin_addr, addr, sizeof(in->sin_addr)); 345 in->sin_port = nvlist_get_number(nvl, "port"); 346 break; 347 } 348 #endif 349 #ifdef INET6 350 case AF_INET6: { 351 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)sa; 352 size_t len; 353 const void *addr = nvlist_get_binary(nvl, "address", &len); 354 in6->sin6_family = af; 355 if (len != sizeof(in6->sin6_addr)) 356 return (EINVAL); 357 358 memcpy(&in6->sin6_addr, addr, sizeof(in6->sin6_addr)); 359 in6->sin6_port = nvlist_get_number(nvl, "port"); 360 break; 361 } 362 #endif 363 default: 364 return (EINVAL); 365 } 366 367 return (0); 368 } 369 370 static bool 371 ovpn_has_peers(struct ovpn_softc *sc) 372 { 373 OVPN_ASSERT(sc); 374 375 return (sc->peercount > 0); 376 } 377 378 static void 379 ovpn_rele_so(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 380 { 381 bool has_peers; 382 383 OVPN_WASSERT(sc); 384 385 if (sc->so == NULL) 386 return; 387 388 has_peers = ovpn_has_peers(sc); 389 390 /* Only remove the tunnel function if we're releasing the socket for 391 * the last peer. */ 392 if (! has_peers) 393 (void)udp_set_kernel_tunneling(sc->so, NULL, NULL, NULL); 394 395 sorele(sc->so); 396 397 if (! has_peers) 398 sc->so = NULL; 399 } 400 401 static void 402 ovpn_notify_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 403 { 404 struct ovpn_notification *n; 405 406 OVPN_WASSERT(sc); 407 408 n = malloc(sizeof(*n), M_OVPN, M_NOWAIT); 409 if (n == NULL) 410 return; 411 412 n->peerid = peer->peerid; 413 n->type = OVPN_NOTIF_DEL_PEER; 414 n->del_reason = peer->del_reason; 415 416 n->counters.pkt_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_in)); 417 n->counters.pkt_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_out)); 418 n->counters.bytes_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_in)); 419 n->counters.bytes_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_out)); 420 421 if (buf_ring_enqueue(sc->notifring, n) != 0) { 422 free(n, M_OVPN); 423 } else if (sc->so != NULL) { 424 /* Wake up userspace */ 425 sc->so->so_error = EAGAIN; 426 sorwakeup(sc->so); 427 sowwakeup(sc->so); 428 } 429 } 430 431 static void 432 ovpn_peer_release_ref(struct ovpn_kpeer *peer, bool locked) 433 { 434 struct ovpn_softc *sc; 435 436 CURVNET_ASSERT_SET(); 437 438 atomic_add_int(&peer->refcount, -1); 439 440 if (atomic_load_int(&peer->refcount) > 0) 441 return; 442 443 sc = peer->sc; 444 445 if (! locked) { 446 OVPN_WLOCK(sc); 447 448 /* Might have changed before we acquired the lock. */ 449 if (atomic_load_int(&peer->refcount) > 0) { 450 OVPN_WUNLOCK(sc); 451 return; 452 } 453 } 454 455 OVPN_ASSERT(sc); 456 457 /* The peer should have been removed from the list already. */ 458 MPASS(ovpn_find_peer(sc, peer->peerid) == NULL); 459 460 ovpn_notify_del_peer(sc, peer); 461 462 for (int i = 0; i < 2; i++) { 463 ovpn_free_kkey_dir(peer->keys[i].encrypt); 464 ovpn_free_kkey_dir(peer->keys[i].decrypt); 465 } 466 467 ovpn_rele_so(sc, peer); 468 469 callout_stop(&peer->ping_send); 470 callout_stop(&peer->ping_rcv); 471 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 472 free(peer, M_OVPN); 473 474 if (! locked) 475 OVPN_WUNLOCK(sc); 476 } 477 478 static int 479 ovpn_new_peer(struct ifnet *ifp, const nvlist_t *nvl) 480 { 481 #ifdef INET6 482 struct epoch_tracker et; 483 #endif 484 struct sockaddr_storage remote; 485 struct ovpn_kpeer *peer = NULL; 486 struct file *fp = NULL; 487 struct sockaddr *name = NULL; 488 struct ovpn_softc *sc = ifp->if_softc; 489 struct thread *td = curthread; 490 struct socket *so = NULL; 491 int fd; 492 uint32_t peerid; 493 int ret = 0; 494 495 if (nvl == NULL) 496 return (EINVAL); 497 498 if (! nvlist_exists_number(nvl, "peerid")) 499 return (EINVAL); 500 501 if (! nvlist_exists_number(nvl, "fd")) 502 return (EINVAL); 503 504 if (! nvlist_exists_nvlist(nvl, "remote")) 505 return (EINVAL); 506 507 peerid = nvlist_get_number(nvl, "peerid"); 508 509 ret = ovpn_nvlist_to_sockaddr(nvlist_get_nvlist(nvl, "remote"), 510 &remote); 511 if (ret != 0) 512 return (ret); 513 514 fd = nvlist_get_number(nvl, "fd"); 515 516 /* Look up the userspace process and use the fd to find the socket. */ 517 ret = getsock(td, fd, &cap_connect_rights, &fp); 518 if (ret != 0) 519 return (ret); 520 521 so = fp->f_data; 522 523 peer = malloc(sizeof(*peer), M_OVPN, M_WAITOK | M_ZERO); 524 peer->peerid = peerid; 525 peer->sc = sc; 526 peer->tx_seq = 1; 527 peer->refcount = 1; 528 peer->last_active = uma_zalloc_pcpu(pcpu_zone_4, M_WAITOK | M_ZERO); 529 COUNTER_ARRAY_ALLOC(peer->counters, OVPN_PEER_COUNTER_SIZE, M_WAITOK); 530 531 if (nvlist_exists_binary(nvl, "vpn_ipv4")) { 532 size_t len; 533 const void *addr = nvlist_get_binary(nvl, "vpn_ipv4", &len); 534 if (len != sizeof(peer->vpn4)) { 535 ret = EINVAL; 536 goto error; 537 } 538 memcpy(&peer->vpn4, addr, len); 539 } 540 541 if (nvlist_exists_binary(nvl, "vpn_ipv6")) { 542 size_t len; 543 const void *addr = nvlist_get_binary(nvl, "vpn_ipv6", &len); 544 if (len != sizeof(peer->vpn6)) { 545 ret = EINVAL; 546 goto error; 547 } 548 memcpy(&peer->vpn6, addr, len); 549 } 550 551 callout_init_rm(&peer->ping_send, &sc->lock, CALLOUT_SHAREDLOCK); 552 callout_init_rm(&peer->ping_rcv, &sc->lock, 0); 553 554 ret = so->so_proto->pr_sockaddr(so, &name); 555 if (ret) 556 goto error; 557 558 if (ovpn_get_port((struct sockaddr_storage *)name) == 0) { 559 ret = EINVAL; 560 goto error; 561 } 562 if (name->sa_family != remote.ss_family) { 563 ret = EINVAL; 564 goto error; 565 } 566 567 memcpy(&peer->local, name, name->sa_len); 568 memcpy(&peer->remote, &remote, sizeof(remote)); 569 free(name, M_SONAME); 570 name = NULL; 571 572 if (peer->local.ss_family == AF_INET6 && 573 IN6_IS_ADDR_V4MAPPED(&TO_IN6(&peer->remote)->sin6_addr)) { 574 /* V4 mapped address, so treat this as v4, not v6. */ 575 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->local); 576 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->remote); 577 } 578 579 #ifdef INET6 580 if (peer->local.ss_family == AF_INET6 && 581 IN6_IS_ADDR_UNSPECIFIED(&TO_IN6(&peer->local)->sin6_addr)) { 582 NET_EPOCH_ENTER(et); 583 ret = in6_selectsrc_addr(curthread->td_proc->p_fibnum, 584 &TO_IN6(&peer->remote)->sin6_addr, 585 0, NULL, &TO_IN6(&peer->local)->sin6_addr, NULL); 586 NET_EPOCH_EXIT(et); 587 if (ret != 0) { 588 goto error; 589 } 590 } 591 #endif 592 OVPN_WLOCK(sc); 593 594 /* Disallow peer id re-use. */ 595 if (ovpn_find_peer(sc, peerid) != NULL) { 596 ret = EEXIST; 597 goto error_locked; 598 } 599 600 /* Make sure this is really a UDP socket. */ 601 if (so->so_type != SOCK_DGRAM || so->so_proto->pr_type != SOCK_DGRAM) { 602 ret = EPROTOTYPE; 603 goto error_locked; 604 } 605 606 /* Must be the same socket as for other peers on this interface. */ 607 if (sc->so != NULL && so != sc->so) 608 goto error_locked; 609 610 if (sc->so == NULL) 611 sc->so = so; 612 613 /* Insert the peer into the list. */ 614 RB_INSERT(ovpn_kpeers, &sc->peers, peer); 615 sc->peercount++; 616 soref(sc->so); 617 618 ret = udp_set_kernel_tunneling(sc->so, ovpn_udp_input, NULL, sc); 619 if (ret == EBUSY) { 620 /* Fine, another peer already set the input function. */ 621 ret = 0; 622 } 623 if (ret != 0) { 624 RB_REMOVE(ovpn_kpeers, &sc->peers, peer); 625 sc->peercount--; 626 goto error_locked; 627 } 628 629 OVPN_WUNLOCK(sc); 630 631 goto done; 632 633 error_locked: 634 OVPN_WUNLOCK(sc); 635 error: 636 free(name, M_SONAME); 637 COUNTER_ARRAY_FREE(peer->counters, OVPN_PEER_COUNTER_SIZE); 638 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 639 free(peer, M_OVPN); 640 done: 641 if (fp != NULL) 642 fdrop(fp, td); 643 644 return (ret); 645 } 646 647 static int 648 _ovpn_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 649 { 650 struct ovpn_kpeer *tmp __diagused; 651 652 OVPN_WASSERT(sc); 653 CURVNET_ASSERT_SET(); 654 655 MPASS(RB_FIND(ovpn_kpeers, &sc->peers, peer) == peer); 656 657 tmp = RB_REMOVE(ovpn_kpeers, &sc->peers, peer); 658 MPASS(tmp != NULL); 659 660 sc->peercount--; 661 662 ovpn_peer_release_ref(peer, true); 663 664 return (0); 665 } 666 667 static int 668 ovpn_del_peer(struct ifnet *ifp, nvlist_t *nvl) 669 { 670 struct ovpn_softc *sc = ifp->if_softc; 671 struct ovpn_kpeer *peer; 672 uint32_t peerid; 673 int ret; 674 675 OVPN_WASSERT(sc); 676 677 if (nvl == NULL) 678 return (EINVAL); 679 680 if (! nvlist_exists_number(nvl, "peerid")) 681 return (EINVAL); 682 683 peerid = nvlist_get_number(nvl, "peerid"); 684 685 peer = ovpn_find_peer(sc, peerid); 686 if (peer == NULL) 687 return (ENOENT); 688 689 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 690 ret = _ovpn_del_peer(sc, peer); 691 692 return (ret); 693 } 694 695 static int 696 ovpn_create_kkey_dir(struct ovpn_kkey_dir **kdirp, 697 const nvlist_t *nvl) 698 { 699 struct crypto_session_params csp; 700 struct ovpn_kkey_dir *kdir; 701 const char *ciphername; 702 enum ovpn_key_cipher cipher; 703 const void *key, *iv; 704 size_t keylen = 0, ivlen = 0; 705 int error; 706 707 if (! nvlist_exists_string(nvl, "cipher")) 708 return (EINVAL); 709 ciphername = nvlist_get_string(nvl, "cipher"); 710 711 if (strcmp(ciphername, "none") == 0) 712 cipher = OVPN_CIPHER_ALG_NONE; 713 else if (strcmp(ciphername, "AES-256-GCM") == 0 || 714 strcmp(ciphername, "AES-192-GCM") == 0 || 715 strcmp(ciphername, "AES-128-GCM") == 0) 716 cipher = OVPN_CIPHER_ALG_AES_GCM; 717 else if (strcmp(ciphername, "CHACHA20-POLY1305") == 0) 718 cipher = OVPN_CIPHER_ALG_CHACHA20_POLY1305; 719 else 720 return (EINVAL); 721 722 if (cipher != OVPN_CIPHER_ALG_NONE) { 723 if (! nvlist_exists_binary(nvl, "key")) 724 return (EINVAL); 725 key = nvlist_get_binary(nvl, "key", &keylen); 726 if (keylen > sizeof(kdir->key)) 727 return (E2BIG); 728 729 if (! nvlist_exists_binary(nvl, "iv")) 730 return (EINVAL); 731 iv = nvlist_get_binary(nvl, "iv", &ivlen); 732 if (ivlen != 8) 733 return (E2BIG); 734 } 735 736 kdir = malloc(sizeof(struct ovpn_kkey_dir), M_OVPN, 737 M_WAITOK | M_ZERO); 738 739 kdir->cipher = cipher; 740 kdir->keylen = keylen; 741 memcpy(kdir->key, key, keylen); 742 kdir->noncelen = ivlen; 743 memcpy(kdir->nonce, iv, ivlen); 744 745 if (kdir->cipher != OVPN_CIPHER_ALG_NONE) { 746 /* Crypto init */ 747 bzero(&csp, sizeof(csp)); 748 csp.csp_mode = CSP_MODE_AEAD; 749 750 if (kdir->cipher == OVPN_CIPHER_ALG_CHACHA20_POLY1305) 751 csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305; 752 else 753 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 754 755 csp.csp_flags |= CSP_F_SEPARATE_AAD; 756 757 csp.csp_cipher_klen = kdir->keylen; 758 csp.csp_cipher_key = kdir->key; 759 csp.csp_ivlen = 96 / 8; 760 761 error = crypto_newsession(&kdir->cryptoid, &csp, 762 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE); 763 if (error) { 764 free(kdir, M_OVPN); 765 return (error); 766 } 767 } 768 769 mtx_init(&kdir->replay_mtx, "if_ovpn rx replay", NULL, MTX_DEF); 770 *kdirp = kdir; 771 772 return (0); 773 } 774 775 static void 776 ovpn_free_kkey_dir(struct ovpn_kkey_dir *kdir) 777 { 778 if (kdir == NULL) 779 return; 780 781 mtx_destroy(&kdir->replay_mtx); 782 783 crypto_freesession(kdir->cryptoid); 784 free(kdir, M_OVPN); 785 } 786 787 static int 788 ovpn_set_key(struct ifnet *ifp, const nvlist_t *nvl) 789 { 790 struct ovpn_softc *sc = ifp->if_softc; 791 struct ovpn_kkey_dir *enc, *dec; 792 struct ovpn_kpeer *peer; 793 int slot, keyid, peerid; 794 int error; 795 796 if (nvl == NULL) 797 return (EINVAL); 798 799 if (! nvlist_exists_number(nvl, "slot")) 800 return (EINVAL); 801 slot = nvlist_get_number(nvl, "slot"); 802 803 if (! nvlist_exists_number(nvl, "keyid")) 804 return (EINVAL); 805 keyid = nvlist_get_number(nvl, "keyid"); 806 807 if (! nvlist_exists_number(nvl, "peerid")) 808 return (EINVAL); 809 peerid = nvlist_get_number(nvl, "peerid"); 810 811 if (slot != OVPN_KEY_SLOT_PRIMARY && 812 slot != OVPN_KEY_SLOT_SECONDARY) 813 return (EINVAL); 814 815 if (! nvlist_exists_nvlist(nvl, "encrypt") || 816 ! nvlist_exists_nvlist(nvl, "decrypt")) 817 return (EINVAL); 818 819 error = ovpn_create_kkey_dir(&enc, nvlist_get_nvlist(nvl, "encrypt")); 820 if (error) 821 return (error); 822 823 error = ovpn_create_kkey_dir(&dec, nvlist_get_nvlist(nvl, "decrypt")); 824 if (error) { 825 ovpn_free_kkey_dir(enc); 826 return (error); 827 } 828 829 OVPN_WLOCK(sc); 830 831 peer = ovpn_find_peer(sc, peerid); 832 if (peer == NULL) { 833 ovpn_free_kkey_dir(dec); 834 ovpn_free_kkey_dir(enc); 835 OVPN_WUNLOCK(sc); 836 return (ENOENT); 837 } 838 839 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 840 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 841 842 peer->keys[slot].encrypt = enc; 843 peer->keys[slot].decrypt = dec; 844 845 peer->keys[slot].keyid = keyid; 846 peer->keys[slot].peerid = peerid; 847 848 OVPN_WUNLOCK(sc); 849 850 return (0); 851 } 852 853 static int 854 ovpn_check_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, enum ovpn_key_slot slot) 855 { 856 OVPN_ASSERT(sc); 857 858 if (peer->keys[slot].encrypt == NULL) 859 return (ENOLINK); 860 861 if (peer->keys[slot].decrypt == NULL) 862 return (ENOLINK); 863 864 return (0); 865 } 866 867 static int 868 ovpn_start(struct ifnet *ifp) 869 { 870 struct ovpn_softc *sc = ifp->if_softc; 871 872 OVPN_WLOCK(sc); 873 874 ifp->if_flags |= IFF_UP; 875 ifp->if_drv_flags |= IFF_DRV_RUNNING; 876 if_link_state_change(ifp, LINK_STATE_UP); 877 878 OVPN_WUNLOCK(sc); 879 880 return (0); 881 } 882 883 static int 884 ovpn_swap_keys(struct ifnet *ifp, nvlist_t *nvl) 885 { 886 struct ovpn_softc *sc = ifp->if_softc; 887 struct ovpn_kpeer *peer; 888 struct ovpn_kkey tmpkey; 889 int error; 890 891 if (nvl == NULL) 892 return (EINVAL); 893 894 if (! nvlist_exists_number(nvl, "peerid")) 895 return (EINVAL); 896 897 OVPN_WLOCK(sc); 898 899 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 900 if (peer == NULL) { 901 OVPN_WUNLOCK(sc); 902 return (ENOENT); 903 } 904 905 /* Check that we have a second key to swap to. */ 906 error = ovpn_check_key(sc, peer, OVPN_KEY_SLOT_SECONDARY); 907 if (error) { 908 OVPN_WUNLOCK(sc); 909 return (error); 910 } 911 912 tmpkey = peer->keys[0]; 913 peer->keys[0] = peer->keys[1]; 914 peer->keys[1] = tmpkey; 915 916 OVPN_WUNLOCK(sc); 917 918 return (0); 919 } 920 921 static int 922 ovpn_del_key(struct ifnet *ifp, const nvlist_t *nvl) 923 { 924 enum ovpn_key_slot slot; 925 struct ovpn_kpeer *peer; 926 struct ovpn_softc *sc = ifp->if_softc; 927 928 if (nvl == NULL) 929 return (EINVAL); 930 931 if (! nvlist_exists_number(nvl, "peerid")) 932 return (EINVAL); 933 934 if (! nvlist_exists_number(nvl, "slot")) 935 return (EINVAL); 936 slot = nvlist_get_number(nvl, "slot"); 937 938 if (slot != OVPN_KEY_SLOT_PRIMARY && 939 slot != OVPN_KEY_SLOT_SECONDARY) 940 return (EINVAL); 941 942 OVPN_WLOCK(sc); 943 944 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 945 if (peer == NULL) { 946 OVPN_WUNLOCK(sc); 947 return (ENOENT); 948 } 949 950 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 951 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 952 953 peer->keys[slot].encrypt = NULL; 954 peer->keys[slot].decrypt = NULL; 955 956 peer->keys[slot].keyid = 0; 957 peer->keys[slot].peerid = 0; 958 959 OVPN_WUNLOCK(sc); 960 961 return (0); 962 } 963 964 static void 965 ovpn_send_ping(void *arg) 966 { 967 static const uint8_t ping_str[] = { 968 0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb, 969 0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48 970 }; 971 972 struct epoch_tracker et; 973 struct ovpn_kpeer *peer = arg; 974 struct ovpn_softc *sc = peer->sc; 975 struct mbuf *m; 976 977 OVPN_RASSERT(sc); 978 979 /* Ensure we repeat! */ 980 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 981 ovpn_send_ping, peer); 982 983 m = m_get2(sizeof(ping_str), M_NOWAIT, MT_DATA, M_PKTHDR); 984 if (m == NULL) 985 return; 986 987 m_copyback(m, 0, sizeof(ping_str), ping_str); 988 m->m_len = m->m_pkthdr.len = sizeof(ping_str); 989 990 CURVNET_SET(sc->ifp->if_vnet); 991 NET_EPOCH_ENTER(et); 992 (void)ovpn_transmit_to_peer(sc->ifp, m, peer, NULL); 993 NET_EPOCH_EXIT(et); 994 CURVNET_RESTORE(); 995 } 996 997 static void 998 ovpn_timeout(void *arg) 999 { 1000 struct ovpn_kpeer *peer = arg; 1001 struct ovpn_softc *sc = peer->sc; 1002 uint32_t last, _last_active; 1003 int ret __diagused; 1004 int cpu; 1005 1006 OVPN_WASSERT(sc); 1007 1008 last = 0; 1009 CPU_FOREACH(cpu) { 1010 _last_active = *zpcpu_get_cpu(peer->last_active, cpu); 1011 if (_last_active > last) 1012 last = _last_active; 1013 } 1014 1015 if (last + peer->keepalive.timeout > time_uptime) { 1016 callout_reset(&peer->ping_rcv, 1017 (peer->keepalive.timeout - (time_uptime - last)) * hz, 1018 ovpn_timeout, peer); 1019 return; 1020 } 1021 1022 CURVNET_SET(sc->ifp->if_vnet); 1023 peer->del_reason = OVPN_DEL_REASON_TIMEOUT; 1024 ret = _ovpn_del_peer(sc, peer); 1025 MPASS(ret == 0); 1026 CURVNET_RESTORE(); 1027 } 1028 1029 static int 1030 ovpn_set_peer(struct ifnet *ifp, const nvlist_t *nvl) 1031 { 1032 struct ovpn_softc *sc = ifp->if_softc; 1033 struct ovpn_kpeer *peer; 1034 1035 if (nvl == NULL) 1036 return (EINVAL); 1037 1038 if (! nvlist_exists_number(nvl, "interval") || 1039 ! nvlist_exists_number(nvl, "timeout") || 1040 ! nvlist_exists_number(nvl, "peerid")) 1041 return (EINVAL); 1042 1043 OVPN_WLOCK(sc); 1044 1045 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 1046 if (peer == NULL) { 1047 OVPN_WUNLOCK(sc); 1048 return (ENOENT); 1049 } 1050 1051 peer->keepalive.interval = nvlist_get_number(nvl, "interval"); 1052 peer->keepalive.timeout = nvlist_get_number(nvl, "timeout"); 1053 1054 if (peer->keepalive.interval > 0) 1055 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 1056 ovpn_send_ping, peer); 1057 if (peer->keepalive.timeout > 0) 1058 callout_reset(&peer->ping_rcv, peer->keepalive.timeout * hz, 1059 ovpn_timeout, peer); 1060 1061 OVPN_WUNLOCK(sc); 1062 1063 return (0); 1064 } 1065 1066 static int 1067 ovpn_set_ifmode(struct ifnet *ifp, const nvlist_t *nvl) 1068 { 1069 struct ovpn_softc *sc = ifp->if_softc; 1070 int ifmode; 1071 1072 if (nvl == NULL) 1073 return (EINVAL); 1074 1075 if (! nvlist_exists_number(nvl, "ifmode") ) 1076 return (EINVAL); 1077 1078 ifmode = nvlist_get_number(nvl, "ifmode"); 1079 1080 OVPN_WLOCK(sc); 1081 1082 /* deny this if UP */ 1083 if (ifp->if_flags & IFF_UP) { 1084 OVPN_WUNLOCK(sc); 1085 return (EBUSY); 1086 } 1087 1088 switch (ifmode & ~IFF_MULTICAST) { 1089 case IFF_POINTOPOINT: 1090 case IFF_BROADCAST: 1091 ifp->if_flags &= 1092 ~(IFF_BROADCAST|IFF_POINTOPOINT|IFF_MULTICAST); 1093 ifp->if_flags |= ifmode; 1094 break; 1095 default: 1096 OVPN_WUNLOCK(sc); 1097 return (EINVAL); 1098 } 1099 1100 OVPN_WUNLOCK(sc); 1101 1102 return (0); 1103 } 1104 1105 static int 1106 ovpn_ioctl_set(struct ifnet *ifp, struct ifdrv *ifd) 1107 { 1108 struct ovpn_softc *sc = ifp->if_softc; 1109 uint8_t *buf = NULL; 1110 nvlist_t *nvl = NULL; 1111 int ret; 1112 1113 if (ifd->ifd_len != 0) { 1114 if (ifd->ifd_len > OVPN_MAX_REQUEST_SIZE) 1115 return (E2BIG); 1116 1117 buf = malloc(ifd->ifd_len, M_OVPN, M_WAITOK); 1118 1119 ret = copyin(ifd->ifd_data, buf, ifd->ifd_len); 1120 if (ret != 0) { 1121 free(buf, M_OVPN); 1122 return (ret); 1123 } 1124 1125 nvl = nvlist_unpack(buf, ifd->ifd_len, 0); 1126 free(buf, M_OVPN); 1127 if (nvl == NULL) { 1128 return (EINVAL); 1129 } 1130 } 1131 1132 switch (ifd->ifd_cmd) { 1133 case OVPN_NEW_PEER: 1134 ret = ovpn_new_peer(ifp, nvl); 1135 break; 1136 case OVPN_DEL_PEER: 1137 OVPN_WLOCK(sc); 1138 ret = ovpn_del_peer(ifp, nvl); 1139 OVPN_WUNLOCK(sc); 1140 break; 1141 case OVPN_NEW_KEY: 1142 ret = ovpn_set_key(ifp, nvl); 1143 break; 1144 case OVPN_START_VPN: 1145 ret = ovpn_start(ifp); 1146 break; 1147 case OVPN_SWAP_KEYS: 1148 ret = ovpn_swap_keys(ifp, nvl); 1149 break; 1150 case OVPN_DEL_KEY: 1151 ret = ovpn_del_key(ifp, nvl); 1152 break; 1153 case OVPN_SET_PEER: 1154 ret = ovpn_set_peer(ifp, nvl); 1155 break; 1156 case OVPN_SET_IFMODE: 1157 ret = ovpn_set_ifmode(ifp, nvl); 1158 break; 1159 default: 1160 ret = ENOTSUP; 1161 } 1162 1163 nvlist_destroy(nvl); 1164 return (ret); 1165 } 1166 1167 static int 1168 ovpn_add_counters(nvlist_t *parent, const char *name, counter_u64_t in, 1169 counter_u64_t out) 1170 { 1171 nvlist_t *nvl; 1172 1173 nvl = nvlist_create(0); 1174 if (nvl == NULL) 1175 return (ENOMEM); 1176 1177 nvlist_add_number(nvl, "in", counter_u64_fetch(in)); 1178 nvlist_add_number(nvl, "out", counter_u64_fetch(out)); 1179 1180 nvlist_add_nvlist(parent, name, nvl); 1181 1182 nvlist_destroy(nvl); 1183 1184 return (0); 1185 } 1186 1187 static int 1188 ovpn_get_stats(struct ovpn_softc *sc, nvlist_t **onvl) 1189 { 1190 nvlist_t *nvl; 1191 int ret; 1192 1193 nvl = nvlist_create(0); 1194 if (nvl == NULL) 1195 return (ENOMEM); 1196 1197 #define OVPN_COUNTER_OUT(name, in, out) \ 1198 do { \ 1199 ret = ovpn_add_counters(nvl, name, OVPN_COUNTER(sc, in), \ 1200 OVPN_COUNTER(sc, out)); \ 1201 if (ret != 0) \ 1202 goto error; \ 1203 } while(0) 1204 1205 OVPN_COUNTER_OUT("lost_ctrl", lost_ctrl_pkts_in, lost_ctrl_pkts_out); 1206 OVPN_COUNTER_OUT("lost_data", lost_data_pkts_in, lost_data_pkts_out); 1207 OVPN_COUNTER_OUT("nomem_data", nomem_data_pkts_in, 1208 nomem_data_pkts_out); 1209 OVPN_COUNTER_OUT("data", received_data_pkts, sent_data_pkts); 1210 OVPN_COUNTER_OUT("ctrl", received_ctrl_pkts, sent_ctrl_pkts); 1211 OVPN_COUNTER_OUT("tunnel", tunnel_bytes_received, 1212 tunnel_bytes_received); 1213 OVPN_COUNTER_OUT("transport", transport_bytes_received, 1214 transport_bytes_received); 1215 #undef OVPN_COUNTER_OUT 1216 1217 *onvl = nvl; 1218 1219 return (0); 1220 1221 error: 1222 nvlist_destroy(nvl); 1223 return (ret); 1224 } 1225 1226 static int 1227 ovpn_get_peer_stats(struct ovpn_softc *sc, nvlist_t **nvl) 1228 { 1229 struct ovpn_kpeer *peer; 1230 nvlist_t *nvpeer = NULL; 1231 int ret; 1232 1233 OVPN_RLOCK_TRACKER; 1234 1235 *nvl = nvlist_create(0); 1236 if (*nvl == NULL) 1237 return (ENOMEM); 1238 1239 #define OVPN_PEER_COUNTER_OUT(name, in, out) \ 1240 do { \ 1241 ret = ovpn_add_counters(nvpeer, name, \ 1242 OVPN_PEER_COUNTER(peer, in), OVPN_PEER_COUNTER(peer, out)); \ 1243 if (ret != 0) \ 1244 goto error; \ 1245 } while(0) 1246 1247 OVPN_RLOCK(sc); 1248 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1249 nvpeer = nvlist_create(0); 1250 if (nvpeer == NULL) { 1251 OVPN_RUNLOCK(sc); 1252 nvlist_destroy(*nvl); 1253 *nvl = NULL; 1254 return (ENOMEM); 1255 } 1256 1257 nvlist_add_number(nvpeer, "peerid", peer->peerid); 1258 1259 OVPN_PEER_COUNTER_OUT("packets", pkt_in, pkt_out); 1260 OVPN_PEER_COUNTER_OUT("bytes", bytes_in, bytes_out); 1261 1262 nvlist_append_nvlist_array(*nvl, "peers", nvpeer); 1263 nvlist_destroy(nvpeer); 1264 } 1265 #undef OVPN_PEER_COUNTER_OUT 1266 OVPN_RUNLOCK(sc); 1267 1268 return (0); 1269 1270 error: 1271 nvlist_destroy(nvpeer); 1272 nvlist_destroy(*nvl); 1273 *nvl = NULL; 1274 return (ret); 1275 } 1276 1277 static int 1278 ovpn_poll_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1279 { 1280 nvlist_t *nvl; 1281 1282 nvl = nvlist_create(0); 1283 if (nvl == NULL) 1284 return (ENOMEM); 1285 1286 nvlist_add_number(nvl, "pending", buf_ring_count(sc->notifring)); 1287 1288 *onvl = nvl; 1289 1290 return (0); 1291 } 1292 1293 static void 1294 ovpn_notif_add_counters(nvlist_t *parent, struct ovpn_notification *n) 1295 { 1296 nvlist_t *nvl; 1297 1298 nvl = nvlist_create(0); 1299 if (nvl == NULL) 1300 return; 1301 1302 nvlist_add_number(nvl, "in", n->counters.pkt_in); 1303 nvlist_add_number(nvl, "out", n->counters.pkt_out); 1304 1305 nvlist_add_nvlist(parent, "packets", nvl); 1306 nvlist_destroy(nvl); 1307 1308 nvl = nvlist_create(0); 1309 if (nvl == NULL) 1310 return; 1311 1312 nvlist_add_number(nvl, "in", n->counters.bytes_in); 1313 nvlist_add_number(nvl, "out", n->counters.bytes_out); 1314 1315 nvlist_add_nvlist(parent, "bytes", nvl); 1316 nvlist_destroy(nvl); 1317 } 1318 1319 static int 1320 opvn_get_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1321 { 1322 struct ovpn_notification *n; 1323 nvlist_t *nvl; 1324 1325 /* Check if we have notifications pending. */ 1326 n = buf_ring_dequeue_mc(sc->notifring); 1327 if (n == NULL) 1328 return (ENOENT); 1329 1330 nvl = nvlist_create(0); 1331 if (nvl == NULL) { 1332 free(n, M_OVPN); 1333 return (ENOMEM); 1334 } 1335 nvlist_add_number(nvl, "peerid", n->peerid); 1336 nvlist_add_number(nvl, "notification", n->type); 1337 if (n->type == OVPN_NOTIF_DEL_PEER) { 1338 nvlist_add_number(nvl, "del_reason", n->del_reason); 1339 1340 /* No error handling, because we want to send the notification 1341 * even if we can't attach the counters. */ 1342 ovpn_notif_add_counters(nvl, n); 1343 } 1344 free(n, M_OVPN); 1345 1346 *onvl = nvl; 1347 1348 return (0); 1349 } 1350 1351 static int 1352 ovpn_ioctl_get(struct ifnet *ifp, struct ifdrv *ifd) 1353 { 1354 struct ovpn_softc *sc = ifp->if_softc; 1355 nvlist_t *nvl = NULL; 1356 int error; 1357 1358 switch (ifd->ifd_cmd) { 1359 case OVPN_GET_STATS: 1360 error = ovpn_get_stats(sc, &nvl); 1361 break; 1362 case OVPN_GET_PEER_STATS: 1363 error = ovpn_get_peer_stats(sc, &nvl); 1364 break; 1365 case OVPN_POLL_PKT: 1366 error = ovpn_poll_pkt(sc, &nvl); 1367 break; 1368 case OVPN_GET_PKT: 1369 error = opvn_get_pkt(sc, &nvl); 1370 break; 1371 default: 1372 error = ENOTSUP; 1373 break; 1374 } 1375 1376 if (error == 0) { 1377 void *packed = NULL; 1378 size_t len; 1379 1380 MPASS(nvl != NULL); 1381 1382 packed = nvlist_pack(nvl, &len); 1383 if (! packed) { 1384 nvlist_destroy(nvl); 1385 return (ENOMEM); 1386 } 1387 1388 if (len > ifd->ifd_len) { 1389 free(packed, M_NVLIST); 1390 nvlist_destroy(nvl); 1391 return (ENOSPC); 1392 } 1393 1394 error = copyout(packed, ifd->ifd_data, len); 1395 ifd->ifd_len = len; 1396 1397 free(packed, M_NVLIST); 1398 nvlist_destroy(nvl); 1399 } 1400 1401 return (error); 1402 } 1403 1404 static int 1405 ovpn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 1406 { 1407 struct ifdrv *ifd; 1408 int error; 1409 1410 CURVNET_ASSERT_SET(); 1411 1412 switch (cmd) { 1413 case SIOCSDRVSPEC: 1414 case SIOCGDRVSPEC: 1415 error = priv_check(curthread, PRIV_NET_OVPN); 1416 if (error) 1417 return (error); 1418 break; 1419 } 1420 1421 switch (cmd) { 1422 case SIOCSDRVSPEC: 1423 ifd = (struct ifdrv *)data; 1424 error = ovpn_ioctl_set(ifp, ifd); 1425 break; 1426 case SIOCGDRVSPEC: 1427 ifd = (struct ifdrv *)data; 1428 error = ovpn_ioctl_get(ifp, ifd); 1429 break; 1430 case SIOCSIFMTU: { 1431 struct ifreq *ifr = (struct ifreq *)data; 1432 if (ifr->ifr_mtu < OVPN_MTU_MIN || ifr->ifr_mtu > OVPN_MTU_MAX) 1433 return (EINVAL); 1434 1435 ifp->if_mtu = ifr->ifr_mtu; 1436 return (0); 1437 } 1438 case SIOCSIFADDR: 1439 case SIOCADDMULTI: 1440 case SIOCDELMULTI: 1441 case SIOCGIFMTU: 1442 case SIOCSIFFLAGS: 1443 return (0); 1444 default: 1445 error = EINVAL; 1446 } 1447 1448 return (error); 1449 } 1450 1451 static int 1452 ovpn_encrypt_tx_cb(struct cryptop *crp) 1453 { 1454 struct epoch_tracker et; 1455 struct ovpn_kpeer *peer = crp->crp_opaque; 1456 struct ovpn_softc *sc = peer->sc; 1457 struct mbuf *m = crp->crp_buf.cb_mbuf; 1458 int tunnel_len; 1459 int ret; 1460 1461 CURVNET_SET(sc->ifp->if_vnet); 1462 NET_EPOCH_ENTER(et); 1463 1464 if (crp->crp_etype != 0) { 1465 crypto_freereq(crp); 1466 ovpn_peer_release_ref(peer, false); 1467 NET_EPOCH_EXIT(et); 1468 CURVNET_RESTORE(); 1469 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1470 m_freem(m); 1471 return (0); 1472 } 1473 1474 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1475 1476 tunnel_len = m->m_pkthdr.len - sizeof(struct ovpn_wire_header); 1477 ret = ovpn_encap(sc, peer->peerid, m); 1478 if (ret == 0) { 1479 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1480 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 1481 } 1482 1483 crypto_freereq(crp); 1484 ovpn_peer_release_ref(peer, false); 1485 1486 NET_EPOCH_EXIT(et); 1487 CURVNET_RESTORE(); 1488 1489 return (0); 1490 } 1491 1492 static void 1493 ovpn_finish_rx(struct ovpn_softc *sc, struct mbuf *m, 1494 struct ovpn_kpeer *peer, struct ovpn_kkey *key, uint32_t seq, 1495 struct rm_priotracker *_ovpn_lock_trackerp) 1496 { 1497 uint32_t af; 1498 1499 OVPN_RASSERT(sc); 1500 NET_EPOCH_ASSERT(); 1501 1502 /* Replay protection. */ 1503 if (V_replay_protection && ! ovpn_check_replay(key->decrypt, seq)) { 1504 OVPN_RUNLOCK(sc); 1505 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1506 m_freem(m); 1507 return; 1508 } 1509 1510 critical_enter(); 1511 *zpcpu_get(peer->last_active) = time_uptime; 1512 critical_exit(); 1513 1514 OVPN_RUNLOCK(sc); 1515 1516 OVPN_COUNTER_ADD(sc, received_data_pkts, 1); 1517 OVPN_COUNTER_ADD(sc, tunnel_bytes_received, m->m_pkthdr.len); 1518 OVPN_PEER_COUNTER_ADD(peer, pkt_in, 1); 1519 OVPN_PEER_COUNTER_ADD(peer, bytes_in, m->m_pkthdr.len); 1520 1521 /* Receive the packet on our interface. */ 1522 m->m_pkthdr.rcvif = sc->ifp; 1523 1524 /* Clear checksum flags in case the real hardware set them. */ 1525 m->m_pkthdr.csum_flags = 0; 1526 1527 /* Ensure we can read the first byte. */ 1528 m = m_pullup(m, 1); 1529 if (m == NULL) { 1530 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 1531 return; 1532 } 1533 1534 /* 1535 * Check for address family, and disregard any control packets (e.g. 1536 * keepalive). 1537 */ 1538 af = ovpn_get_af(m); 1539 if (af != 0) { 1540 BPF_MTAP2(sc->ifp, &af, sizeof(af), m); 1541 if (V_async_netisr_queue) 1542 netisr_queue(af == AF_INET ? NETISR_IP : NETISR_IPV6, m); 1543 else 1544 netisr_dispatch(af == AF_INET ? NETISR_IP : NETISR_IPV6, m); 1545 } else { 1546 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1547 m_freem(m); 1548 } 1549 } 1550 1551 static struct ovpn_kkey * 1552 ovpn_find_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, 1553 const struct ovpn_wire_header *ohdr) 1554 { 1555 struct ovpn_kkey *key = NULL; 1556 uint8_t keyid; 1557 1558 OVPN_RASSERT(sc); 1559 1560 keyid = (ntohl(ohdr->opcode) >> 24) & 0x07; 1561 1562 if (peer->keys[0].keyid == keyid) 1563 key = &peer->keys[0]; 1564 else if (peer->keys[1].keyid == keyid) 1565 key = &peer->keys[1]; 1566 1567 return (key); 1568 } 1569 1570 static int 1571 ovpn_decrypt_rx_cb(struct cryptop *crp) 1572 { 1573 struct epoch_tracker et; 1574 struct ovpn_softc *sc = crp->crp_opaque; 1575 struct mbuf *m = crp->crp_buf.cb_mbuf; 1576 struct ovpn_kkey *key; 1577 struct ovpn_kpeer *peer; 1578 struct ovpn_wire_header *ohdr; 1579 uint32_t peerid; 1580 1581 OVPN_RLOCK_TRACKER; 1582 1583 OVPN_RLOCK(sc); 1584 1585 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1586 1587 if (crp->crp_etype != 0) { 1588 crypto_freereq(crp); 1589 atomic_add_int(&sc->refcount, -1); 1590 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1591 OVPN_RUNLOCK(sc); 1592 m_freem(m); 1593 return (0); 1594 } 1595 1596 CURVNET_SET(sc->ifp->if_vnet); 1597 1598 ohdr = mtodo(m, sizeof(struct udphdr)); 1599 1600 peerid = ntohl(ohdr->opcode) & 0x00ffffff; 1601 peer = ovpn_find_peer(sc, peerid); 1602 if (peer == NULL) { 1603 /* No such peer. Drop packet. */ 1604 crypto_freereq(crp); 1605 atomic_add_int(&sc->refcount, -1); 1606 OVPN_RUNLOCK(sc); 1607 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1608 m_freem(m); 1609 CURVNET_RESTORE(); 1610 return (0); 1611 } 1612 1613 key = ovpn_find_key(sc, peer, ohdr); 1614 if (key == NULL) { 1615 crypto_freereq(crp); 1616 atomic_add_int(&sc->refcount, -1); 1617 /* 1618 * Has this key been removed between us starting the decrypt 1619 * and finishing it? 1620 */ 1621 OVPN_RUNLOCK(sc); 1622 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1623 m_freem(m); 1624 CURVNET_RESTORE(); 1625 return (0); 1626 } 1627 1628 /* Now remove the outer headers */ 1629 m_adj_decap(m, sizeof(struct udphdr) + 1630 sizeof(struct ovpn_wire_header)); 1631 1632 NET_EPOCH_ENTER(et); 1633 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), _ovpn_lock_trackerp); 1634 NET_EPOCH_EXIT(et); 1635 OVPN_UNLOCK_ASSERT(sc); 1636 1637 CURVNET_RESTORE(); 1638 1639 crypto_freereq(crp); 1640 atomic_add_int(&sc->refcount, -1); 1641 1642 return (0); 1643 } 1644 1645 static int 1646 ovpn_get_af(struct mbuf *m) 1647 { 1648 struct ip *ip; 1649 struct ip6_hdr *ip6; 1650 1651 /* 1652 * We should pullup, but we're only interested in the first byte, so 1653 * that'll always be contiguous. 1654 */ 1655 ip = mtod(m, struct ip *); 1656 if (ip->ip_v == IPVERSION) 1657 return (AF_INET); 1658 1659 ip6 = mtod(m, struct ip6_hdr *); 1660 if ((ip6->ip6_vfc & IPV6_VERSION_MASK) == IPV6_VERSION) 1661 return (AF_INET6); 1662 1663 return (0); 1664 } 1665 1666 #ifdef INET 1667 static struct ovpn_kpeer * 1668 ovpn_find_peer_by_ip(struct ovpn_softc *sc, const struct in_addr addr) 1669 { 1670 struct ovpn_kpeer *peer = NULL; 1671 1672 OVPN_ASSERT(sc); 1673 1674 /* TODO: Add a second RB so we can look up by IP. */ 1675 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1676 if (addr.s_addr == peer->vpn4.s_addr) 1677 return (peer); 1678 } 1679 1680 return (peer); 1681 } 1682 #endif 1683 1684 #ifdef INET6 1685 static struct ovpn_kpeer * 1686 ovpn_find_peer_by_ip6(struct ovpn_softc *sc, const struct in6_addr *addr) 1687 { 1688 struct ovpn_kpeer *peer = NULL; 1689 1690 OVPN_ASSERT(sc); 1691 1692 /* TODO: Add a third RB so we can look up by IPv6 address. */ 1693 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1694 if (memcmp(addr, &peer->vpn6, sizeof(*addr)) == 0) 1695 return (peer); 1696 } 1697 1698 return (peer); 1699 } 1700 #endif 1701 1702 static struct ovpn_kpeer * 1703 ovpn_route_peer(struct ovpn_softc *sc, struct mbuf **m0, 1704 const struct sockaddr *dst) 1705 { 1706 struct ovpn_kpeer *peer = NULL; 1707 int af; 1708 1709 NET_EPOCH_ASSERT(); 1710 OVPN_ASSERT(sc); 1711 1712 /* Shortcut if we're a client (or are a server and have only one client). */ 1713 if (sc->peercount == 1) 1714 return (ovpn_find_only_peer(sc)); 1715 1716 if (dst != NULL) 1717 af = dst->sa_family; 1718 else 1719 af = ovpn_get_af(*m0); 1720 1721 switch (af) { 1722 #ifdef INET 1723 case AF_INET: { 1724 const struct sockaddr_in *sa = (const struct sockaddr_in *)dst; 1725 struct nhop_object *nh; 1726 const struct in_addr *ip_dst; 1727 1728 if (sa != NULL) { 1729 ip_dst = &sa->sin_addr; 1730 } else { 1731 struct ip *ip; 1732 1733 *m0 = m_pullup(*m0, sizeof(struct ip)); 1734 if (*m0 == NULL) 1735 return (NULL); 1736 ip = mtod(*m0, struct ip *); 1737 ip_dst = &ip->ip_dst; 1738 } 1739 1740 peer = ovpn_find_peer_by_ip(sc, *ip_dst); 1741 SDT_PROBE2(if_ovpn, tx, route, ip4, ip_dst, peer); 1742 if (peer == NULL) { 1743 nh = fib4_lookup(M_GETFIB(*m0), *ip_dst, 0, 1744 NHR_NONE, 0); 1745 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1746 peer = ovpn_find_peer_by_ip(sc, 1747 nh->gw4_sa.sin_addr); 1748 SDT_PROBE2(if_ovpn, tx, route, ip4, 1749 &nh->gw4_sa.sin_addr, peer); 1750 } 1751 } 1752 break; 1753 } 1754 #endif 1755 #ifdef INET6 1756 case AF_INET6: { 1757 const struct sockaddr_in6 *sa6 = 1758 (const struct sockaddr_in6 *)dst; 1759 struct nhop_object *nh; 1760 const struct in6_addr *ip6_dst; 1761 1762 if (sa6 != NULL) { 1763 ip6_dst = &sa6->sin6_addr; 1764 } else { 1765 struct ip6_hdr *ip6; 1766 1767 *m0 = m_pullup(*m0, sizeof(struct ip6_hdr)); 1768 if (*m0 == NULL) 1769 return (NULL); 1770 ip6 = mtod(*m0, struct ip6_hdr *); 1771 ip6_dst = &ip6->ip6_dst; 1772 } 1773 1774 peer = ovpn_find_peer_by_ip6(sc, ip6_dst); 1775 SDT_PROBE2(if_ovpn, tx, route, ip6, ip6_dst, peer); 1776 if (peer == NULL) { 1777 nh = fib6_lookup(M_GETFIB(*m0), ip6_dst, 0, 1778 NHR_NONE, 0); 1779 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1780 peer = ovpn_find_peer_by_ip6(sc, 1781 &nh->gw6_sa.sin6_addr); 1782 SDT_PROBE2(if_ovpn, tx, route, ip6, 1783 &nh->gw6_sa.sin6_addr, peer); 1784 } 1785 } 1786 break; 1787 } 1788 #endif 1789 } 1790 1791 return (peer); 1792 } 1793 1794 static int 1795 ovpn_transmit(struct ifnet *ifp, struct mbuf *m) 1796 { 1797 return (ifp->if_output(ifp, m, NULL, NULL)); 1798 } 1799 1800 static int 1801 ovpn_transmit_to_peer(struct ifnet *ifp, struct mbuf *m, 1802 struct ovpn_kpeer *peer, struct rm_priotracker *_ovpn_lock_trackerp) 1803 { 1804 struct ovpn_wire_header *ohdr; 1805 struct ovpn_kkey *key; 1806 struct ovpn_softc *sc; 1807 struct cryptop *crp; 1808 uint32_t af, seq; 1809 size_t len, ovpn_hdr_len; 1810 int tunnel_len; 1811 int ret; 1812 1813 sc = ifp->if_softc; 1814 1815 OVPN_RASSERT(sc); 1816 1817 tunnel_len = m->m_pkthdr.len; 1818 1819 key = &peer->keys[OVPN_KEY_SLOT_PRIMARY]; 1820 if (key->encrypt == NULL) { 1821 if (_ovpn_lock_trackerp != NULL) 1822 OVPN_RUNLOCK(sc); 1823 m_freem(m); 1824 return (ENOLINK); 1825 } 1826 1827 af = ovpn_get_af(m); 1828 /* Don't capture control packets. */ 1829 if (af != 0) 1830 BPF_MTAP2(ifp, &af, sizeof(af), m); 1831 1832 len = m->m_pkthdr.len; 1833 MPASS(len <= ifp->if_mtu); 1834 1835 ovpn_hdr_len = sizeof(struct ovpn_wire_header); 1836 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) 1837 ovpn_hdr_len -= 16; /* No auth tag. */ 1838 1839 M_PREPEND(m, ovpn_hdr_len, M_NOWAIT); 1840 if (m == NULL) { 1841 if (_ovpn_lock_trackerp != NULL) 1842 OVPN_RUNLOCK(sc); 1843 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1844 return (ENOBUFS); 1845 } 1846 ohdr = mtod(m, struct ovpn_wire_header *); 1847 ohdr->opcode = (OVPN_OP_DATA_V2 << OVPN_OP_SHIFT) | key->keyid; 1848 ohdr->opcode <<= 24; 1849 ohdr->opcode |= key->peerid; 1850 ohdr->opcode = htonl(ohdr->opcode); 1851 1852 seq = atomic_fetchadd_32(&peer->tx_seq, 1); 1853 seq = htonl(seq); 1854 ohdr->seq = seq; 1855 1856 OVPN_PEER_COUNTER_ADD(peer, pkt_out, 1); 1857 OVPN_PEER_COUNTER_ADD(peer, bytes_out, len); 1858 1859 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) { 1860 ret = ovpn_encap(sc, peer->peerid, m); 1861 if (_ovpn_lock_trackerp != NULL) 1862 OVPN_RUNLOCK(sc); 1863 if (ret == 0) { 1864 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1865 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 1866 } 1867 return (ret); 1868 } 1869 1870 crp = crypto_getreq(key->encrypt->cryptoid, M_NOWAIT); 1871 if (crp == NULL) { 1872 if (_ovpn_lock_trackerp != NULL) 1873 OVPN_RUNLOCK(sc); 1874 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1875 m_freem(m); 1876 return (ENOBUFS); 1877 } 1878 1879 /* Encryption covers only the payload, not the header. */ 1880 crp->crp_payload_start = sizeof(*ohdr); 1881 crp->crp_payload_length = len; 1882 crp->crp_op = CRYPTO_OP_ENCRYPT; 1883 1884 /* 1885 * AAD data covers the ovpn_wire_header minus the auth 1886 * tag. 1887 */ 1888 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 1889 crp->crp_aad = ohdr; 1890 crp->crp_aad_start = 0; 1891 crp->crp_op |= CRYPTO_OP_COMPUTE_DIGEST; 1892 crp->crp_digest_start = offsetof(struct ovpn_wire_header, auth_tag); 1893 1894 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 1895 memcpy(crp->crp_iv, &seq, sizeof(seq)); 1896 memcpy(crp->crp_iv + sizeof(seq), key->encrypt->nonce, 1897 key->encrypt->noncelen); 1898 1899 crypto_use_mbuf(crp, m); 1900 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 1901 crp->crp_callback = ovpn_encrypt_tx_cb; 1902 crp->crp_opaque = peer; 1903 1904 atomic_add_int(&peer->refcount, 1); 1905 if (_ovpn_lock_trackerp != NULL) 1906 OVPN_RUNLOCK(sc); 1907 if (V_async_crypto) 1908 ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); 1909 else 1910 ret = crypto_dispatch(crp); 1911 if (ret) { 1912 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1913 } 1914 1915 return (ret); 1916 } 1917 1918 /* 1919 * Note: Expects to hold the read lock on entry, and will release it itself. 1920 */ 1921 static int 1922 ovpn_encap(struct ovpn_softc *sc, uint32_t peerid, struct mbuf *m) 1923 { 1924 struct udphdr *udp; 1925 struct ovpn_kpeer *peer; 1926 int len; 1927 1928 OVPN_RLOCK_TRACKER; 1929 1930 OVPN_RLOCK(sc); 1931 NET_EPOCH_ASSERT(); 1932 1933 peer = ovpn_find_peer(sc, peerid); 1934 if (peer == NULL || sc->ifp->if_link_state != LINK_STATE_UP) { 1935 OVPN_RUNLOCK(sc); 1936 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1937 m_freem(m); 1938 return (ENETDOWN); 1939 } 1940 1941 len = m->m_pkthdr.len; 1942 1943 M_PREPEND(m, sizeof(struct udphdr), M_NOWAIT); 1944 if (m == NULL) { 1945 OVPN_RUNLOCK(sc); 1946 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1947 m_freem(m); 1948 return (ENOBUFS); 1949 } 1950 udp = mtod(m, struct udphdr *); 1951 1952 MPASS(peer->local.ss_family == peer->remote.ss_family); 1953 1954 udp->uh_sport = ovpn_get_port(&peer->local); 1955 udp->uh_dport = ovpn_get_port(&peer->remote); 1956 udp->uh_ulen = htons(sizeof(struct udphdr) + len); 1957 1958 switch (peer->remote.ss_family) { 1959 #ifdef INET 1960 case AF_INET: { 1961 struct sockaddr_in *in_local = TO_IN(&peer->local); 1962 struct sockaddr_in *in_remote = TO_IN(&peer->remote); 1963 struct ip *ip; 1964 1965 /* 1966 * This requires knowing the source IP, which we don't. Happily 1967 * we're allowed to keep this at 0, and the checksum won't do 1968 * anything the crypto won't already do. 1969 */ 1970 udp->uh_sum = 0; 1971 1972 /* Set the checksum flags so we recalculate checksums. */ 1973 m->m_pkthdr.csum_flags |= CSUM_IP; 1974 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 1975 1976 M_PREPEND(m, sizeof(struct ip), M_NOWAIT); 1977 if (m == NULL) { 1978 OVPN_RUNLOCK(sc); 1979 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1980 return (ENOBUFS); 1981 } 1982 ip = mtod(m, struct ip *); 1983 1984 ip->ip_tos = 0; 1985 ip->ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) + 1986 len); 1987 ip->ip_off = 0; 1988 ip->ip_ttl = V_ip_defttl; 1989 ip->ip_p = IPPROTO_UDP; 1990 ip->ip_sum = 0; 1991 if (in_local->sin_port != 0) 1992 ip->ip_src = in_local->sin_addr; 1993 else 1994 ip->ip_src.s_addr = INADDR_ANY; 1995 ip->ip_dst = in_remote->sin_addr; 1996 1997 OVPN_RUNLOCK(sc); 1998 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 1999 2000 return (ip_output(m, NULL, NULL, 0, NULL, NULL)); 2001 } 2002 #endif 2003 #ifdef INET6 2004 case AF_INET6: { 2005 struct sockaddr_in6 *in6_local = TO_IN6(&peer->local); 2006 struct sockaddr_in6 *in6_remote = TO_IN6(&peer->remote); 2007 struct ip6_hdr *ip6; 2008 2009 M_PREPEND(m, sizeof(struct ip6_hdr), M_NOWAIT); 2010 if (m == NULL) { 2011 OVPN_RUNLOCK(sc); 2012 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2013 return (ENOBUFS); 2014 } 2015 m = m_pullup(m, sizeof(*ip6) + sizeof(*udp)); 2016 if (m == NULL) { 2017 OVPN_RUNLOCK(sc); 2018 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2019 return (ENOBUFS); 2020 } 2021 2022 ip6 = mtod(m, struct ip6_hdr *); 2023 2024 ip6->ip6_vfc = IPV6_VERSION; 2025 ip6->ip6_flow &= ~IPV6_FLOWINFO_MASK; 2026 ip6->ip6_plen = htons(sizeof(*ip6) + sizeof(struct udphdr) + 2027 len); 2028 ip6->ip6_nxt = IPPROTO_UDP; 2029 ip6->ip6_hlim = V_ip6_defhlim; 2030 2031 memcpy(&ip6->ip6_src, &in6_local->sin6_addr, 2032 sizeof(ip6->ip6_src)); 2033 memcpy(&ip6->ip6_dst, &in6_remote->sin6_addr, 2034 sizeof(ip6->ip6_dst)); 2035 2036 udp = mtodo(m, sizeof(*ip6)); 2037 udp->uh_sum = in6_cksum_pseudo(ip6, 2038 m->m_pkthdr.len - sizeof(struct ip6_hdr), 2039 IPPROTO_UDP, 0); 2040 2041 m->m_pkthdr.csum_flags |= CSUM_UDP_IPV6; 2042 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 2043 2044 OVPN_RUNLOCK(sc); 2045 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 2046 2047 return (ip6_output(m, NULL, NULL, IPV6_UNSPECSRC, NULL, NULL, 2048 NULL)); 2049 } 2050 #endif 2051 default: 2052 panic("Unsupported address family %d", 2053 peer->remote.ss_family); 2054 } 2055 } 2056 2057 static int 2058 ovpn_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 2059 struct route *ro) 2060 { 2061 struct ovpn_softc *sc; 2062 struct ovpn_kpeer *peer; 2063 2064 OVPN_RLOCK_TRACKER; 2065 2066 sc = ifp->if_softc; 2067 2068 OVPN_RLOCK(sc); 2069 2070 SDT_PROBE1(if_ovpn, tx, transmit, start, m); 2071 2072 if (__predict_false(ifp->if_link_state != LINK_STATE_UP)) { 2073 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2074 OVPN_RUNLOCK(sc); 2075 m_freem(m); 2076 return (ENETDOWN); 2077 } 2078 2079 /** 2080 * Only obey 'dst' (i.e. the gateway) if no route is supplied. 2081 * That's our indication that we're being called through pf's route-to, 2082 * and we should route according to 'dst' instead. We can't do so 2083 * consistently, because the usual openvpn configuration sets the first 2084 * non-server IP in the subnet as the gateway. If we always use that 2085 * one we'd end up routing all traffic to the first client. 2086 * tl;dr: 'ro == NULL' tells us pf is doing a route-to, and then but 2087 * only then, we should treat 'dst' as the destination. */ 2088 peer = ovpn_route_peer(sc, &m, ro == NULL ? dst : NULL); 2089 if (peer == NULL) { 2090 /* No destination. */ 2091 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2092 OVPN_RUNLOCK(sc); 2093 m_freem(m); 2094 return (ENETDOWN); 2095 } 2096 2097 return (ovpn_transmit_to_peer(ifp, m, peer, _ovpn_lock_trackerp)); 2098 } 2099 2100 static bool 2101 ovpn_check_replay(struct ovpn_kkey_dir *key, uint32_t seq) 2102 { 2103 uint32_t d; 2104 2105 mtx_lock(&key->replay_mtx); 2106 2107 /* Sequence number must be strictly greater than rx_seq */ 2108 if (seq <= key->rx_seq) { 2109 mtx_unlock(&key->replay_mtx); 2110 return (false); 2111 } 2112 2113 /* Large jump. The packet authenticated okay, so just accept that. */ 2114 if (seq > (key->rx_seq + (sizeof(key->rx_window) * 8))) { 2115 key->rx_seq = seq; 2116 key->rx_window = 0; 2117 mtx_unlock(&key->replay_mtx); 2118 return (true); 2119 } 2120 2121 /* Happy case. */ 2122 if ((seq == key->rx_seq + 1) && key->rx_window == 0) { 2123 key->rx_seq++; 2124 mtx_unlock(&key->replay_mtx); 2125 return (true); 2126 } 2127 2128 d = seq - key->rx_seq - 1; 2129 2130 if (key->rx_window & ((uint64_t)1 << d)) { 2131 /* Dupe! */ 2132 mtx_unlock(&key->replay_mtx); 2133 return (false); 2134 } 2135 2136 key->rx_window |= (uint64_t)1 << d; 2137 2138 while (key->rx_window & 1) { 2139 key->rx_seq++; 2140 key->rx_window >>= 1; 2141 } 2142 2143 mtx_unlock(&key->replay_mtx); 2144 2145 return (true); 2146 } 2147 2148 static struct ovpn_kpeer * 2149 ovpn_peer_from_mbuf(struct ovpn_softc *sc, struct mbuf *m, int off) 2150 { 2151 struct ovpn_wire_header ohdr; 2152 uint32_t peerid; 2153 const size_t hdrlen = sizeof(ohdr) - sizeof(ohdr.auth_tag); 2154 2155 OVPN_RASSERT(sc); 2156 2157 if (m_length(m, NULL) < (off + sizeof(struct udphdr) + hdrlen)) 2158 return (NULL); 2159 2160 m_copydata(m, off + sizeof(struct udphdr), hdrlen, (caddr_t)&ohdr); 2161 2162 peerid = ntohl(ohdr.opcode) & 0x00ffffff; 2163 2164 return (ovpn_find_peer(sc, peerid)); 2165 } 2166 2167 static bool 2168 ovpn_udp_input(struct mbuf *m, int off, struct inpcb *inp, 2169 const struct sockaddr *sa, void *ctx) 2170 { 2171 struct ovpn_softc *sc = ctx; 2172 struct ovpn_wire_header tmphdr; 2173 struct ovpn_wire_header *ohdr; 2174 struct udphdr *uhdr; 2175 struct ovpn_kkey *key; 2176 struct cryptop *crp; 2177 struct ovpn_kpeer *peer; 2178 size_t ohdrlen; 2179 int ret; 2180 uint8_t op; 2181 2182 OVPN_RLOCK_TRACKER; 2183 2184 M_ASSERTPKTHDR(m); 2185 2186 OVPN_COUNTER_ADD(sc, transport_bytes_received, m->m_pkthdr.len - off); 2187 2188 ohdrlen = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2189 2190 OVPN_RLOCK(sc); 2191 2192 peer = ovpn_peer_from_mbuf(sc, m, off); 2193 if (peer == NULL) { 2194 OVPN_RUNLOCK(sc); 2195 return (false); 2196 } 2197 2198 if (m_length(m, NULL) < (off + sizeof(*uhdr) + ohdrlen)) { 2199 /* Short packet. */ 2200 OVPN_RUNLOCK(sc); 2201 return (false); 2202 } 2203 2204 m_copydata(m, off + sizeof(*uhdr), ohdrlen, (caddr_t)&tmphdr); 2205 2206 op = ntohl(tmphdr.opcode) >> 24 >> OVPN_OP_SHIFT; 2207 if (op != OVPN_OP_DATA_V2) { 2208 /* Control packet? */ 2209 OVPN_RUNLOCK(sc); 2210 return (false); 2211 } 2212 2213 m = m_pullup(m, off + sizeof(*uhdr) + ohdrlen); 2214 if (m == NULL) { 2215 OVPN_RUNLOCK(sc); 2216 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2217 return (true); 2218 } 2219 2220 /* 2221 * Simplify things by getting rid of the preceding headers, we don't 2222 * care about them. 2223 */ 2224 m_adj_decap(m, off); 2225 2226 uhdr = mtodo(m, 0); 2227 ohdr = mtodo(m, sizeof(*uhdr)); 2228 2229 key = ovpn_find_key(sc, peer, ohdr); 2230 if (key == NULL || key->decrypt == NULL) { 2231 OVPN_RUNLOCK(sc); 2232 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2233 m_freem(m); 2234 return (true); 2235 } 2236 2237 if (key->decrypt->cipher == OVPN_CIPHER_ALG_NONE) { 2238 /* Now remove the outer headers */ 2239 m_adj_decap(m, sizeof(struct udphdr) + ohdrlen); 2240 2241 ohdr = mtodo(m, sizeof(*uhdr)); 2242 2243 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), 2244 _ovpn_lock_trackerp); 2245 OVPN_UNLOCK_ASSERT(sc); 2246 return (true); 2247 } 2248 2249 ohdrlen += sizeof(ohdr->auth_tag); 2250 2251 m = m_pullup(m, sizeof(*uhdr) + ohdrlen); 2252 if (m == NULL) { 2253 OVPN_RUNLOCK(sc); 2254 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2255 return (true); 2256 } 2257 uhdr = mtodo(m, 0); 2258 ohdr = mtodo(m, sizeof(*uhdr)); 2259 2260 /* Decrypt */ 2261 crp = crypto_getreq(key->decrypt->cryptoid, M_NOWAIT); 2262 if (crp == NULL) { 2263 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2264 OVPN_RUNLOCK(sc); 2265 m_freem(m); 2266 return (true); 2267 } 2268 2269 crp->crp_payload_start = sizeof(struct udphdr) + sizeof(*ohdr); 2270 crp->crp_payload_length = ntohs(uhdr->uh_ulen) - 2271 sizeof(*uhdr) - sizeof(*ohdr); 2272 crp->crp_op = CRYPTO_OP_DECRYPT; 2273 2274 /* AAD validation. */ 2275 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2276 crp->crp_aad = ohdr; 2277 crp->crp_aad_start = 0; 2278 crp->crp_op |= CRYPTO_OP_VERIFY_DIGEST; 2279 crp->crp_digest_start = sizeof(struct udphdr) + 2280 offsetof(struct ovpn_wire_header, auth_tag); 2281 2282 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 2283 memcpy(crp->crp_iv, &ohdr->seq, sizeof(ohdr->seq)); 2284 memcpy(crp->crp_iv + sizeof(ohdr->seq), key->decrypt->nonce, 2285 key->decrypt->noncelen); 2286 2287 crypto_use_mbuf(crp, m); 2288 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 2289 crp->crp_callback = ovpn_decrypt_rx_cb; 2290 crp->crp_opaque = sc; 2291 2292 atomic_add_int(&sc->refcount, 1); 2293 OVPN_RUNLOCK(sc); 2294 if (V_async_crypto) 2295 ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); 2296 else 2297 ret = crypto_dispatch(crp); 2298 if (ret != 0) { 2299 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2300 } 2301 2302 return (true); 2303 } 2304 2305 static void 2306 ovpn_qflush(struct ifnet *ifp __unused) 2307 { 2308 2309 } 2310 2311 static void 2312 ovpn_flush_rxring(struct ovpn_softc *sc) 2313 { 2314 struct ovpn_notification *n; 2315 2316 OVPN_WASSERT(sc); 2317 2318 while (! buf_ring_empty(sc->notifring)) { 2319 n = buf_ring_dequeue_sc(sc->notifring); 2320 free(n, M_OVPN); 2321 } 2322 } 2323 2324 #ifdef VIMAGE 2325 static void 2326 ovpn_reassign(struct ifnet *ifp, struct vnet *new_vnet __unused, 2327 char *unused __unused) 2328 { 2329 struct ovpn_softc *sc = ifp->if_softc; 2330 struct ovpn_kpeer *peer, *tmppeer; 2331 int ret __diagused; 2332 2333 OVPN_WLOCK(sc); 2334 2335 /* Flush keys & configuration. */ 2336 RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) { 2337 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 2338 ret = _ovpn_del_peer(sc, peer); 2339 MPASS(ret == 0); 2340 } 2341 2342 ovpn_flush_rxring(sc); 2343 2344 OVPN_WUNLOCK(sc); 2345 } 2346 #endif 2347 2348 static int 2349 ovpn_clone_match(struct if_clone *ifc, const char *name) 2350 { 2351 /* 2352 * Allow all names that start with 'ovpn', specifically because pfSense 2353 * uses ovpnc1 / ovpns2 2354 */ 2355 return (strncmp(ovpnname, name, strlen(ovpnname)) == 0); 2356 } 2357 2358 static int 2359 ovpn_clone_create(struct if_clone *ifc, char *name, size_t len, 2360 struct ifc_data *ifd, struct ifnet **ifpp) 2361 { 2362 struct ovpn_softc *sc; 2363 struct ifnet *ifp; 2364 char *dp; 2365 int error, unit, wildcard; 2366 2367 /* Try to see if a special unit was requested. */ 2368 error = ifc_name2unit(name, &unit); 2369 if (error != 0) 2370 return (error); 2371 wildcard = (unit < 0); 2372 2373 error = ifc_alloc_unit(ifc, &unit); 2374 if (error != 0) 2375 return (error); 2376 2377 /* 2378 * If no unit had been given, we need to adjust the ifName. 2379 */ 2380 for (dp = name; *dp != '\0'; dp++); 2381 if (wildcard) { 2382 error = snprintf(dp, len - (dp - name), "%d", unit); 2383 if (error > len - (dp - name)) { 2384 /* ifName too long. */ 2385 ifc_free_unit(ifc, unit); 2386 return (ENOSPC); 2387 } 2388 dp += error; 2389 } 2390 2391 /* Make sure it doesn't already exist. */ 2392 if (ifunit(name) != NULL) 2393 return (EEXIST); 2394 2395 sc = malloc(sizeof(struct ovpn_softc), M_OVPN, M_WAITOK | M_ZERO); 2396 sc->ifp = if_alloc(IFT_ENC); 2397 rm_init_flags(&sc->lock, "if_ovpn_lock", RM_RECURSE); 2398 sc->refcount = 0; 2399 2400 sc->notifring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL); 2401 2402 COUNTER_ARRAY_ALLOC(sc->counters, OVPN_COUNTER_SIZE, M_WAITOK); 2403 2404 ifp = sc->ifp; 2405 ifp->if_softc = sc; 2406 strlcpy(ifp->if_xname, name, IFNAMSIZ); 2407 ifp->if_dname = ovpngroupname; 2408 ifp->if_dunit = unit; 2409 2410 ifp->if_addrlen = 0; 2411 ifp->if_mtu = 1428; 2412 ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST; 2413 ifp->if_ioctl = ovpn_ioctl; 2414 ifp->if_transmit = ovpn_transmit; 2415 ifp->if_output = ovpn_output; 2416 ifp->if_qflush = ovpn_qflush; 2417 #ifdef VIMAGE 2418 ifp->if_reassign = ovpn_reassign; 2419 #endif 2420 ifp->if_capabilities |= IFCAP_LINKSTATE; 2421 ifp->if_capenable |= IFCAP_LINKSTATE; 2422 2423 if_attach(ifp); 2424 bpfattach(ifp, DLT_NULL, sizeof(uint32_t)); 2425 *ifpp = ifp; 2426 2427 return (0); 2428 } 2429 2430 static void 2431 ovpn_clone_destroy_cb(struct epoch_context *ctx) 2432 { 2433 struct ovpn_softc *sc; 2434 2435 sc = __containerof(ctx, struct ovpn_softc, epoch_ctx); 2436 2437 MPASS(sc->peercount == 0); 2438 MPASS(RB_EMPTY(&sc->peers)); 2439 2440 COUNTER_ARRAY_FREE(sc->counters, OVPN_COUNTER_SIZE); 2441 2442 if_free(sc->ifp); 2443 free(sc, M_OVPN); 2444 } 2445 2446 static int 2447 ovpn_clone_destroy(struct if_clone *ifc, struct ifnet *ifp, uint32_t flags) 2448 { 2449 struct ovpn_softc *sc; 2450 struct ovpn_kpeer *peer, *tmppeer; 2451 int unit; 2452 int ret __diagused; 2453 2454 sc = ifp->if_softc; 2455 unit = ifp->if_dunit; 2456 2457 OVPN_WLOCK(sc); 2458 2459 if (atomic_load_int(&sc->refcount) > 0) { 2460 OVPN_WUNLOCK(sc); 2461 return (EBUSY); 2462 } 2463 2464 RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) { 2465 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 2466 ret = _ovpn_del_peer(sc, peer); 2467 MPASS(ret == 0); 2468 } 2469 2470 ovpn_flush_rxring(sc); 2471 buf_ring_free(sc->notifring, M_OVPN); 2472 2473 OVPN_WUNLOCK(sc); 2474 2475 bpfdetach(ifp); 2476 if_detach(ifp); 2477 ifp->if_softc = NULL; 2478 2479 NET_EPOCH_CALL(ovpn_clone_destroy_cb, &sc->epoch_ctx); 2480 2481 if (unit != IF_DUNIT_NONE) 2482 ifc_free_unit(ifc, unit); 2483 2484 NET_EPOCH_DRAIN_CALLBACKS(); 2485 2486 return (0); 2487 } 2488 2489 static void 2490 vnet_ovpn_init(const void *unused __unused) 2491 { 2492 struct if_clone_addreq req = { 2493 .match_f = ovpn_clone_match, 2494 .create_f = ovpn_clone_create, 2495 .destroy_f = ovpn_clone_destroy, 2496 }; 2497 V_ovpn_cloner = ifc_attach_cloner(ovpngroupname, &req); 2498 } 2499 VNET_SYSINIT(vnet_ovpn_init, SI_SUB_PSEUDO, SI_ORDER_ANY, 2500 vnet_ovpn_init, NULL); 2501 2502 static void 2503 vnet_ovpn_uninit(const void *unused __unused) 2504 { 2505 if_clone_detach(V_ovpn_cloner); 2506 } 2507 VNET_SYSUNINIT(vnet_ovpn_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY, 2508 vnet_ovpn_uninit, NULL); 2509 2510 static int 2511 ovpnmodevent(module_t mod, int type, void *data) 2512 { 2513 switch (type) { 2514 case MOD_LOAD: 2515 /* Done in vnet_ovpn_init() */ 2516 break; 2517 case MOD_UNLOAD: 2518 /* Done in vnet_ovpn_uninit() */ 2519 break; 2520 default: 2521 return (EOPNOTSUPP); 2522 } 2523 2524 return (0); 2525 } 2526 2527 static moduledata_t ovpn_mod = { 2528 "if_ovpn", 2529 ovpnmodevent, 2530 0 2531 }; 2532 2533 DECLARE_MODULE(if_ovpn, ovpn_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 2534 MODULE_VERSION(if_ovpn, 1); 2535