1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2021-2022 Rubicon Communications, LLC (Netgate) 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 */ 28 #include "opt_inet.h" 29 #include "opt_inet6.h" 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/buf_ring.h> 34 #include <sys/epoch.h> 35 #include <sys/file.h> 36 #include <sys/filedesc.h> 37 #include <sys/kernel.h> 38 #include <sys/malloc.h> 39 #include <sys/mbuf.h> 40 #include <sys/module.h> 41 #include <sys/nv.h> 42 #include <sys/priv.h> 43 #include <sys/protosw.h> 44 #include <sys/rmlock.h> 45 #include <sys/sdt.h> 46 #include <sys/smp.h> 47 #include <sys/socket.h> 48 #include <sys/socketvar.h> 49 #include <sys/sockio.h> 50 #include <sys/sysctl.h> 51 #include <sys/time.h> 52 53 #include <machine/atomic.h> 54 55 #include <net/bpf.h> 56 #include <net/if.h> 57 #include <net/if_clone.h> 58 #include <net/if_types.h> 59 #include <net/if_var.h> 60 #include <net/if_private.h> 61 #include <net/netisr.h> 62 #include <net/route/nhop.h> 63 64 #include <netinet/in.h> 65 #include <netinet/in_fib.h> 66 #include <netinet/ip.h> 67 #include <netinet/ip6.h> 68 #include <netinet/ip_var.h> 69 #include <netinet/udp.h> 70 #include <netinet/udp_var.h> 71 72 #include <netinet6/ip6_var.h> 73 #include <netinet6/in6_fib.h> 74 75 #include <machine/in_cksum.h> 76 77 #include <opencrypto/cryptodev.h> 78 79 #include "if_ovpn.h" 80 81 struct ovpn_kkey_dir { 82 int refcount; 83 uint8_t key[32]; 84 uint8_t keylen; 85 uint8_t nonce[8]; 86 uint8_t noncelen; 87 enum ovpn_key_cipher cipher; 88 crypto_session_t cryptoid; 89 90 struct mtx replay_mtx; 91 /* 92 * Last seen gapless sequence number. New rx seq numbers must be 93 * strictly higher than this. 94 */ 95 uint32_t rx_seq; 96 uint64_t tx_seq; 97 98 /* Seen packets, relative to rx_seq. bit(0) will always be 0. */ 99 uint64_t rx_window; 100 }; 101 102 struct ovpn_kkey { 103 struct ovpn_kkey_dir *encrypt; 104 struct ovpn_kkey_dir *decrypt; 105 uint8_t keyid; 106 uint32_t peerid; 107 }; 108 109 struct ovpn_keepalive { 110 uint32_t interval; 111 uint32_t timeout; 112 }; 113 114 struct ovpn_wire_header { 115 uint32_t opcode; /* opcode, key id, peer id */ 116 uint32_t seq; 117 uint8_t auth_tag[16]; 118 }; 119 120 struct ovpn_peer_counters { 121 uint64_t pkt_in; 122 uint64_t pkt_out; 123 uint64_t bytes_in; 124 uint64_t bytes_out; 125 }; 126 #define OVPN_PEER_COUNTER_SIZE (sizeof(struct ovpn_peer_counters)/sizeof(uint64_t)) 127 128 struct ovpn_notification { 129 enum ovpn_notif_type type; 130 uint32_t peerid; 131 132 /* Delete notification */ 133 enum ovpn_del_reason del_reason; 134 struct ovpn_peer_counters counters; 135 }; 136 137 struct ovpn_softc; 138 139 struct ovpn_kpeer { 140 RB_ENTRY(ovpn_kpeer) tree; 141 int refcount; 142 uint32_t peerid; 143 144 struct ovpn_softc *sc; 145 struct sockaddr_storage local; 146 struct sockaddr_storage remote; 147 148 struct in_addr vpn4; 149 struct in6_addr vpn6; 150 151 struct ovpn_kkey keys[2]; 152 153 enum ovpn_del_reason del_reason; 154 struct ovpn_keepalive keepalive; 155 uint32_t *last_active; 156 struct callout ping_send; 157 struct callout ping_rcv; 158 159 counter_u64_t counters[OVPN_PEER_COUNTER_SIZE]; 160 }; 161 162 struct ovpn_counters { 163 uint64_t lost_ctrl_pkts_in; 164 uint64_t lost_ctrl_pkts_out; 165 uint64_t lost_data_pkts_in; 166 uint64_t lost_data_pkts_out; 167 uint64_t nomem_data_pkts_in; 168 uint64_t nomem_data_pkts_out; 169 uint64_t received_ctrl_pkts; 170 uint64_t received_data_pkts; 171 uint64_t sent_ctrl_pkts; 172 uint64_t sent_data_pkts; 173 174 uint64_t transport_bytes_sent; 175 uint64_t transport_bytes_received; 176 uint64_t tunnel_bytes_sent; 177 uint64_t tunnel_bytes_received; 178 }; 179 #define OVPN_COUNTER_SIZE (sizeof(struct ovpn_counters)/sizeof(uint64_t)) 180 181 RB_HEAD(ovpn_kpeers, ovpn_kpeer); 182 183 struct ovpn_softc { 184 int refcount; 185 struct rmlock lock; 186 struct ifnet *ifp; 187 struct socket *so; 188 int peercount; 189 struct ovpn_kpeers peers; 190 191 /* Pending notification */ 192 struct buf_ring *notifring; 193 194 counter_u64_t counters[OVPN_COUNTER_SIZE]; 195 196 struct epoch_context epoch_ctx; 197 }; 198 199 static struct ovpn_kpeer *ovpn_find_peer(struct ovpn_softc *, uint32_t); 200 static bool ovpn_udp_input(struct mbuf *, int, struct inpcb *, 201 const struct sockaddr *, void *); 202 static int ovpn_transmit_to_peer(struct ifnet *, struct mbuf *, 203 struct ovpn_kpeer *, struct rm_priotracker *); 204 static int ovpn_encap(struct ovpn_softc *, uint32_t, struct mbuf *); 205 static int ovpn_get_af(struct mbuf *); 206 static void ovpn_free_kkey_dir(struct ovpn_kkey_dir *); 207 static bool ovpn_check_replay(struct ovpn_kkey_dir *, uint32_t); 208 static int ovpn_peer_compare(struct ovpn_kpeer *, struct ovpn_kpeer *); 209 210 static RB_PROTOTYPE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare); 211 static RB_GENERATE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare); 212 213 #define OVPN_MTU_MIN 576 214 #define OVPN_MTU_MAX (IP_MAXPACKET - sizeof(struct ip) - \ 215 sizeof(struct udphdr) - sizeof(struct ovpn_wire_header)) 216 217 #define OVPN_OP_DATA_V2 0x09 218 #define OVPN_OP_SHIFT 3 219 #define OVPN_SEQ_ROTATE 0x80000000 220 221 VNET_DEFINE_STATIC(struct if_clone *, ovpn_cloner); 222 #define V_ovpn_cloner VNET(ovpn_cloner) 223 224 #define OVPN_RLOCK_TRACKER struct rm_priotracker _ovpn_lock_tracker; \ 225 struct rm_priotracker *_ovpn_lock_trackerp = &_ovpn_lock_tracker 226 #define OVPN_RLOCK(sc) rm_rlock(&(sc)->lock, _ovpn_lock_trackerp) 227 #define OVPN_RUNLOCK(sc) rm_runlock(&(sc)->lock, _ovpn_lock_trackerp) 228 #define OVPN_WLOCK(sc) rm_wlock(&(sc)->lock) 229 #define OVPN_WUNLOCK(sc) rm_wunlock(&(sc)->lock) 230 #define OVPN_ASSERT(sc) rm_assert(&(sc)->lock, RA_LOCKED) 231 #define OVPN_RASSERT(sc) rm_assert(&(sc)->lock, RA_RLOCKED) 232 #define OVPN_WASSERT(sc) rm_assert(&(sc)->lock, RA_WLOCKED) 233 #define OVPN_UNLOCK_ASSERT(sc) rm_assert(&(sc)->lock, RA_UNLOCKED) 234 235 #define OVPN_COUNTER(sc, name) \ 236 ((sc)->counters[offsetof(struct ovpn_counters, name)/sizeof(uint64_t)]) 237 #define OVPN_PEER_COUNTER(peer, name) \ 238 ((peer)->counters[offsetof(struct ovpn_peer_counters, name) / \ 239 sizeof(uint64_t)]) 240 241 #define OVPN_COUNTER_ADD(sc, name, val) \ 242 counter_u64_add(OVPN_COUNTER(sc, name), val) 243 #define OVPN_PEER_COUNTER_ADD(p, name, val) \ 244 counter_u64_add(OVPN_PEER_COUNTER(p, name), val) 245 246 #define TO_IN(x) ((struct sockaddr_in *)(x)) 247 #define TO_IN6(x) ((struct sockaddr_in6 *)(x)) 248 249 SDT_PROVIDER_DEFINE(if_ovpn); 250 SDT_PROBE_DEFINE1(if_ovpn, tx, transmit, start, "struct mbuf *"); 251 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip4, "struct in_addr *", "struct ovpn_kpeer *"); 252 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip6, "struct in6_addr *", "struct ovpn_kpeer *"); 253 254 static const char ovpnname[] = "ovpn"; 255 static const char ovpngroupname[] = "openvpn"; 256 257 static MALLOC_DEFINE(M_OVPN, ovpnname, "OpenVPN DCO Interface"); 258 #define MTAG_OVPN_LOOP 0x6f76706e /* ovpn */ 259 260 SYSCTL_DECL(_net_link); 261 static SYSCTL_NODE(_net_link, IFT_OTHER, openvpn, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 262 "OpenVPN DCO Interface"); 263 VNET_DEFINE_STATIC(int, replay_protection) = 0; 264 #define V_replay_protection VNET(replay_protection) 265 SYSCTL_INT(_net_link_openvpn, OID_AUTO, replay_protection, CTLFLAG_VNET | CTLFLAG_RW, 266 &VNET_NAME(replay_protection), 0, "Validate sequence numbers"); 267 268 VNET_DEFINE_STATIC(int, async_crypto); 269 #define V_async_crypto VNET(async_crypto) 270 SYSCTL_INT(_net_link_openvpn, OID_AUTO, async_crypto, 271 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_crypto), 0, 272 "Use asynchronous mode to parallelize crypto jobs."); 273 274 VNET_DEFINE_STATIC(int, async_netisr_queue); 275 #define V_async_netisr_queue VNET(async_netisr_queue) 276 SYSCTL_INT(_net_link_openvpn, OID_AUTO, netisr_queue, 277 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_netisr_queue), 0, 278 "Use netisr_queue() rather than netisr_dispatch()."); 279 280 static int 281 ovpn_peer_compare(struct ovpn_kpeer *a, struct ovpn_kpeer *b) 282 { 283 return (a->peerid - b->peerid); 284 } 285 286 static struct ovpn_kpeer * 287 ovpn_find_peer(struct ovpn_softc *sc, uint32_t peerid) 288 { 289 struct ovpn_kpeer p; 290 291 OVPN_ASSERT(sc); 292 293 p.peerid = peerid; 294 295 return (RB_FIND(ovpn_kpeers, &sc->peers, &p)); 296 } 297 298 static struct ovpn_kpeer * 299 ovpn_find_only_peer(struct ovpn_softc *sc) 300 { 301 OVPN_ASSERT(sc); 302 303 return (RB_ROOT(&sc->peers)); 304 } 305 306 static uint16_t 307 ovpn_get_port(struct sockaddr_storage *s) 308 { 309 switch (s->ss_family) { 310 case AF_INET: { 311 struct sockaddr_in *in = (struct sockaddr_in *)s; 312 return (in->sin_port); 313 } 314 case AF_INET6: { 315 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)s; 316 return (in6->sin6_port); 317 } 318 default: 319 panic("Unsupported address family %d", s->ss_family); 320 } 321 } 322 323 static int 324 ovpn_nvlist_to_sockaddr(const nvlist_t *nvl, struct sockaddr_storage *sa) 325 { 326 int af; 327 328 if (! nvlist_exists_number(nvl, "af")) 329 return (EINVAL); 330 if (! nvlist_exists_binary(nvl, "address")) 331 return (EINVAL); 332 if (! nvlist_exists_number(nvl, "port")) 333 return (EINVAL); 334 335 af = nvlist_get_number(nvl, "af"); 336 337 switch (af) { 338 #ifdef INET 339 case AF_INET: { 340 struct sockaddr_in *in = (struct sockaddr_in *)sa; 341 size_t len; 342 const void *addr = nvlist_get_binary(nvl, "address", &len); 343 in->sin_family = af; 344 if (len != sizeof(in->sin_addr)) 345 return (EINVAL); 346 347 memcpy(&in->sin_addr, addr, sizeof(in->sin_addr)); 348 in->sin_port = nvlist_get_number(nvl, "port"); 349 break; 350 } 351 #endif 352 #ifdef INET6 353 case AF_INET6: { 354 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)sa; 355 size_t len; 356 const void *addr = nvlist_get_binary(nvl, "address", &len); 357 in6->sin6_family = af; 358 if (len != sizeof(in6->sin6_addr)) 359 return (EINVAL); 360 361 memcpy(&in6->sin6_addr, addr, sizeof(in6->sin6_addr)); 362 in6->sin6_port = nvlist_get_number(nvl, "port"); 363 break; 364 } 365 #endif 366 default: 367 return (EINVAL); 368 } 369 370 return (0); 371 } 372 373 static bool 374 ovpn_has_peers(struct ovpn_softc *sc) 375 { 376 OVPN_ASSERT(sc); 377 378 return (sc->peercount > 0); 379 } 380 381 static void 382 ovpn_rele_so(struct ovpn_softc *sc) 383 { 384 bool has_peers; 385 386 OVPN_WASSERT(sc); 387 388 if (sc->so == NULL) 389 return; 390 391 has_peers = ovpn_has_peers(sc); 392 393 if (! has_peers) { 394 MPASS(sc->peercount == 0); 395 } else { 396 MPASS(sc->peercount > 0); 397 } 398 } 399 400 static void 401 ovpn_notify_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 402 { 403 struct ovpn_notification *n; 404 405 OVPN_WASSERT(sc); 406 407 n = malloc(sizeof(*n), M_OVPN, M_NOWAIT); 408 if (n == NULL) 409 return; 410 411 n->peerid = peer->peerid; 412 n->type = OVPN_NOTIF_DEL_PEER; 413 n->del_reason = peer->del_reason; 414 415 n->counters.pkt_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_in)); 416 n->counters.pkt_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_out)); 417 n->counters.bytes_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_in)); 418 n->counters.bytes_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_out)); 419 420 if (buf_ring_enqueue(sc->notifring, n) != 0) { 421 free(n, M_OVPN); 422 } else if (sc->so != NULL) { 423 /* Wake up userspace */ 424 sc->so->so_error = EAGAIN; 425 sorwakeup(sc->so); 426 sowwakeup(sc->so); 427 } 428 } 429 430 static void 431 ovpn_notify_key_rotation(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 432 { 433 struct ovpn_notification *n; 434 435 n = malloc(sizeof(*n), M_OVPN, M_NOWAIT | M_ZERO); 436 if (n == NULL) 437 return; 438 439 n->peerid = peer->peerid; 440 n->type = OVPN_NOTIF_ROTATE_KEY; 441 442 if (buf_ring_enqueue(sc->notifring, n) != 0) { 443 free(n, M_OVPN); 444 } else if (sc->so != NULL) { 445 /* Wake up userspace */ 446 sc->so->so_error = EAGAIN; 447 sorwakeup(sc->so); 448 sowwakeup(sc->so); 449 } 450 } 451 452 static void 453 ovpn_peer_release_ref(struct ovpn_kpeer *peer, bool locked) 454 { 455 struct ovpn_softc *sc; 456 457 CURVNET_ASSERT_SET(); 458 459 atomic_add_int(&peer->refcount, -1); 460 461 if (atomic_load_int(&peer->refcount) > 0) 462 return; 463 464 sc = peer->sc; 465 466 if (! locked) { 467 OVPN_WLOCK(sc); 468 469 /* Might have changed before we acquired the lock. */ 470 if (atomic_load_int(&peer->refcount) > 0) { 471 OVPN_WUNLOCK(sc); 472 return; 473 } 474 } 475 476 OVPN_ASSERT(sc); 477 478 /* The peer should have been removed from the list already. */ 479 MPASS(ovpn_find_peer(sc, peer->peerid) == NULL); 480 481 ovpn_notify_del_peer(sc, peer); 482 483 for (int i = 0; i < 2; i++) { 484 ovpn_free_kkey_dir(peer->keys[i].encrypt); 485 ovpn_free_kkey_dir(peer->keys[i].decrypt); 486 } 487 488 ovpn_rele_so(sc); 489 490 callout_stop(&peer->ping_send); 491 callout_stop(&peer->ping_rcv); 492 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 493 free(peer, M_OVPN); 494 495 if (! locked) 496 OVPN_WUNLOCK(sc); 497 } 498 499 static int 500 ovpn_new_peer(struct ifnet *ifp, const nvlist_t *nvl) 501 { 502 #ifdef INET6 503 struct epoch_tracker et; 504 #endif 505 struct sockaddr_storage remote; 506 struct ovpn_kpeer *peer = NULL; 507 struct file *fp = NULL; 508 struct ovpn_softc *sc = ifp->if_softc; 509 struct thread *td = curthread; 510 struct socket *so = NULL; 511 int fd; 512 uint32_t peerid; 513 int ret = 0; 514 515 if (nvl == NULL) 516 return (EINVAL); 517 518 if (! nvlist_exists_number(nvl, "peerid")) 519 return (EINVAL); 520 521 if (! nvlist_exists_number(nvl, "fd")) 522 return (EINVAL); 523 524 if (! nvlist_exists_nvlist(nvl, "remote")) 525 return (EINVAL); 526 527 peerid = nvlist_get_number(nvl, "peerid"); 528 529 ret = ovpn_nvlist_to_sockaddr(nvlist_get_nvlist(nvl, "remote"), 530 &remote); 531 if (ret != 0) 532 return (ret); 533 534 fd = nvlist_get_number(nvl, "fd"); 535 536 /* Look up the userspace process and use the fd to find the socket. */ 537 ret = getsock(td, fd, &cap_connect_rights, &fp); 538 if (ret != 0) 539 return (ret); 540 541 so = fp->f_data; 542 543 peer = malloc(sizeof(*peer), M_OVPN, M_WAITOK | M_ZERO); 544 peer->peerid = peerid; 545 peer->sc = sc; 546 peer->refcount = 1; 547 peer->last_active = uma_zalloc_pcpu(pcpu_zone_4, M_WAITOK | M_ZERO); 548 COUNTER_ARRAY_ALLOC(peer->counters, OVPN_PEER_COUNTER_SIZE, M_WAITOK); 549 550 if (nvlist_exists_binary(nvl, "vpn_ipv4")) { 551 size_t len; 552 const void *addr = nvlist_get_binary(nvl, "vpn_ipv4", &len); 553 if (len != sizeof(peer->vpn4)) { 554 ret = EINVAL; 555 goto error; 556 } 557 memcpy(&peer->vpn4, addr, len); 558 } 559 560 if (nvlist_exists_binary(nvl, "vpn_ipv6")) { 561 size_t len; 562 const void *addr = nvlist_get_binary(nvl, "vpn_ipv6", &len); 563 if (len != sizeof(peer->vpn6)) { 564 ret = EINVAL; 565 goto error; 566 } 567 memcpy(&peer->vpn6, addr, len); 568 } 569 570 callout_init_rm(&peer->ping_send, &sc->lock, CALLOUT_SHAREDLOCK); 571 callout_init_rm(&peer->ping_rcv, &sc->lock, 0); 572 573 peer->local.ss_len = sizeof(peer->local); 574 ret = sosockaddr(so, (struct sockaddr *)&peer->local); 575 if (ret) 576 goto error; 577 578 if (ovpn_get_port(&peer->local) == 0) { 579 ret = EINVAL; 580 goto error; 581 } 582 if (peer->local.ss_family != remote.ss_family) { 583 ret = EINVAL; 584 goto error; 585 } 586 587 memcpy(&peer->remote, &remote, sizeof(remote)); 588 589 if (peer->local.ss_family == AF_INET6 && 590 IN6_IS_ADDR_V4MAPPED(&TO_IN6(&peer->remote)->sin6_addr)) { 591 /* V4 mapped address, so treat this as v4, not v6. */ 592 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->local); 593 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->remote); 594 } 595 596 #ifdef INET6 597 if (peer->local.ss_family == AF_INET6 && 598 IN6_IS_ADDR_UNSPECIFIED(&TO_IN6(&peer->local)->sin6_addr)) { 599 NET_EPOCH_ENTER(et); 600 ret = in6_selectsrc_addr(curthread->td_proc->p_fibnum, 601 &TO_IN6(&peer->remote)->sin6_addr, 602 0, NULL, &TO_IN6(&peer->local)->sin6_addr, NULL); 603 NET_EPOCH_EXIT(et); 604 if (ret != 0) { 605 goto error; 606 } 607 } 608 #endif 609 OVPN_WLOCK(sc); 610 611 /* Disallow peer id re-use. */ 612 if (ovpn_find_peer(sc, peerid) != NULL) { 613 ret = EEXIST; 614 goto error_locked; 615 } 616 617 /* Make sure this is really a UDP socket. */ 618 if (so->so_type != SOCK_DGRAM || so->so_proto->pr_type != SOCK_DGRAM) { 619 ret = EPROTOTYPE; 620 goto error_locked; 621 } 622 623 /* Must be the same socket as for other peers on this interface. */ 624 if (sc->so != NULL && so != sc->so) 625 goto error_locked; 626 627 if (sc->so == NULL) { 628 sc->so = so; 629 /* 630 * Maintain one extra ref so the socket doesn't go away until 631 * we're destroying the ifp. 632 */ 633 soref(sc->so); 634 } 635 636 /* Insert the peer into the list. */ 637 RB_INSERT(ovpn_kpeers, &sc->peers, peer); 638 sc->peercount++; 639 640 OVPN_WUNLOCK(sc); 641 ret = udp_set_kernel_tunneling(sc->so, ovpn_udp_input, NULL, sc); 642 MPASS(ret == 0 || ret == EBUSY); 643 ret = 0; 644 645 goto done; 646 647 error_locked: 648 OVPN_WUNLOCK(sc); 649 error: 650 COUNTER_ARRAY_FREE(peer->counters, OVPN_PEER_COUNTER_SIZE); 651 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 652 free(peer, M_OVPN); 653 done: 654 if (fp != NULL) 655 fdrop(fp, td); 656 657 return (ret); 658 } 659 660 static int 661 _ovpn_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 662 { 663 struct ovpn_kpeer *tmp __diagused; 664 665 OVPN_WASSERT(sc); 666 CURVNET_ASSERT_SET(); 667 668 MPASS(RB_FIND(ovpn_kpeers, &sc->peers, peer) == peer); 669 670 tmp = RB_REMOVE(ovpn_kpeers, &sc->peers, peer); 671 MPASS(tmp != NULL); 672 673 sc->peercount--; 674 675 ovpn_peer_release_ref(peer, true); 676 677 return (0); 678 } 679 680 static int 681 ovpn_del_peer(struct ifnet *ifp, nvlist_t *nvl) 682 { 683 struct ovpn_softc *sc = ifp->if_softc; 684 struct ovpn_kpeer *peer; 685 uint32_t peerid; 686 int ret; 687 688 OVPN_WASSERT(sc); 689 690 if (nvl == NULL) 691 return (EINVAL); 692 693 if (! nvlist_exists_number(nvl, "peerid")) 694 return (EINVAL); 695 696 peerid = nvlist_get_number(nvl, "peerid"); 697 698 peer = ovpn_find_peer(sc, peerid); 699 if (peer == NULL) 700 return (ENOENT); 701 702 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 703 ret = _ovpn_del_peer(sc, peer); 704 705 return (ret); 706 } 707 708 static int 709 ovpn_create_kkey_dir(struct ovpn_kkey_dir **kdirp, 710 const nvlist_t *nvl) 711 { 712 struct crypto_session_params csp; 713 struct ovpn_kkey_dir *kdir; 714 const char *ciphername; 715 enum ovpn_key_cipher cipher; 716 const void *key, *iv; 717 size_t keylen = 0, ivlen = 0; 718 int error; 719 720 if (! nvlist_exists_string(nvl, "cipher")) 721 return (EINVAL); 722 ciphername = nvlist_get_string(nvl, "cipher"); 723 724 if (strcmp(ciphername, "none") == 0) 725 cipher = OVPN_CIPHER_ALG_NONE; 726 else if (strcmp(ciphername, "AES-256-GCM") == 0 || 727 strcmp(ciphername, "AES-192-GCM") == 0 || 728 strcmp(ciphername, "AES-128-GCM") == 0) 729 cipher = OVPN_CIPHER_ALG_AES_GCM; 730 else if (strcmp(ciphername, "CHACHA20-POLY1305") == 0) 731 cipher = OVPN_CIPHER_ALG_CHACHA20_POLY1305; 732 else 733 return (EINVAL); 734 735 if (cipher != OVPN_CIPHER_ALG_NONE) { 736 if (! nvlist_exists_binary(nvl, "key")) 737 return (EINVAL); 738 key = nvlist_get_binary(nvl, "key", &keylen); 739 if (keylen > sizeof(kdir->key)) 740 return (E2BIG); 741 742 if (! nvlist_exists_binary(nvl, "iv")) 743 return (EINVAL); 744 iv = nvlist_get_binary(nvl, "iv", &ivlen); 745 if (ivlen != 8) 746 return (E2BIG); 747 } 748 749 kdir = malloc(sizeof(struct ovpn_kkey_dir), M_OVPN, 750 M_WAITOK | M_ZERO); 751 752 kdir->cipher = cipher; 753 kdir->keylen = keylen; 754 kdir->tx_seq = 1; 755 memcpy(kdir->key, key, keylen); 756 kdir->noncelen = ivlen; 757 memcpy(kdir->nonce, iv, ivlen); 758 759 if (kdir->cipher != OVPN_CIPHER_ALG_NONE) { 760 /* Crypto init */ 761 bzero(&csp, sizeof(csp)); 762 csp.csp_mode = CSP_MODE_AEAD; 763 764 if (kdir->cipher == OVPN_CIPHER_ALG_CHACHA20_POLY1305) 765 csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305; 766 else 767 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 768 769 csp.csp_flags |= CSP_F_SEPARATE_AAD; 770 771 csp.csp_cipher_klen = kdir->keylen; 772 csp.csp_cipher_key = kdir->key; 773 csp.csp_ivlen = 96 / 8; 774 775 error = crypto_newsession(&kdir->cryptoid, &csp, 776 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE); 777 if (error) { 778 free(kdir, M_OVPN); 779 return (error); 780 } 781 } 782 783 mtx_init(&kdir->replay_mtx, "if_ovpn rx replay", NULL, MTX_DEF); 784 *kdirp = kdir; 785 786 return (0); 787 } 788 789 static void 790 ovpn_free_kkey_dir(struct ovpn_kkey_dir *kdir) 791 { 792 if (kdir == NULL) 793 return; 794 795 mtx_destroy(&kdir->replay_mtx); 796 797 crypto_freesession(kdir->cryptoid); 798 free(kdir, M_OVPN); 799 } 800 801 static int 802 ovpn_set_key(struct ifnet *ifp, const nvlist_t *nvl) 803 { 804 struct ovpn_softc *sc = ifp->if_softc; 805 struct ovpn_kkey_dir *enc, *dec; 806 struct ovpn_kpeer *peer; 807 int slot, keyid, peerid; 808 int error; 809 810 if (nvl == NULL) 811 return (EINVAL); 812 813 if (! nvlist_exists_number(nvl, "slot")) 814 return (EINVAL); 815 slot = nvlist_get_number(nvl, "slot"); 816 817 if (! nvlist_exists_number(nvl, "keyid")) 818 return (EINVAL); 819 keyid = nvlist_get_number(nvl, "keyid"); 820 821 if (! nvlist_exists_number(nvl, "peerid")) 822 return (EINVAL); 823 peerid = nvlist_get_number(nvl, "peerid"); 824 825 if (slot != OVPN_KEY_SLOT_PRIMARY && 826 slot != OVPN_KEY_SLOT_SECONDARY) 827 return (EINVAL); 828 829 if (! nvlist_exists_nvlist(nvl, "encrypt") || 830 ! nvlist_exists_nvlist(nvl, "decrypt")) 831 return (EINVAL); 832 833 error = ovpn_create_kkey_dir(&enc, nvlist_get_nvlist(nvl, "encrypt")); 834 if (error) 835 return (error); 836 837 error = ovpn_create_kkey_dir(&dec, nvlist_get_nvlist(nvl, "decrypt")); 838 if (error) { 839 ovpn_free_kkey_dir(enc); 840 return (error); 841 } 842 843 OVPN_WLOCK(sc); 844 845 peer = ovpn_find_peer(sc, peerid); 846 if (peer == NULL) { 847 ovpn_free_kkey_dir(dec); 848 ovpn_free_kkey_dir(enc); 849 OVPN_WUNLOCK(sc); 850 return (ENOENT); 851 } 852 853 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 854 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 855 856 peer->keys[slot].encrypt = enc; 857 peer->keys[slot].decrypt = dec; 858 859 peer->keys[slot].keyid = keyid; 860 peer->keys[slot].peerid = peerid; 861 862 OVPN_WUNLOCK(sc); 863 864 return (0); 865 } 866 867 static int 868 ovpn_check_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, enum ovpn_key_slot slot) 869 { 870 OVPN_ASSERT(sc); 871 872 if (peer->keys[slot].encrypt == NULL) 873 return (ENOLINK); 874 875 if (peer->keys[slot].decrypt == NULL) 876 return (ENOLINK); 877 878 return (0); 879 } 880 881 static int 882 ovpn_start(struct ifnet *ifp) 883 { 884 struct ovpn_softc *sc = ifp->if_softc; 885 886 OVPN_WLOCK(sc); 887 888 ifp->if_flags |= IFF_UP; 889 ifp->if_drv_flags |= IFF_DRV_RUNNING; 890 if_link_state_change(ifp, LINK_STATE_UP); 891 892 OVPN_WUNLOCK(sc); 893 894 return (0); 895 } 896 897 static int 898 ovpn_swap_keys(struct ifnet *ifp, nvlist_t *nvl) 899 { 900 struct ovpn_softc *sc = ifp->if_softc; 901 struct ovpn_kpeer *peer; 902 struct ovpn_kkey tmpkey; 903 int error; 904 905 if (nvl == NULL) 906 return (EINVAL); 907 908 if (! nvlist_exists_number(nvl, "peerid")) 909 return (EINVAL); 910 911 OVPN_WLOCK(sc); 912 913 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 914 if (peer == NULL) { 915 OVPN_WUNLOCK(sc); 916 return (ENOENT); 917 } 918 919 /* Check that we have a second key to swap to. */ 920 error = ovpn_check_key(sc, peer, OVPN_KEY_SLOT_SECONDARY); 921 if (error) { 922 OVPN_WUNLOCK(sc); 923 return (error); 924 } 925 926 tmpkey = peer->keys[0]; 927 peer->keys[0] = peer->keys[1]; 928 peer->keys[1] = tmpkey; 929 930 OVPN_WUNLOCK(sc); 931 932 return (0); 933 } 934 935 static int 936 ovpn_del_key(struct ifnet *ifp, const nvlist_t *nvl) 937 { 938 enum ovpn_key_slot slot; 939 struct ovpn_kpeer *peer; 940 struct ovpn_softc *sc = ifp->if_softc; 941 942 if (nvl == NULL) 943 return (EINVAL); 944 945 if (! nvlist_exists_number(nvl, "peerid")) 946 return (EINVAL); 947 948 if (! nvlist_exists_number(nvl, "slot")) 949 return (EINVAL); 950 slot = nvlist_get_number(nvl, "slot"); 951 952 if (slot != OVPN_KEY_SLOT_PRIMARY && 953 slot != OVPN_KEY_SLOT_SECONDARY) 954 return (EINVAL); 955 956 OVPN_WLOCK(sc); 957 958 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 959 if (peer == NULL) { 960 OVPN_WUNLOCK(sc); 961 return (ENOENT); 962 } 963 964 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 965 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 966 967 peer->keys[slot].encrypt = NULL; 968 peer->keys[slot].decrypt = NULL; 969 970 peer->keys[slot].keyid = 0; 971 peer->keys[slot].peerid = 0; 972 973 OVPN_WUNLOCK(sc); 974 975 return (0); 976 } 977 978 static void 979 ovpn_send_ping(void *arg) 980 { 981 static const uint8_t ping_str[] = { 982 0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb, 983 0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48 984 }; 985 986 struct epoch_tracker et; 987 struct ovpn_kpeer *peer = arg; 988 struct ovpn_softc *sc = peer->sc; 989 struct mbuf *m; 990 991 OVPN_RASSERT(sc); 992 993 /* Ensure we repeat! */ 994 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 995 ovpn_send_ping, peer); 996 997 m = m_get2(sizeof(ping_str), M_NOWAIT, MT_DATA, M_PKTHDR); 998 if (m == NULL) 999 return; 1000 1001 m_copyback(m, 0, sizeof(ping_str), ping_str); 1002 m->m_len = m->m_pkthdr.len = sizeof(ping_str); 1003 1004 CURVNET_SET(sc->ifp->if_vnet); 1005 NET_EPOCH_ENTER(et); 1006 (void)ovpn_transmit_to_peer(sc->ifp, m, peer, NULL); 1007 NET_EPOCH_EXIT(et); 1008 CURVNET_RESTORE(); 1009 } 1010 1011 static void 1012 ovpn_timeout(void *arg) 1013 { 1014 struct ovpn_kpeer *peer = arg; 1015 struct ovpn_softc *sc = peer->sc; 1016 uint32_t last, _last_active; 1017 int ret __diagused; 1018 int cpu; 1019 1020 OVPN_WASSERT(sc); 1021 1022 last = 0; 1023 CPU_FOREACH(cpu) { 1024 _last_active = *zpcpu_get_cpu(peer->last_active, cpu); 1025 if (_last_active > last) 1026 last = _last_active; 1027 } 1028 1029 if (last + peer->keepalive.timeout > time_uptime) { 1030 callout_reset(&peer->ping_rcv, 1031 (peer->keepalive.timeout - (time_uptime - last)) * hz, 1032 ovpn_timeout, peer); 1033 return; 1034 } 1035 1036 CURVNET_SET(sc->ifp->if_vnet); 1037 peer->del_reason = OVPN_DEL_REASON_TIMEOUT; 1038 ret = _ovpn_del_peer(sc, peer); 1039 MPASS(ret == 0); 1040 CURVNET_RESTORE(); 1041 } 1042 1043 static int 1044 ovpn_set_peer(struct ifnet *ifp, const nvlist_t *nvl) 1045 { 1046 struct ovpn_softc *sc = ifp->if_softc; 1047 struct ovpn_kpeer *peer; 1048 1049 if (nvl == NULL) 1050 return (EINVAL); 1051 1052 if (! nvlist_exists_number(nvl, "interval") || 1053 ! nvlist_exists_number(nvl, "timeout") || 1054 ! nvlist_exists_number(nvl, "peerid")) 1055 return (EINVAL); 1056 1057 OVPN_WLOCK(sc); 1058 1059 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 1060 if (peer == NULL) { 1061 OVPN_WUNLOCK(sc); 1062 return (ENOENT); 1063 } 1064 1065 peer->keepalive.interval = nvlist_get_number(nvl, "interval"); 1066 peer->keepalive.timeout = nvlist_get_number(nvl, "timeout"); 1067 1068 if (peer->keepalive.interval > 0) 1069 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 1070 ovpn_send_ping, peer); 1071 if (peer->keepalive.timeout > 0) 1072 callout_reset(&peer->ping_rcv, peer->keepalive.timeout * hz, 1073 ovpn_timeout, peer); 1074 1075 OVPN_WUNLOCK(sc); 1076 1077 return (0); 1078 } 1079 1080 static int 1081 ovpn_set_ifmode(struct ifnet *ifp, const nvlist_t *nvl) 1082 { 1083 struct ovpn_softc *sc = ifp->if_softc; 1084 int ifmode; 1085 1086 if (nvl == NULL) 1087 return (EINVAL); 1088 1089 if (! nvlist_exists_number(nvl, "ifmode") ) 1090 return (EINVAL); 1091 1092 ifmode = nvlist_get_number(nvl, "ifmode"); 1093 1094 OVPN_WLOCK(sc); 1095 1096 /* deny this if UP */ 1097 if (ifp->if_flags & IFF_UP) { 1098 OVPN_WUNLOCK(sc); 1099 return (EBUSY); 1100 } 1101 1102 switch (ifmode & ~IFF_MULTICAST) { 1103 case IFF_POINTOPOINT: 1104 case IFF_BROADCAST: 1105 ifp->if_flags &= 1106 ~(IFF_BROADCAST|IFF_POINTOPOINT|IFF_MULTICAST); 1107 ifp->if_flags |= ifmode; 1108 break; 1109 default: 1110 OVPN_WUNLOCK(sc); 1111 return (EINVAL); 1112 } 1113 1114 OVPN_WUNLOCK(sc); 1115 1116 return (0); 1117 } 1118 1119 static int 1120 ovpn_ioctl_set(struct ifnet *ifp, struct ifdrv *ifd) 1121 { 1122 struct ovpn_softc *sc = ifp->if_softc; 1123 uint8_t *buf = NULL; 1124 nvlist_t *nvl = NULL; 1125 int ret; 1126 1127 if (ifd->ifd_len != 0) { 1128 if (ifd->ifd_len > OVPN_MAX_REQUEST_SIZE) 1129 return (E2BIG); 1130 1131 buf = malloc(ifd->ifd_len, M_OVPN, M_WAITOK); 1132 1133 ret = copyin(ifd->ifd_data, buf, ifd->ifd_len); 1134 if (ret != 0) { 1135 free(buf, M_OVPN); 1136 return (ret); 1137 } 1138 1139 nvl = nvlist_unpack(buf, ifd->ifd_len, 0); 1140 free(buf, M_OVPN); 1141 if (nvl == NULL) { 1142 return (EINVAL); 1143 } 1144 } 1145 1146 switch (ifd->ifd_cmd) { 1147 case OVPN_NEW_PEER: 1148 ret = ovpn_new_peer(ifp, nvl); 1149 break; 1150 case OVPN_DEL_PEER: 1151 OVPN_WLOCK(sc); 1152 ret = ovpn_del_peer(ifp, nvl); 1153 OVPN_WUNLOCK(sc); 1154 break; 1155 case OVPN_NEW_KEY: 1156 ret = ovpn_set_key(ifp, nvl); 1157 break; 1158 case OVPN_START_VPN: 1159 ret = ovpn_start(ifp); 1160 break; 1161 case OVPN_SWAP_KEYS: 1162 ret = ovpn_swap_keys(ifp, nvl); 1163 break; 1164 case OVPN_DEL_KEY: 1165 ret = ovpn_del_key(ifp, nvl); 1166 break; 1167 case OVPN_SET_PEER: 1168 ret = ovpn_set_peer(ifp, nvl); 1169 break; 1170 case OVPN_SET_IFMODE: 1171 ret = ovpn_set_ifmode(ifp, nvl); 1172 break; 1173 default: 1174 ret = ENOTSUP; 1175 } 1176 1177 nvlist_destroy(nvl); 1178 return (ret); 1179 } 1180 1181 static int 1182 ovpn_add_counters(nvlist_t *parent, const char *name, counter_u64_t in, 1183 counter_u64_t out) 1184 { 1185 nvlist_t *nvl; 1186 1187 nvl = nvlist_create(0); 1188 if (nvl == NULL) 1189 return (ENOMEM); 1190 1191 nvlist_add_number(nvl, "in", counter_u64_fetch(in)); 1192 nvlist_add_number(nvl, "out", counter_u64_fetch(out)); 1193 1194 nvlist_add_nvlist(parent, name, nvl); 1195 1196 nvlist_destroy(nvl); 1197 1198 return (0); 1199 } 1200 1201 static int 1202 ovpn_get_stats(struct ovpn_softc *sc, nvlist_t **onvl) 1203 { 1204 nvlist_t *nvl; 1205 int ret; 1206 1207 nvl = nvlist_create(0); 1208 if (nvl == NULL) 1209 return (ENOMEM); 1210 1211 #define OVPN_COUNTER_OUT(name, in, out) \ 1212 do { \ 1213 ret = ovpn_add_counters(nvl, name, OVPN_COUNTER(sc, in), \ 1214 OVPN_COUNTER(sc, out)); \ 1215 if (ret != 0) \ 1216 goto error; \ 1217 } while(0) 1218 1219 OVPN_COUNTER_OUT("lost_ctrl", lost_ctrl_pkts_in, lost_ctrl_pkts_out); 1220 OVPN_COUNTER_OUT("lost_data", lost_data_pkts_in, lost_data_pkts_out); 1221 OVPN_COUNTER_OUT("nomem_data", nomem_data_pkts_in, 1222 nomem_data_pkts_out); 1223 OVPN_COUNTER_OUT("data", received_data_pkts, sent_data_pkts); 1224 OVPN_COUNTER_OUT("ctrl", received_ctrl_pkts, sent_ctrl_pkts); 1225 OVPN_COUNTER_OUT("tunnel", tunnel_bytes_received, 1226 tunnel_bytes_received); 1227 OVPN_COUNTER_OUT("transport", transport_bytes_received, 1228 transport_bytes_received); 1229 #undef OVPN_COUNTER_OUT 1230 1231 *onvl = nvl; 1232 1233 return (0); 1234 1235 error: 1236 nvlist_destroy(nvl); 1237 return (ret); 1238 } 1239 1240 static int 1241 ovpn_get_peer_stats(struct ovpn_softc *sc, nvlist_t **nvl) 1242 { 1243 struct ovpn_kpeer *peer; 1244 nvlist_t *nvpeer = NULL; 1245 int ret; 1246 1247 OVPN_RLOCK_TRACKER; 1248 1249 *nvl = nvlist_create(0); 1250 if (*nvl == NULL) 1251 return (ENOMEM); 1252 1253 #define OVPN_PEER_COUNTER_OUT(name, in, out) \ 1254 do { \ 1255 ret = ovpn_add_counters(nvpeer, name, \ 1256 OVPN_PEER_COUNTER(peer, in), OVPN_PEER_COUNTER(peer, out)); \ 1257 if (ret != 0) \ 1258 goto error; \ 1259 } while(0) 1260 1261 OVPN_RLOCK(sc); 1262 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1263 nvpeer = nvlist_create(0); 1264 if (nvpeer == NULL) { 1265 OVPN_RUNLOCK(sc); 1266 nvlist_destroy(*nvl); 1267 *nvl = NULL; 1268 return (ENOMEM); 1269 } 1270 1271 nvlist_add_number(nvpeer, "peerid", peer->peerid); 1272 1273 OVPN_PEER_COUNTER_OUT("packets", pkt_in, pkt_out); 1274 OVPN_PEER_COUNTER_OUT("bytes", bytes_in, bytes_out); 1275 1276 nvlist_append_nvlist_array(*nvl, "peers", nvpeer); 1277 nvlist_destroy(nvpeer); 1278 } 1279 #undef OVPN_PEER_COUNTER_OUT 1280 OVPN_RUNLOCK(sc); 1281 1282 return (0); 1283 1284 error: 1285 nvlist_destroy(nvpeer); 1286 nvlist_destroy(*nvl); 1287 *nvl = NULL; 1288 return (ret); 1289 } 1290 1291 static int 1292 ovpn_poll_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1293 { 1294 nvlist_t *nvl; 1295 1296 nvl = nvlist_create(0); 1297 if (nvl == NULL) 1298 return (ENOMEM); 1299 1300 nvlist_add_number(nvl, "pending", buf_ring_count(sc->notifring)); 1301 1302 *onvl = nvl; 1303 1304 return (0); 1305 } 1306 1307 static void 1308 ovpn_notif_add_counters(nvlist_t *parent, struct ovpn_notification *n) 1309 { 1310 nvlist_t *nvl; 1311 1312 nvl = nvlist_create(0); 1313 if (nvl == NULL) 1314 return; 1315 1316 nvlist_add_number(nvl, "in", n->counters.pkt_in); 1317 nvlist_add_number(nvl, "out", n->counters.pkt_out); 1318 1319 nvlist_add_nvlist(parent, "packets", nvl); 1320 nvlist_destroy(nvl); 1321 1322 nvl = nvlist_create(0); 1323 if (nvl == NULL) 1324 return; 1325 1326 nvlist_add_number(nvl, "in", n->counters.bytes_in); 1327 nvlist_add_number(nvl, "out", n->counters.bytes_out); 1328 1329 nvlist_add_nvlist(parent, "bytes", nvl); 1330 nvlist_destroy(nvl); 1331 } 1332 1333 static int 1334 opvn_get_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1335 { 1336 struct ovpn_notification *n; 1337 nvlist_t *nvl; 1338 1339 /* Check if we have notifications pending. */ 1340 n = buf_ring_dequeue_mc(sc->notifring); 1341 if (n == NULL) 1342 return (ENOENT); 1343 1344 nvl = nvlist_create(0); 1345 if (nvl == NULL) { 1346 free(n, M_OVPN); 1347 return (ENOMEM); 1348 } 1349 nvlist_add_number(nvl, "peerid", n->peerid); 1350 nvlist_add_number(nvl, "notification", n->type); 1351 if (n->type == OVPN_NOTIF_DEL_PEER) { 1352 nvlist_add_number(nvl, "del_reason", n->del_reason); 1353 1354 /* No error handling, because we want to send the notification 1355 * even if we can't attach the counters. */ 1356 ovpn_notif_add_counters(nvl, n); 1357 } 1358 free(n, M_OVPN); 1359 1360 *onvl = nvl; 1361 1362 return (0); 1363 } 1364 1365 static int 1366 ovpn_ioctl_get(struct ifnet *ifp, struct ifdrv *ifd) 1367 { 1368 struct ovpn_softc *sc = ifp->if_softc; 1369 nvlist_t *nvl = NULL; 1370 int error; 1371 1372 switch (ifd->ifd_cmd) { 1373 case OVPN_GET_STATS: 1374 error = ovpn_get_stats(sc, &nvl); 1375 break; 1376 case OVPN_GET_PEER_STATS: 1377 error = ovpn_get_peer_stats(sc, &nvl); 1378 break; 1379 case OVPN_POLL_PKT: 1380 error = ovpn_poll_pkt(sc, &nvl); 1381 break; 1382 case OVPN_GET_PKT: 1383 error = opvn_get_pkt(sc, &nvl); 1384 break; 1385 default: 1386 error = ENOTSUP; 1387 break; 1388 } 1389 1390 if (error == 0) { 1391 void *packed = NULL; 1392 size_t len; 1393 1394 MPASS(nvl != NULL); 1395 1396 packed = nvlist_pack(nvl, &len); 1397 if (! packed) { 1398 nvlist_destroy(nvl); 1399 return (ENOMEM); 1400 } 1401 1402 if (len > ifd->ifd_len) { 1403 free(packed, M_NVLIST); 1404 nvlist_destroy(nvl); 1405 return (ENOSPC); 1406 } 1407 1408 error = copyout(packed, ifd->ifd_data, len); 1409 ifd->ifd_len = len; 1410 1411 free(packed, M_NVLIST); 1412 nvlist_destroy(nvl); 1413 } 1414 1415 return (error); 1416 } 1417 1418 static int 1419 ovpn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 1420 { 1421 struct ifdrv *ifd; 1422 int error; 1423 1424 CURVNET_ASSERT_SET(); 1425 1426 switch (cmd) { 1427 case SIOCSDRVSPEC: 1428 case SIOCGDRVSPEC: 1429 error = priv_check(curthread, PRIV_NET_OVPN); 1430 if (error) 1431 return (error); 1432 break; 1433 } 1434 1435 switch (cmd) { 1436 case SIOCSDRVSPEC: 1437 ifd = (struct ifdrv *)data; 1438 error = ovpn_ioctl_set(ifp, ifd); 1439 break; 1440 case SIOCGDRVSPEC: 1441 ifd = (struct ifdrv *)data; 1442 error = ovpn_ioctl_get(ifp, ifd); 1443 break; 1444 case SIOCSIFMTU: { 1445 struct ifreq *ifr = (struct ifreq *)data; 1446 if (ifr->ifr_mtu < OVPN_MTU_MIN || ifr->ifr_mtu > OVPN_MTU_MAX) 1447 return (EINVAL); 1448 1449 ifp->if_mtu = ifr->ifr_mtu; 1450 return (0); 1451 } 1452 case SIOCSIFADDR: 1453 case SIOCADDMULTI: 1454 case SIOCDELMULTI: 1455 case SIOCGIFMTU: 1456 case SIOCSIFFLAGS: 1457 return (0); 1458 default: 1459 error = EINVAL; 1460 } 1461 1462 return (error); 1463 } 1464 1465 static int 1466 ovpn_encrypt_tx_cb(struct cryptop *crp) 1467 { 1468 struct epoch_tracker et; 1469 struct ovpn_kpeer *peer = crp->crp_opaque; 1470 struct ovpn_softc *sc = peer->sc; 1471 struct mbuf *m = crp->crp_buf.cb_mbuf; 1472 int tunnel_len; 1473 int ret; 1474 1475 CURVNET_SET(sc->ifp->if_vnet); 1476 NET_EPOCH_ENTER(et); 1477 1478 if (crp->crp_etype != 0) { 1479 crypto_freereq(crp); 1480 ovpn_peer_release_ref(peer, false); 1481 NET_EPOCH_EXIT(et); 1482 CURVNET_RESTORE(); 1483 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1484 m_freem(m); 1485 return (0); 1486 } 1487 1488 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1489 1490 tunnel_len = m->m_pkthdr.len - sizeof(struct ovpn_wire_header); 1491 ret = ovpn_encap(sc, peer->peerid, m); 1492 if (ret == 0) { 1493 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1494 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 1495 } 1496 1497 crypto_freereq(crp); 1498 ovpn_peer_release_ref(peer, false); 1499 1500 NET_EPOCH_EXIT(et); 1501 CURVNET_RESTORE(); 1502 1503 return (0); 1504 } 1505 1506 static void 1507 ovpn_finish_rx(struct ovpn_softc *sc, struct mbuf *m, 1508 struct ovpn_kpeer *peer, struct ovpn_kkey *key, uint32_t seq, 1509 struct rm_priotracker *_ovpn_lock_trackerp) 1510 { 1511 uint32_t af; 1512 1513 OVPN_RASSERT(sc); 1514 NET_EPOCH_ASSERT(); 1515 1516 /* Replay protection. */ 1517 if (V_replay_protection && ! ovpn_check_replay(key->decrypt, seq)) { 1518 OVPN_RUNLOCK(sc); 1519 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1520 m_freem(m); 1521 return; 1522 } 1523 1524 critical_enter(); 1525 *zpcpu_get(peer->last_active) = time_uptime; 1526 critical_exit(); 1527 1528 OVPN_RUNLOCK(sc); 1529 1530 OVPN_COUNTER_ADD(sc, received_data_pkts, 1); 1531 OVPN_COUNTER_ADD(sc, tunnel_bytes_received, m->m_pkthdr.len); 1532 OVPN_PEER_COUNTER_ADD(peer, pkt_in, 1); 1533 OVPN_PEER_COUNTER_ADD(peer, bytes_in, m->m_pkthdr.len); 1534 1535 /* Receive the packet on our interface. */ 1536 m->m_pkthdr.rcvif = sc->ifp; 1537 1538 /* Clear checksum flags in case the real hardware set them. */ 1539 m->m_pkthdr.csum_flags = 0; 1540 1541 /* Clear mbuf tags & flags */ 1542 m_tag_delete_nonpersistent(m); 1543 m_clrprotoflags(m); 1544 1545 /* Ensure we can read the first byte. */ 1546 m = m_pullup(m, 1); 1547 if (m == NULL) { 1548 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 1549 return; 1550 } 1551 1552 /* 1553 * Check for address family, and disregard any control packets (e.g. 1554 * keepalive). 1555 */ 1556 af = ovpn_get_af(m); 1557 if (af != 0) { 1558 BPF_MTAP2(sc->ifp, &af, sizeof(af), m); 1559 if (V_async_netisr_queue) 1560 netisr_queue(af == AF_INET ? NETISR_IP : NETISR_IPV6, m); 1561 else 1562 netisr_dispatch(af == AF_INET ? NETISR_IP : NETISR_IPV6, m); 1563 } else { 1564 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1565 m_freem(m); 1566 } 1567 } 1568 1569 static struct ovpn_kkey * 1570 ovpn_find_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, 1571 const struct ovpn_wire_header *ohdr) 1572 { 1573 struct ovpn_kkey *key = NULL; 1574 uint8_t keyid; 1575 1576 OVPN_RASSERT(sc); 1577 1578 keyid = (ntohl(ohdr->opcode) >> 24) & 0x07; 1579 1580 if (peer->keys[0].keyid == keyid) 1581 key = &peer->keys[0]; 1582 else if (peer->keys[1].keyid == keyid) 1583 key = &peer->keys[1]; 1584 1585 return (key); 1586 } 1587 1588 static int 1589 ovpn_decrypt_rx_cb(struct cryptop *crp) 1590 { 1591 struct epoch_tracker et; 1592 struct ovpn_softc *sc = crp->crp_opaque; 1593 struct mbuf *m = crp->crp_buf.cb_mbuf; 1594 struct ovpn_kkey *key; 1595 struct ovpn_kpeer *peer; 1596 struct ovpn_wire_header *ohdr; 1597 uint32_t peerid; 1598 1599 OVPN_RLOCK_TRACKER; 1600 1601 OVPN_RLOCK(sc); 1602 1603 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1604 1605 if (crp->crp_etype != 0) { 1606 crypto_freereq(crp); 1607 atomic_add_int(&sc->refcount, -1); 1608 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1609 OVPN_RUNLOCK(sc); 1610 m_freem(m); 1611 return (0); 1612 } 1613 1614 CURVNET_SET(sc->ifp->if_vnet); 1615 1616 ohdr = mtodo(m, sizeof(struct udphdr)); 1617 1618 peerid = ntohl(ohdr->opcode) & 0x00ffffff; 1619 peer = ovpn_find_peer(sc, peerid); 1620 if (peer == NULL) { 1621 /* No such peer. Drop packet. */ 1622 crypto_freereq(crp); 1623 atomic_add_int(&sc->refcount, -1); 1624 OVPN_RUNLOCK(sc); 1625 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1626 m_freem(m); 1627 CURVNET_RESTORE(); 1628 return (0); 1629 } 1630 1631 key = ovpn_find_key(sc, peer, ohdr); 1632 if (key == NULL) { 1633 crypto_freereq(crp); 1634 atomic_add_int(&sc->refcount, -1); 1635 /* 1636 * Has this key been removed between us starting the decrypt 1637 * and finishing it? 1638 */ 1639 OVPN_RUNLOCK(sc); 1640 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1641 m_freem(m); 1642 CURVNET_RESTORE(); 1643 return (0); 1644 } 1645 1646 /* Now remove the outer headers */ 1647 m_adj_decap(m, sizeof(struct udphdr) + 1648 sizeof(struct ovpn_wire_header)); 1649 1650 NET_EPOCH_ENTER(et); 1651 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), _ovpn_lock_trackerp); 1652 NET_EPOCH_EXIT(et); 1653 OVPN_UNLOCK_ASSERT(sc); 1654 1655 CURVNET_RESTORE(); 1656 1657 crypto_freereq(crp); 1658 atomic_add_int(&sc->refcount, -1); 1659 1660 return (0); 1661 } 1662 1663 static int 1664 ovpn_get_af(struct mbuf *m) 1665 { 1666 struct ip *ip; 1667 struct ip6_hdr *ip6; 1668 1669 /* 1670 * We should pullup, but we're only interested in the first byte, so 1671 * that'll always be contiguous. 1672 */ 1673 ip = mtod(m, struct ip *); 1674 if (ip->ip_v == IPVERSION) 1675 return (AF_INET); 1676 1677 ip6 = mtod(m, struct ip6_hdr *); 1678 if ((ip6->ip6_vfc & IPV6_VERSION_MASK) == IPV6_VERSION) 1679 return (AF_INET6); 1680 1681 return (0); 1682 } 1683 1684 #ifdef INET 1685 static struct ovpn_kpeer * 1686 ovpn_find_peer_by_ip(struct ovpn_softc *sc, const struct in_addr addr) 1687 { 1688 struct ovpn_kpeer *peer = NULL; 1689 1690 OVPN_ASSERT(sc); 1691 1692 /* TODO: Add a second RB so we can look up by IP. */ 1693 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1694 if (addr.s_addr == peer->vpn4.s_addr) 1695 return (peer); 1696 } 1697 1698 return (peer); 1699 } 1700 #endif 1701 1702 #ifdef INET6 1703 static struct ovpn_kpeer * 1704 ovpn_find_peer_by_ip6(struct ovpn_softc *sc, const struct in6_addr *addr) 1705 { 1706 struct ovpn_kpeer *peer = NULL; 1707 1708 OVPN_ASSERT(sc); 1709 1710 /* TODO: Add a third RB so we can look up by IPv6 address. */ 1711 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1712 if (memcmp(addr, &peer->vpn6, sizeof(*addr)) == 0) 1713 return (peer); 1714 } 1715 1716 return (peer); 1717 } 1718 #endif 1719 1720 static struct ovpn_kpeer * 1721 ovpn_route_peer(struct ovpn_softc *sc, struct mbuf **m0, 1722 const struct sockaddr *dst) 1723 { 1724 struct ovpn_kpeer *peer = NULL; 1725 int af; 1726 1727 NET_EPOCH_ASSERT(); 1728 OVPN_ASSERT(sc); 1729 1730 /* Shortcut if we're a client (or are a server and have only one client). */ 1731 if (sc->peercount == 1) 1732 return (ovpn_find_only_peer(sc)); 1733 1734 if (dst != NULL) 1735 af = dst->sa_family; 1736 else 1737 af = ovpn_get_af(*m0); 1738 1739 switch (af) { 1740 #ifdef INET 1741 case AF_INET: { 1742 const struct sockaddr_in *sa = (const struct sockaddr_in *)dst; 1743 struct nhop_object *nh; 1744 const struct in_addr *ip_dst; 1745 1746 if (sa != NULL) { 1747 ip_dst = &sa->sin_addr; 1748 } else { 1749 struct ip *ip; 1750 1751 *m0 = m_pullup(*m0, sizeof(struct ip)); 1752 if (*m0 == NULL) 1753 return (NULL); 1754 ip = mtod(*m0, struct ip *); 1755 ip_dst = &ip->ip_dst; 1756 } 1757 1758 peer = ovpn_find_peer_by_ip(sc, *ip_dst); 1759 SDT_PROBE2(if_ovpn, tx, route, ip4, ip_dst, peer); 1760 if (peer == NULL) { 1761 nh = fib4_lookup(M_GETFIB(*m0), *ip_dst, 0, 1762 NHR_NONE, 0); 1763 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1764 peer = ovpn_find_peer_by_ip(sc, 1765 nh->gw4_sa.sin_addr); 1766 SDT_PROBE2(if_ovpn, tx, route, ip4, 1767 &nh->gw4_sa.sin_addr, peer); 1768 } 1769 } 1770 break; 1771 } 1772 #endif 1773 #ifdef INET6 1774 case AF_INET6: { 1775 const struct sockaddr_in6 *sa6 = 1776 (const struct sockaddr_in6 *)dst; 1777 struct nhop_object *nh; 1778 const struct in6_addr *ip6_dst; 1779 1780 if (sa6 != NULL) { 1781 ip6_dst = &sa6->sin6_addr; 1782 } else { 1783 struct ip6_hdr *ip6; 1784 1785 *m0 = m_pullup(*m0, sizeof(struct ip6_hdr)); 1786 if (*m0 == NULL) 1787 return (NULL); 1788 ip6 = mtod(*m0, struct ip6_hdr *); 1789 ip6_dst = &ip6->ip6_dst; 1790 } 1791 1792 peer = ovpn_find_peer_by_ip6(sc, ip6_dst); 1793 SDT_PROBE2(if_ovpn, tx, route, ip6, ip6_dst, peer); 1794 if (peer == NULL) { 1795 nh = fib6_lookup(M_GETFIB(*m0), ip6_dst, 0, 1796 NHR_NONE, 0); 1797 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1798 peer = ovpn_find_peer_by_ip6(sc, 1799 &nh->gw6_sa.sin6_addr); 1800 SDT_PROBE2(if_ovpn, tx, route, ip6, 1801 &nh->gw6_sa.sin6_addr, peer); 1802 } 1803 } 1804 break; 1805 } 1806 #endif 1807 } 1808 1809 return (peer); 1810 } 1811 1812 static int 1813 ovpn_transmit(struct ifnet *ifp, struct mbuf *m) 1814 { 1815 return (ifp->if_output(ifp, m, NULL, NULL)); 1816 } 1817 1818 static int 1819 ovpn_transmit_to_peer(struct ifnet *ifp, struct mbuf *m, 1820 struct ovpn_kpeer *peer, struct rm_priotracker *_ovpn_lock_trackerp) 1821 { 1822 struct ovpn_wire_header *ohdr; 1823 struct ovpn_kkey *key; 1824 struct ovpn_softc *sc; 1825 struct cryptop *crp; 1826 uint32_t af, seq; 1827 uint64_t seq64; 1828 size_t len, ovpn_hdr_len; 1829 int tunnel_len; 1830 int ret; 1831 1832 sc = ifp->if_softc; 1833 1834 OVPN_RASSERT(sc); 1835 1836 tunnel_len = m->m_pkthdr.len; 1837 1838 key = &peer->keys[OVPN_KEY_SLOT_PRIMARY]; 1839 if (key->encrypt == NULL) { 1840 if (_ovpn_lock_trackerp != NULL) 1841 OVPN_RUNLOCK(sc); 1842 m_freem(m); 1843 return (ENOLINK); 1844 } 1845 1846 af = ovpn_get_af(m); 1847 /* Don't capture control packets. */ 1848 if (af != 0) 1849 BPF_MTAP2(ifp, &af, sizeof(af), m); 1850 1851 if (__predict_false(if_tunnel_check_nesting(ifp, m, MTAG_OVPN_LOOP, 3))) { 1852 if (_ovpn_lock_trackerp != NULL) 1853 OVPN_RUNLOCK(sc); 1854 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1855 m_freem(m); 1856 return (ELOOP); 1857 } 1858 1859 len = m->m_pkthdr.len; 1860 MPASS(len <= ifp->if_mtu); 1861 1862 ovpn_hdr_len = sizeof(struct ovpn_wire_header); 1863 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) 1864 ovpn_hdr_len -= 16; /* No auth tag. */ 1865 1866 M_PREPEND(m, ovpn_hdr_len, M_NOWAIT); 1867 if (m == NULL) { 1868 if (_ovpn_lock_trackerp != NULL) 1869 OVPN_RUNLOCK(sc); 1870 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1871 return (ENOBUFS); 1872 } 1873 ohdr = mtod(m, struct ovpn_wire_header *); 1874 ohdr->opcode = (OVPN_OP_DATA_V2 << OVPN_OP_SHIFT) | key->keyid; 1875 ohdr->opcode <<= 24; 1876 ohdr->opcode |= key->peerid; 1877 ohdr->opcode = htonl(ohdr->opcode); 1878 1879 seq64 = atomic_fetchadd_64(&peer->keys[OVPN_KEY_SLOT_PRIMARY].encrypt->tx_seq, 1); 1880 if (seq64 == OVPN_SEQ_ROTATE) { 1881 ovpn_notify_key_rotation(sc, peer); 1882 } else if (seq64 > UINT32_MAX) { 1883 /* We've wrapped, give up on this packet. */ 1884 if (_ovpn_lock_trackerp != NULL) 1885 OVPN_RUNLOCK(sc); 1886 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1887 1888 /* Let's avoid (very unlikely, but still) wraparounds of the 1889 * 64-bit counter taking us back to 0. */ 1890 atomic_store_64(&peer->keys[OVPN_KEY_SLOT_PRIMARY].encrypt->tx_seq, 1891 UINT32_MAX); 1892 1893 return (ENOBUFS); 1894 } 1895 1896 seq = htonl(seq64 & UINT32_MAX); 1897 ohdr->seq = seq; 1898 1899 OVPN_PEER_COUNTER_ADD(peer, pkt_out, 1); 1900 OVPN_PEER_COUNTER_ADD(peer, bytes_out, len); 1901 1902 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) { 1903 ret = ovpn_encap(sc, peer->peerid, m); 1904 if (_ovpn_lock_trackerp != NULL) 1905 OVPN_RUNLOCK(sc); 1906 if (ret == 0) { 1907 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1908 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 1909 } 1910 return (ret); 1911 } 1912 1913 crp = crypto_getreq(key->encrypt->cryptoid, M_NOWAIT); 1914 if (crp == NULL) { 1915 if (_ovpn_lock_trackerp != NULL) 1916 OVPN_RUNLOCK(sc); 1917 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1918 m_freem(m); 1919 return (ENOBUFS); 1920 } 1921 1922 /* Encryption covers only the payload, not the header. */ 1923 crp->crp_payload_start = sizeof(*ohdr); 1924 crp->crp_payload_length = len; 1925 crp->crp_op = CRYPTO_OP_ENCRYPT; 1926 1927 /* 1928 * AAD data covers the ovpn_wire_header minus the auth 1929 * tag. 1930 */ 1931 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 1932 crp->crp_aad = ohdr; 1933 crp->crp_aad_start = 0; 1934 crp->crp_op |= CRYPTO_OP_COMPUTE_DIGEST; 1935 crp->crp_digest_start = offsetof(struct ovpn_wire_header, auth_tag); 1936 1937 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 1938 memcpy(crp->crp_iv, &seq, sizeof(seq)); 1939 memcpy(crp->crp_iv + sizeof(seq), key->encrypt->nonce, 1940 key->encrypt->noncelen); 1941 1942 crypto_use_mbuf(crp, m); 1943 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 1944 crp->crp_callback = ovpn_encrypt_tx_cb; 1945 crp->crp_opaque = peer; 1946 1947 atomic_add_int(&peer->refcount, 1); 1948 if (_ovpn_lock_trackerp != NULL) 1949 OVPN_RUNLOCK(sc); 1950 if (V_async_crypto) 1951 ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); 1952 else 1953 ret = crypto_dispatch(crp); 1954 if (ret) { 1955 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1956 } 1957 1958 return (ret); 1959 } 1960 1961 /* 1962 * Note: Expects to hold the read lock on entry, and will release it itself. 1963 */ 1964 static int 1965 ovpn_encap(struct ovpn_softc *sc, uint32_t peerid, struct mbuf *m) 1966 { 1967 struct udphdr *udp; 1968 struct ovpn_kpeer *peer; 1969 int len; 1970 1971 OVPN_RLOCK_TRACKER; 1972 1973 OVPN_RLOCK(sc); 1974 NET_EPOCH_ASSERT(); 1975 1976 peer = ovpn_find_peer(sc, peerid); 1977 if (peer == NULL || sc->ifp->if_link_state != LINK_STATE_UP) { 1978 OVPN_RUNLOCK(sc); 1979 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1980 m_freem(m); 1981 return (ENETDOWN); 1982 } 1983 1984 len = m->m_pkthdr.len; 1985 1986 M_PREPEND(m, sizeof(struct udphdr), M_NOWAIT); 1987 if (m == NULL) { 1988 OVPN_RUNLOCK(sc); 1989 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1990 m_freem(m); 1991 return (ENOBUFS); 1992 } 1993 udp = mtod(m, struct udphdr *); 1994 1995 MPASS(peer->local.ss_family == peer->remote.ss_family); 1996 1997 udp->uh_sport = ovpn_get_port(&peer->local); 1998 udp->uh_dport = ovpn_get_port(&peer->remote); 1999 udp->uh_ulen = htons(sizeof(struct udphdr) + len); 2000 2001 switch (peer->remote.ss_family) { 2002 #ifdef INET 2003 case AF_INET: { 2004 struct sockaddr_in *in_local = TO_IN(&peer->local); 2005 struct sockaddr_in *in_remote = TO_IN(&peer->remote); 2006 struct ip *ip; 2007 2008 /* 2009 * This requires knowing the source IP, which we don't. Happily 2010 * we're allowed to keep this at 0, and the checksum won't do 2011 * anything the crypto won't already do. 2012 */ 2013 udp->uh_sum = 0; 2014 2015 /* Set the checksum flags so we recalculate checksums. */ 2016 m->m_pkthdr.csum_flags |= CSUM_IP; 2017 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 2018 2019 M_PREPEND(m, sizeof(struct ip), M_NOWAIT); 2020 if (m == NULL) { 2021 OVPN_RUNLOCK(sc); 2022 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2023 return (ENOBUFS); 2024 } 2025 ip = mtod(m, struct ip *); 2026 2027 ip->ip_tos = 0; 2028 ip->ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) + 2029 len); 2030 ip->ip_off = 0; 2031 ip->ip_ttl = V_ip_defttl; 2032 ip->ip_p = IPPROTO_UDP; 2033 ip->ip_sum = 0; 2034 if (in_local->sin_port != 0) 2035 ip->ip_src = in_local->sin_addr; 2036 else 2037 ip->ip_src.s_addr = INADDR_ANY; 2038 ip->ip_dst = in_remote->sin_addr; 2039 2040 OVPN_RUNLOCK(sc); 2041 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 2042 2043 return (ip_output(m, NULL, NULL, 0, NULL, NULL)); 2044 } 2045 #endif 2046 #ifdef INET6 2047 case AF_INET6: { 2048 struct sockaddr_in6 *in6_local = TO_IN6(&peer->local); 2049 struct sockaddr_in6 *in6_remote = TO_IN6(&peer->remote); 2050 struct ip6_hdr *ip6; 2051 2052 M_PREPEND(m, sizeof(struct ip6_hdr), M_NOWAIT); 2053 if (m == NULL) { 2054 OVPN_RUNLOCK(sc); 2055 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2056 return (ENOBUFS); 2057 } 2058 m = m_pullup(m, sizeof(*ip6) + sizeof(*udp)); 2059 if (m == NULL) { 2060 OVPN_RUNLOCK(sc); 2061 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2062 return (ENOBUFS); 2063 } 2064 2065 ip6 = mtod(m, struct ip6_hdr *); 2066 2067 ip6->ip6_vfc = IPV6_VERSION; 2068 ip6->ip6_flow &= ~IPV6_FLOWINFO_MASK; 2069 ip6->ip6_plen = htons(sizeof(*ip6) + sizeof(struct udphdr) + 2070 len); 2071 ip6->ip6_nxt = IPPROTO_UDP; 2072 ip6->ip6_hlim = V_ip6_defhlim; 2073 2074 memcpy(&ip6->ip6_src, &in6_local->sin6_addr, 2075 sizeof(ip6->ip6_src)); 2076 memcpy(&ip6->ip6_dst, &in6_remote->sin6_addr, 2077 sizeof(ip6->ip6_dst)); 2078 2079 udp = mtodo(m, sizeof(*ip6)); 2080 udp->uh_sum = in6_cksum_pseudo(ip6, 2081 m->m_pkthdr.len - sizeof(struct ip6_hdr), 2082 IPPROTO_UDP, 0); 2083 2084 m->m_pkthdr.csum_flags |= CSUM_UDP_IPV6; 2085 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 2086 2087 OVPN_RUNLOCK(sc); 2088 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 2089 2090 return (ip6_output(m, NULL, NULL, IPV6_UNSPECSRC, NULL, NULL, 2091 NULL)); 2092 } 2093 #endif 2094 default: 2095 panic("Unsupported address family %d", 2096 peer->remote.ss_family); 2097 } 2098 } 2099 2100 static int 2101 ovpn_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 2102 struct route *ro) 2103 { 2104 struct ovpn_softc *sc; 2105 struct ovpn_kpeer *peer; 2106 2107 OVPN_RLOCK_TRACKER; 2108 2109 sc = ifp->if_softc; 2110 2111 m = m_unshare(m, M_NOWAIT); 2112 if (m == NULL) { 2113 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2114 return (ENOBUFS); 2115 } 2116 2117 OVPN_RLOCK(sc); 2118 2119 SDT_PROBE1(if_ovpn, tx, transmit, start, m); 2120 2121 if (__predict_false(ifp->if_link_state != LINK_STATE_UP)) { 2122 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2123 OVPN_RUNLOCK(sc); 2124 m_freem(m); 2125 return (ENETDOWN); 2126 } 2127 2128 /** 2129 * Only obey 'dst' (i.e. the gateway) if no route is supplied. 2130 * That's our indication that we're being called through pf's route-to, 2131 * and we should route according to 'dst' instead. We can't do so 2132 * consistently, because the usual openvpn configuration sets the first 2133 * non-server IP in the subnet as the gateway. If we always use that 2134 * one we'd end up routing all traffic to the first client. 2135 * tl;dr: 'ro == NULL' tells us pf is doing a route-to, and then but 2136 * only then, we should treat 'dst' as the destination. */ 2137 peer = ovpn_route_peer(sc, &m, ro == NULL ? dst : NULL); 2138 if (peer == NULL) { 2139 /* No destination. */ 2140 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2141 OVPN_RUNLOCK(sc); 2142 m_freem(m); 2143 return (ENETDOWN); 2144 } 2145 2146 return (ovpn_transmit_to_peer(ifp, m, peer, _ovpn_lock_trackerp)); 2147 } 2148 2149 static bool 2150 ovpn_check_replay(struct ovpn_kkey_dir *key, uint32_t seq) 2151 { 2152 uint32_t d; 2153 2154 mtx_lock(&key->replay_mtx); 2155 2156 /* Sequence number must be strictly greater than rx_seq */ 2157 if (seq <= key->rx_seq) { 2158 mtx_unlock(&key->replay_mtx); 2159 return (false); 2160 } 2161 2162 /* Large jump. The packet authenticated okay, so just accept that. */ 2163 if (seq > (key->rx_seq + (sizeof(key->rx_window) * 8))) { 2164 key->rx_seq = seq; 2165 key->rx_window = 0; 2166 mtx_unlock(&key->replay_mtx); 2167 return (true); 2168 } 2169 2170 /* Happy case. */ 2171 if ((seq == key->rx_seq + 1) && key->rx_window == 0) { 2172 key->rx_seq++; 2173 mtx_unlock(&key->replay_mtx); 2174 return (true); 2175 } 2176 2177 d = seq - key->rx_seq - 1; 2178 2179 if (key->rx_window & ((uint64_t)1 << d)) { 2180 /* Dupe! */ 2181 mtx_unlock(&key->replay_mtx); 2182 return (false); 2183 } 2184 2185 key->rx_window |= (uint64_t)1 << d; 2186 2187 while (key->rx_window & 1) { 2188 key->rx_seq++; 2189 key->rx_window >>= 1; 2190 } 2191 2192 mtx_unlock(&key->replay_mtx); 2193 2194 return (true); 2195 } 2196 2197 static struct ovpn_kpeer * 2198 ovpn_peer_from_mbuf(struct ovpn_softc *sc, struct mbuf *m, int off) 2199 { 2200 struct ovpn_wire_header ohdr; 2201 uint32_t peerid; 2202 const size_t hdrlen = sizeof(ohdr) - sizeof(ohdr.auth_tag); 2203 2204 OVPN_RASSERT(sc); 2205 2206 if (m_length(m, NULL) < (off + sizeof(struct udphdr) + hdrlen)) 2207 return (NULL); 2208 2209 m_copydata(m, off + sizeof(struct udphdr), hdrlen, (caddr_t)&ohdr); 2210 2211 peerid = ntohl(ohdr.opcode) & 0x00ffffff; 2212 2213 return (ovpn_find_peer(sc, peerid)); 2214 } 2215 2216 static bool 2217 ovpn_udp_input(struct mbuf *m, int off, struct inpcb *inp, 2218 const struct sockaddr *sa, void *ctx) 2219 { 2220 struct ovpn_softc *sc = ctx; 2221 struct ovpn_wire_header tmphdr; 2222 struct ovpn_wire_header *ohdr; 2223 struct udphdr *uhdr; 2224 struct ovpn_kkey *key; 2225 struct cryptop *crp; 2226 struct ovpn_kpeer *peer; 2227 size_t ohdrlen; 2228 int ret; 2229 uint8_t op; 2230 2231 OVPN_RLOCK_TRACKER; 2232 2233 M_ASSERTPKTHDR(m); 2234 2235 m = m_unshare(m, M_NOWAIT); 2236 if (m == NULL) { 2237 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2238 return (true); 2239 } 2240 2241 OVPN_COUNTER_ADD(sc, transport_bytes_received, m->m_pkthdr.len - off); 2242 2243 ohdrlen = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2244 2245 OVPN_RLOCK(sc); 2246 2247 peer = ovpn_peer_from_mbuf(sc, m, off); 2248 if (peer == NULL) { 2249 OVPN_RUNLOCK(sc); 2250 return (false); 2251 } 2252 2253 if (m_length(m, NULL) < (off + sizeof(*uhdr) + ohdrlen)) { 2254 /* Short packet. */ 2255 OVPN_RUNLOCK(sc); 2256 return (false); 2257 } 2258 2259 m_copydata(m, off + sizeof(*uhdr), ohdrlen, (caddr_t)&tmphdr); 2260 2261 op = ntohl(tmphdr.opcode) >> 24 >> OVPN_OP_SHIFT; 2262 if (op != OVPN_OP_DATA_V2) { 2263 /* Control packet? */ 2264 OVPN_RUNLOCK(sc); 2265 return (false); 2266 } 2267 2268 m = m_pullup(m, off + sizeof(*uhdr) + ohdrlen); 2269 if (m == NULL) { 2270 OVPN_RUNLOCK(sc); 2271 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2272 return (true); 2273 } 2274 2275 /* 2276 * Simplify things by getting rid of the preceding headers, we don't 2277 * care about them. 2278 */ 2279 m_adj_decap(m, off); 2280 2281 uhdr = mtodo(m, 0); 2282 ohdr = mtodo(m, sizeof(*uhdr)); 2283 2284 key = ovpn_find_key(sc, peer, ohdr); 2285 if (key == NULL || key->decrypt == NULL) { 2286 OVPN_RUNLOCK(sc); 2287 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2288 m_freem(m); 2289 return (true); 2290 } 2291 2292 if (key->decrypt->cipher == OVPN_CIPHER_ALG_NONE) { 2293 /* Now remove the outer headers */ 2294 m_adj_decap(m, sizeof(struct udphdr) + ohdrlen); 2295 2296 ohdr = mtodo(m, sizeof(*uhdr)); 2297 2298 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), 2299 _ovpn_lock_trackerp); 2300 OVPN_UNLOCK_ASSERT(sc); 2301 return (true); 2302 } 2303 2304 ohdrlen += sizeof(ohdr->auth_tag); 2305 2306 m = m_pullup(m, sizeof(*uhdr) + ohdrlen); 2307 if (m == NULL) { 2308 OVPN_RUNLOCK(sc); 2309 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2310 return (true); 2311 } 2312 uhdr = mtodo(m, 0); 2313 ohdr = mtodo(m, sizeof(*uhdr)); 2314 2315 /* Decrypt */ 2316 crp = crypto_getreq(key->decrypt->cryptoid, M_NOWAIT); 2317 if (crp == NULL) { 2318 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2319 OVPN_RUNLOCK(sc); 2320 m_freem(m); 2321 return (true); 2322 } 2323 2324 crp->crp_payload_start = sizeof(struct udphdr) + sizeof(*ohdr); 2325 crp->crp_payload_length = ntohs(uhdr->uh_ulen) - 2326 sizeof(*uhdr) - sizeof(*ohdr); 2327 crp->crp_op = CRYPTO_OP_DECRYPT; 2328 2329 /* AAD validation. */ 2330 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2331 crp->crp_aad = ohdr; 2332 crp->crp_aad_start = 0; 2333 crp->crp_op |= CRYPTO_OP_VERIFY_DIGEST; 2334 crp->crp_digest_start = sizeof(struct udphdr) + 2335 offsetof(struct ovpn_wire_header, auth_tag); 2336 2337 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 2338 memcpy(crp->crp_iv, &ohdr->seq, sizeof(ohdr->seq)); 2339 memcpy(crp->crp_iv + sizeof(ohdr->seq), key->decrypt->nonce, 2340 key->decrypt->noncelen); 2341 2342 crypto_use_mbuf(crp, m); 2343 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 2344 crp->crp_callback = ovpn_decrypt_rx_cb; 2345 crp->crp_opaque = sc; 2346 2347 atomic_add_int(&sc->refcount, 1); 2348 OVPN_RUNLOCK(sc); 2349 if (V_async_crypto) 2350 ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); 2351 else 2352 ret = crypto_dispatch(crp); 2353 if (ret != 0) { 2354 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2355 } 2356 2357 return (true); 2358 } 2359 2360 static void 2361 ovpn_qflush(struct ifnet *ifp __unused) 2362 { 2363 2364 } 2365 2366 static void 2367 ovpn_flush_rxring(struct ovpn_softc *sc) 2368 { 2369 struct ovpn_notification *n; 2370 2371 OVPN_WASSERT(sc); 2372 2373 while (! buf_ring_empty(sc->notifring)) { 2374 n = buf_ring_dequeue_sc(sc->notifring); 2375 free(n, M_OVPN); 2376 } 2377 } 2378 2379 #ifdef VIMAGE 2380 static void 2381 ovpn_reassign(struct ifnet *ifp, struct vnet *new_vnet __unused, 2382 char *unused __unused) 2383 { 2384 struct ovpn_softc *sc = ifp->if_softc; 2385 struct ovpn_kpeer *peer, *tmppeer; 2386 int ret __diagused; 2387 2388 OVPN_WLOCK(sc); 2389 2390 /* Flush keys & configuration. */ 2391 RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) { 2392 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 2393 ret = _ovpn_del_peer(sc, peer); 2394 MPASS(ret == 0); 2395 } 2396 2397 ovpn_flush_rxring(sc); 2398 2399 OVPN_WUNLOCK(sc); 2400 } 2401 #endif 2402 2403 static int 2404 ovpn_clone_match(struct if_clone *ifc, const char *name) 2405 { 2406 /* 2407 * Allow all names that start with 'ovpn', specifically because pfSense 2408 * uses ovpnc1 / ovpns2 2409 */ 2410 return (strncmp(ovpnname, name, strlen(ovpnname)) == 0); 2411 } 2412 2413 static int 2414 ovpn_clone_create(struct if_clone *ifc, char *name, size_t len, 2415 struct ifc_data *ifd, struct ifnet **ifpp) 2416 { 2417 struct ovpn_softc *sc; 2418 struct ifnet *ifp; 2419 char *dp; 2420 int error, unit, wildcard; 2421 2422 /* Try to see if a special unit was requested. */ 2423 error = ifc_name2unit(name, &unit); 2424 if (error != 0) 2425 return (error); 2426 wildcard = (unit < 0); 2427 2428 error = ifc_alloc_unit(ifc, &unit); 2429 if (error != 0) 2430 return (error); 2431 2432 /* 2433 * If no unit had been given, we need to adjust the ifName. 2434 */ 2435 for (dp = name; *dp != '\0'; dp++); 2436 if (wildcard) { 2437 error = snprintf(dp, len - (dp - name), "%d", unit); 2438 if (error > len - (dp - name)) { 2439 /* ifName too long. */ 2440 ifc_free_unit(ifc, unit); 2441 return (ENOSPC); 2442 } 2443 dp += error; 2444 } 2445 2446 /* Make sure it doesn't already exist. */ 2447 if (ifunit(name) != NULL) 2448 return (EEXIST); 2449 2450 sc = malloc(sizeof(struct ovpn_softc), M_OVPN, M_WAITOK | M_ZERO); 2451 sc->ifp = if_alloc(IFT_ENC); 2452 rm_init_flags(&sc->lock, "if_ovpn_lock", RM_RECURSE); 2453 sc->refcount = 0; 2454 2455 sc->notifring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL); 2456 2457 COUNTER_ARRAY_ALLOC(sc->counters, OVPN_COUNTER_SIZE, M_WAITOK); 2458 2459 ifp = sc->ifp; 2460 ifp->if_softc = sc; 2461 strlcpy(ifp->if_xname, name, IFNAMSIZ); 2462 ifp->if_dname = ovpngroupname; 2463 ifp->if_dunit = unit; 2464 2465 ifp->if_addrlen = 0; 2466 ifp->if_mtu = 1428; 2467 ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST; 2468 ifp->if_ioctl = ovpn_ioctl; 2469 ifp->if_transmit = ovpn_transmit; 2470 ifp->if_output = ovpn_output; 2471 ifp->if_qflush = ovpn_qflush; 2472 #ifdef VIMAGE 2473 ifp->if_reassign = ovpn_reassign; 2474 #endif 2475 ifp->if_capabilities |= IFCAP_LINKSTATE; 2476 ifp->if_capenable |= IFCAP_LINKSTATE; 2477 2478 if_attach(ifp); 2479 bpfattach(ifp, DLT_NULL, sizeof(uint32_t)); 2480 *ifpp = ifp; 2481 2482 return (0); 2483 } 2484 2485 static void 2486 ovpn_clone_destroy_cb(struct epoch_context *ctx) 2487 { 2488 struct ovpn_softc *sc; 2489 int ret __diagused; 2490 2491 sc = __containerof(ctx, struct ovpn_softc, epoch_ctx); 2492 2493 MPASS(sc->peercount == 0); 2494 MPASS(RB_EMPTY(&sc->peers)); 2495 2496 if (sc->so != NULL) { 2497 CURVNET_SET(sc->ifp->if_vnet); 2498 ret = udp_set_kernel_tunneling(sc->so, NULL, NULL, NULL); 2499 MPASS(ret == 0); 2500 sorele(sc->so); 2501 CURVNET_RESTORE(); 2502 } 2503 2504 COUNTER_ARRAY_FREE(sc->counters, OVPN_COUNTER_SIZE); 2505 2506 if_free(sc->ifp); 2507 free(sc, M_OVPN); 2508 } 2509 2510 static int 2511 ovpn_clone_destroy(struct if_clone *ifc, struct ifnet *ifp, uint32_t flags) 2512 { 2513 struct ovpn_softc *sc; 2514 struct ovpn_kpeer *peer, *tmppeer; 2515 int unit; 2516 int ret __diagused; 2517 2518 sc = ifp->if_softc; 2519 unit = ifp->if_dunit; 2520 2521 OVPN_WLOCK(sc); 2522 2523 if (atomic_load_int(&sc->refcount) > 0) { 2524 OVPN_WUNLOCK(sc); 2525 return (EBUSY); 2526 } 2527 2528 RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) { 2529 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 2530 ret = _ovpn_del_peer(sc, peer); 2531 MPASS(ret == 0); 2532 } 2533 2534 ovpn_flush_rxring(sc); 2535 buf_ring_free(sc->notifring, M_OVPN); 2536 2537 OVPN_WUNLOCK(sc); 2538 2539 bpfdetach(ifp); 2540 if_detach(ifp); 2541 ifp->if_softc = NULL; 2542 2543 NET_EPOCH_CALL(ovpn_clone_destroy_cb, &sc->epoch_ctx); 2544 2545 if (unit != IF_DUNIT_NONE) 2546 ifc_free_unit(ifc, unit); 2547 2548 NET_EPOCH_DRAIN_CALLBACKS(); 2549 2550 return (0); 2551 } 2552 2553 static void 2554 vnet_ovpn_init(const void *unused __unused) 2555 { 2556 struct if_clone_addreq req = { 2557 .match_f = ovpn_clone_match, 2558 .create_f = ovpn_clone_create, 2559 .destroy_f = ovpn_clone_destroy, 2560 }; 2561 V_ovpn_cloner = ifc_attach_cloner(ovpngroupname, &req); 2562 } 2563 VNET_SYSINIT(vnet_ovpn_init, SI_SUB_PSEUDO, SI_ORDER_ANY, 2564 vnet_ovpn_init, NULL); 2565 2566 static void 2567 vnet_ovpn_uninit(const void *unused __unused) 2568 { 2569 if_clone_detach(V_ovpn_cloner); 2570 } 2571 VNET_SYSUNINIT(vnet_ovpn_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY, 2572 vnet_ovpn_uninit, NULL); 2573 2574 static int 2575 ovpnmodevent(module_t mod, int type, void *data) 2576 { 2577 switch (type) { 2578 case MOD_LOAD: 2579 /* Done in vnet_ovpn_init() */ 2580 break; 2581 case MOD_UNLOAD: 2582 /* Done in vnet_ovpn_uninit() */ 2583 break; 2584 default: 2585 return (EOPNOTSUPP); 2586 } 2587 2588 return (0); 2589 } 2590 2591 static moduledata_t ovpn_mod = { 2592 "if_ovpn", 2593 ovpnmodevent, 2594 0 2595 }; 2596 2597 DECLARE_MODULE(if_ovpn, ovpn_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 2598 MODULE_VERSION(if_ovpn, 1); 2599