1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2021-2022 Rubicon Communications, LLC (Netgate) 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 */ 28 #include "opt_inet.h" 29 #include "opt_inet6.h" 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/buf_ring.h> 34 #include <sys/epoch.h> 35 #include <sys/file.h> 36 #include <sys/filedesc.h> 37 #include <sys/jail.h> 38 #include <sys/kernel.h> 39 #include <sys/malloc.h> 40 #include <sys/mbuf.h> 41 #include <sys/module.h> 42 #include <sys/nv.h> 43 #include <sys/osd.h> 44 #include <sys/priv.h> 45 #include <sys/protosw.h> 46 #include <sys/rmlock.h> 47 #include <sys/sdt.h> 48 #include <sys/smp.h> 49 #include <sys/socket.h> 50 #include <sys/socketvar.h> 51 #include <sys/sockio.h> 52 #include <sys/sysctl.h> 53 #include <sys/time.h> 54 55 #include <machine/atomic.h> 56 57 #include <net/bpf.h> 58 #include <net/if.h> 59 #include <net/if_clone.h> 60 #include <net/if_types.h> 61 #include <net/if_var.h> 62 #include <net/if_private.h> 63 #include <net/netisr.h> 64 #include <net/route/nhop.h> 65 66 #include <netinet/in.h> 67 #include <netinet/in_fib.h> 68 #include <netinet/ip.h> 69 #include <netinet/ip6.h> 70 #include <netinet/ip_var.h> 71 #include <netinet/udp.h> 72 #include <netinet/udp_var.h> 73 74 #include <netinet6/ip6_var.h> 75 #include <netinet6/in6_fib.h> 76 77 #include <machine/in_cksum.h> 78 79 #include <opencrypto/cryptodev.h> 80 81 #include "if_ovpn.h" 82 83 struct ovpn_kkey_dir { 84 uint8_t key[32]; 85 uint8_t keylen; 86 uint8_t nonce[8]; 87 uint8_t noncelen; 88 enum ovpn_key_cipher cipher; 89 crypto_session_t cryptoid; 90 91 struct mtx replay_mtx; 92 /* 93 * Last seen gapless sequence number. New rx seq numbers must be 94 * strictly higher than this. 95 */ 96 uint32_t rx_seq; 97 uint64_t tx_seq; 98 99 /* Seen packets, relative to rx_seq. bit(0) will always be 0. */ 100 uint64_t rx_window; 101 }; 102 103 struct ovpn_kkey { 104 struct ovpn_kkey_dir *encrypt; 105 struct ovpn_kkey_dir *decrypt; 106 uint8_t keyid; 107 uint32_t peerid; 108 }; 109 110 struct ovpn_keepalive { 111 uint32_t interval; 112 uint32_t timeout; 113 }; 114 115 struct ovpn_wire_header { 116 uint32_t opcode; /* opcode, key id, peer id */ 117 uint32_t seq; 118 uint8_t auth_tag[16]; 119 }; 120 121 struct ovpn_peer_counters { 122 uint64_t pkt_in; 123 uint64_t pkt_out; 124 uint64_t bytes_in; 125 uint64_t bytes_out; 126 }; 127 #define OVPN_PEER_COUNTER_SIZE (sizeof(struct ovpn_peer_counters)/sizeof(uint64_t)) 128 129 struct ovpn_notification { 130 enum ovpn_notif_type type; 131 uint32_t peerid; 132 133 /* Delete notification */ 134 enum ovpn_del_reason del_reason; 135 struct ovpn_peer_counters counters; 136 137 /* Float notification */ 138 struct sockaddr_storage address; 139 }; 140 141 struct ovpn_softc; 142 143 struct ovpn_kpeer { 144 RB_ENTRY(ovpn_kpeer) tree; 145 int refcount; 146 uint32_t peerid; 147 148 struct ovpn_softc *sc; 149 struct sockaddr_storage local; 150 struct sockaddr_storage remote; 151 152 struct in_addr vpn4; 153 struct in6_addr vpn6; 154 155 struct ovpn_kkey keys[2]; 156 157 enum ovpn_del_reason del_reason; 158 struct ovpn_keepalive keepalive; 159 uint32_t *last_active; 160 struct callout ping_send; 161 struct callout ping_rcv; 162 163 counter_u64_t counters[OVPN_PEER_COUNTER_SIZE]; 164 struct epoch_context epoch_ctx; 165 }; 166 167 struct ovpn_counters { 168 uint64_t lost_ctrl_pkts_in; 169 uint64_t lost_ctrl_pkts_out; 170 uint64_t lost_data_pkts_in; 171 uint64_t lost_data_pkts_out; 172 uint64_t nomem_data_pkts_in; 173 uint64_t nomem_data_pkts_out; 174 uint64_t received_ctrl_pkts; 175 uint64_t received_data_pkts; 176 uint64_t sent_ctrl_pkts; 177 uint64_t sent_data_pkts; 178 179 uint64_t transport_bytes_sent; 180 uint64_t transport_bytes_received; 181 uint64_t tunnel_bytes_sent; 182 uint64_t tunnel_bytes_received; 183 }; 184 #define OVPN_COUNTER_SIZE (sizeof(struct ovpn_counters)/sizeof(uint64_t)) 185 186 RB_HEAD(ovpn_kpeers, ovpn_kpeer); 187 188 struct ovpn_softc { 189 int refcount; 190 struct rmlock lock; 191 struct ifnet *ifp; 192 struct socket *so; 193 int peercount; 194 struct ovpn_kpeers peers; 195 196 /* Pending notification */ 197 struct buf_ring *notifring; 198 199 counter_u64_t counters[OVPN_COUNTER_SIZE]; 200 201 struct epoch_context epoch_ctx; 202 }; 203 204 struct ovpn_mtag { 205 struct sockaddr_storage addr; 206 }; 207 208 static struct ovpn_kpeer *ovpn_find_peer(struct ovpn_softc *, uint32_t); 209 static bool ovpn_udp_input(struct mbuf *, int, struct inpcb *, 210 const struct sockaddr *, void *); 211 static int ovpn_transmit_to_peer(struct ifnet *, struct mbuf *, 212 struct ovpn_kpeer *, struct rm_priotracker *); 213 static int ovpn_encap(struct ovpn_softc *, uint32_t, struct mbuf *); 214 static int ovpn_get_af(struct mbuf *); 215 static void ovpn_free_kkey_dir(struct ovpn_kkey_dir *); 216 static bool ovpn_check_replay(struct ovpn_kkey_dir *, uint32_t); 217 static int ovpn_peer_compare(const struct ovpn_kpeer *, 218 const struct ovpn_kpeer *); 219 static bool ovpn_sockaddr_compare(const struct sockaddr *, 220 const struct sockaddr *); 221 222 static RB_PROTOTYPE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare); 223 static RB_GENERATE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare); 224 225 #define OVPN_MTU_MIN 576 226 #define OVPN_MTU_MAX (IP_MAXPACKET - sizeof(struct ip) - \ 227 sizeof(struct udphdr) - sizeof(struct ovpn_wire_header)) 228 229 #define OVPN_OP_DATA_V2 0x09 230 #define OVPN_OP_SHIFT 3 231 #define OVPN_SEQ_ROTATE 0x80000000 232 233 VNET_DEFINE_STATIC(struct if_clone *, ovpn_cloner); 234 #define V_ovpn_cloner VNET(ovpn_cloner) 235 236 #define OVPN_RLOCK_TRACKER struct rm_priotracker _ovpn_lock_tracker; \ 237 struct rm_priotracker *_ovpn_lock_trackerp = &_ovpn_lock_tracker 238 #define OVPN_RLOCK(sc) rm_rlock(&(sc)->lock, _ovpn_lock_trackerp) 239 #define OVPN_RUNLOCK(sc) rm_runlock(&(sc)->lock, _ovpn_lock_trackerp) 240 #define OVPN_WLOCK(sc) rm_wlock(&(sc)->lock) 241 #define OVPN_WUNLOCK(sc) rm_wunlock(&(sc)->lock) 242 #define OVPN_ASSERT(sc) rm_assert(&(sc)->lock, RA_LOCKED) 243 #define OVPN_RASSERT(sc) rm_assert(&(sc)->lock, RA_RLOCKED) 244 #define OVPN_WASSERT(sc) rm_assert(&(sc)->lock, RA_WLOCKED) 245 #define OVPN_UNLOCK_ASSERT(sc) rm_assert(&(sc)->lock, RA_UNLOCKED) 246 247 #define OVPN_COUNTER(sc, name) \ 248 ((sc)->counters[offsetof(struct ovpn_counters, name)/sizeof(uint64_t)]) 249 #define OVPN_PEER_COUNTER(peer, name) \ 250 ((peer)->counters[offsetof(struct ovpn_peer_counters, name) / \ 251 sizeof(uint64_t)]) 252 253 #define OVPN_COUNTER_ADD(sc, name, val) \ 254 counter_u64_add(OVPN_COUNTER(sc, name), val) 255 #define OVPN_PEER_COUNTER_ADD(p, name, val) \ 256 counter_u64_add(OVPN_PEER_COUNTER(p, name), val) 257 258 #define TO_IN(x) ((struct sockaddr_in *)(x)) 259 #define TO_IN6(x) ((struct sockaddr_in6 *)(x)) 260 261 SDT_PROVIDER_DEFINE(if_ovpn); 262 SDT_PROBE_DEFINE1(if_ovpn, tx, transmit, start, "struct mbuf *"); 263 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip4, "struct in_addr *", "struct ovpn_kpeer *"); 264 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip6, "struct in6_addr *", "struct ovpn_kpeer *"); 265 266 static const char ovpnname[] = "ovpn"; 267 static const char ovpngroupname[] = "openvpn"; 268 269 static MALLOC_DEFINE(M_OVPN, ovpnname, "OpenVPN DCO Interface"); 270 #define MTAG_OVPN_LOOP 0x6f76706e /* ovpn */ 271 272 SYSCTL_DECL(_net_link); 273 static SYSCTL_NODE(_net_link, IFT_OTHER, openvpn, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 274 "OpenVPN DCO Interface"); 275 VNET_DEFINE_STATIC(int, replay_protection) = 0; 276 #define V_replay_protection VNET(replay_protection) 277 SYSCTL_INT(_net_link_openvpn, OID_AUTO, replay_protection, CTLFLAG_VNET | CTLFLAG_RW, 278 &VNET_NAME(replay_protection), 0, "Validate sequence numbers"); 279 280 VNET_DEFINE_STATIC(int, async_crypto); 281 #define V_async_crypto VNET(async_crypto) 282 SYSCTL_INT(_net_link_openvpn, OID_AUTO, async_crypto, 283 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_crypto), 0, 284 "Use asynchronous mode to parallelize crypto jobs."); 285 286 VNET_DEFINE_STATIC(int, async_netisr_queue); 287 #define V_async_netisr_queue VNET(async_netisr_queue) 288 SYSCTL_INT(_net_link_openvpn, OID_AUTO, netisr_queue, 289 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_netisr_queue), 0, 290 "Use netisr_queue() rather than netisr_dispatch()."); 291 292 static int 293 ovpn_peer_compare(const struct ovpn_kpeer *a, const struct ovpn_kpeer *b) 294 { 295 return (a->peerid - b->peerid); 296 } 297 298 static bool 299 ovpn_sockaddr_compare(const struct sockaddr *a, 300 const struct sockaddr *b) 301 { 302 if (a->sa_family != b->sa_family) 303 return (false); 304 MPASS(a->sa_len == b->sa_len); 305 306 switch (a->sa_family) { 307 case AF_INET: { 308 const struct sockaddr_in *a4, *b4; 309 310 a4 = (const struct sockaddr_in *)a; 311 b4 = (const struct sockaddr_in *)b; 312 313 if (a4->sin_port != b4->sin_port) 314 return (false); 315 316 return (a4->sin_addr.s_addr == b4->sin_addr.s_addr); 317 } 318 case AF_INET6: { 319 const struct sockaddr_in6 *a6, *b6; 320 321 a6 = (const struct sockaddr_in6 *)a; 322 b6 = (const struct sockaddr_in6 *)b; 323 324 if (a6->sin6_port != b6->sin6_port) 325 return (false); 326 if (a6->sin6_scope_id != b6->sin6_scope_id) 327 return (false); 328 329 return (memcmp(&a6->sin6_addr, &b6->sin6_addr, 330 sizeof(a6->sin6_addr)) == 0); 331 } 332 default: 333 panic("Unknown address family %d", a->sa_family); 334 } 335 } 336 337 static struct ovpn_kpeer * 338 ovpn_find_peer(struct ovpn_softc *sc, uint32_t peerid) 339 { 340 struct ovpn_kpeer p; 341 342 OVPN_ASSERT(sc); 343 344 p.peerid = peerid; 345 346 return (RB_FIND(ovpn_kpeers, &sc->peers, &p)); 347 } 348 349 static struct ovpn_kpeer * 350 ovpn_find_only_peer(struct ovpn_softc *sc) 351 { 352 OVPN_ASSERT(sc); 353 354 return (RB_ROOT(&sc->peers)); 355 } 356 357 static uint16_t 358 ovpn_get_port(const struct sockaddr_storage *s) 359 { 360 switch (s->ss_family) { 361 case AF_INET: { 362 const struct sockaddr_in *in = (const struct sockaddr_in *)s; 363 return (in->sin_port); 364 } 365 case AF_INET6: { 366 const struct sockaddr_in6 *in6 = (const struct sockaddr_in6 *)s; 367 return (in6->sin6_port); 368 } 369 default: 370 panic("Unsupported address family %d", s->ss_family); 371 } 372 } 373 374 static void 375 ovpn_set_port(struct sockaddr_storage *s, unsigned short port) 376 { 377 switch (s->ss_family) { 378 case AF_INET: { 379 struct sockaddr_in *in = (struct sockaddr_in *)s; 380 in->sin_port = port; 381 break; 382 } 383 case AF_INET6: { 384 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)s; 385 in6->sin6_port = port; 386 break; 387 } 388 default: 389 panic("Unsupported address family %d", s->ss_family); 390 } 391 } 392 393 static int 394 ovpn_nvlist_to_sockaddr(const nvlist_t *nvl, struct sockaddr_storage *sa) 395 { 396 int af; 397 398 memset(sa, 0, sizeof(*sa)); 399 400 if (! nvlist_exists_number(nvl, "af")) 401 return (EINVAL); 402 if (! nvlist_exists_binary(nvl, "address")) 403 return (EINVAL); 404 if (! nvlist_exists_number(nvl, "port")) 405 return (EINVAL); 406 407 af = nvlist_get_number(nvl, "af"); 408 switch (af) { 409 #ifdef INET 410 case AF_INET: { 411 struct sockaddr_in *in = (struct sockaddr_in *)sa; 412 size_t len; 413 const void *addr = nvlist_get_binary(nvl, "address", &len); 414 415 memset(in, 0, sizeof(*in)); 416 in->sin_family = af; 417 in->sin_len = sizeof(*in); 418 if (len != sizeof(in->sin_addr)) 419 return (EINVAL); 420 421 memcpy(&in->sin_addr, addr, sizeof(in->sin_addr)); 422 in->sin_port = nvlist_get_number(nvl, "port"); 423 break; 424 } 425 #endif 426 #ifdef INET6 427 case AF_INET6: { 428 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)sa; 429 size_t len; 430 const void *addr = nvlist_get_binary(nvl, "address", &len); 431 432 memset(in6, 0, sizeof(*in6)); 433 in6->sin6_family = af; 434 in6->sin6_len = sizeof(*in6); 435 if (len != sizeof(in6->sin6_addr)) 436 return (EINVAL); 437 438 memcpy(&in6->sin6_addr, addr, sizeof(in6->sin6_addr)); 439 in6->sin6_port = nvlist_get_number(nvl, "port"); 440 441 if (nvlist_exists_number(nvl, "scopeid")) 442 in6->sin6_scope_id = nvlist_get_number(nvl, "scopeid"); 443 444 break; 445 } 446 #endif 447 default: 448 return (EINVAL); 449 } 450 451 return (0); 452 } 453 454 static int 455 ovpn_add_sockaddr(nvlist_t *parent, const char *name, const struct sockaddr *s) 456 { 457 nvlist_t *nvl; 458 459 nvl = nvlist_create(0); 460 if (nvl == NULL) 461 return (ENOMEM); 462 463 nvlist_add_number(nvl, "af", s->sa_family); 464 465 switch (s->sa_family) { 466 case AF_INET: { 467 const struct sockaddr_in *s4 = (const struct sockaddr_in *)s; 468 469 nvlist_add_number(nvl, "port", s4->sin_port); 470 nvlist_add_binary(nvl, "address", &s4->sin_addr, 471 sizeof(s4->sin_addr)); 472 break; 473 } 474 case AF_INET6: { 475 const struct sockaddr_in6 *s6 = (const struct sockaddr_in6 *)s; 476 477 nvlist_add_number(nvl, "port", s6->sin6_port); 478 nvlist_add_binary(nvl, "address", &s6->sin6_addr, 479 sizeof(s6->sin6_addr)); 480 nvlist_add_number(nvl, "scopeid", s6->sin6_scope_id); 481 break; 482 } 483 default: 484 nvlist_destroy(nvl); 485 return (EINVAL); 486 } 487 488 nvlist_move_nvlist(parent, name, nvl); 489 490 return (0); 491 } 492 493 static void 494 ovpn_notify_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 495 { 496 struct ovpn_notification *n; 497 498 OVPN_WASSERT(sc); 499 500 n = malloc(sizeof(*n), M_OVPN, M_NOWAIT); 501 if (n == NULL) 502 return; 503 504 n->peerid = peer->peerid; 505 n->type = OVPN_NOTIF_DEL_PEER; 506 n->del_reason = peer->del_reason; 507 508 n->counters.pkt_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_in)); 509 n->counters.pkt_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_out)); 510 n->counters.bytes_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_in)); 511 n->counters.bytes_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_out)); 512 513 if (buf_ring_enqueue(sc->notifring, n) != 0) { 514 free(n, M_OVPN); 515 } else if (sc->so != NULL) { 516 /* Wake up userspace */ 517 sc->so->so_error = EAGAIN; 518 sorwakeup(sc->so); 519 sowwakeup(sc->so); 520 } 521 } 522 523 static void 524 ovpn_notify_key_rotation(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 525 { 526 struct ovpn_notification *n; 527 528 n = malloc(sizeof(*n), M_OVPN, M_NOWAIT | M_ZERO); 529 if (n == NULL) 530 return; 531 532 n->peerid = peer->peerid; 533 n->type = OVPN_NOTIF_ROTATE_KEY; 534 535 if (buf_ring_enqueue(sc->notifring, n) != 0) { 536 free(n, M_OVPN); 537 } else if (sc->so != NULL) { 538 /* Wake up userspace */ 539 sc->so->so_error = EAGAIN; 540 sorwakeup(sc->so); 541 sowwakeup(sc->so); 542 } 543 } 544 545 static int 546 ovpn_notify_float(struct ovpn_softc *sc, uint32_t peerid, 547 const struct sockaddr_storage *remote) 548 { 549 struct ovpn_notification *n; 550 551 n = malloc(sizeof(*n), M_OVPN, M_NOWAIT | M_ZERO); 552 if (n == NULL) 553 return (ENOMEM); 554 555 n->peerid = peerid; 556 n->type = OVPN_NOTIF_FLOAT; 557 memcpy(&n->address, remote, sizeof(n->address)); 558 559 if (buf_ring_enqueue(sc->notifring, n) != 0) { 560 free(n, M_OVPN); 561 return (ENOMEM); 562 } else if (sc->so != NULL) { 563 /* Wake up userspace */ 564 sc->so->so_error = EAGAIN; 565 sorwakeup(sc->so); 566 sowwakeup(sc->so); 567 } 568 569 return (0); 570 } 571 572 static void 573 _ovpn_free_peer(struct epoch_context *ctx) { 574 struct ovpn_kpeer *peer = __containerof(ctx, struct ovpn_kpeer, 575 epoch_ctx); 576 577 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 578 free(peer, M_OVPN); 579 } 580 581 static void 582 ovpn_peer_release_ref(struct ovpn_kpeer *peer, bool locked) 583 { 584 struct ovpn_softc *sc; 585 586 CURVNET_ASSERT_SET(); 587 588 atomic_add_int(&peer->refcount, -1); 589 590 if (atomic_load_int(&peer->refcount) > 0) 591 return; 592 593 sc = peer->sc; 594 595 if (! locked) { 596 OVPN_WLOCK(sc); 597 598 /* Might have changed before we acquired the lock. */ 599 if (atomic_load_int(&peer->refcount) > 0) { 600 OVPN_WUNLOCK(sc); 601 return; 602 } 603 } 604 605 OVPN_ASSERT(sc); 606 607 /* The peer should have been removed from the list already. */ 608 MPASS(ovpn_find_peer(sc, peer->peerid) == NULL); 609 610 ovpn_notify_del_peer(sc, peer); 611 612 for (int i = 0; i < 2; i++) { 613 ovpn_free_kkey_dir(peer->keys[i].encrypt); 614 ovpn_free_kkey_dir(peer->keys[i].decrypt); 615 } 616 617 callout_stop(&peer->ping_send); 618 callout_stop(&peer->ping_rcv); 619 620 NET_EPOCH_CALL(_ovpn_free_peer, &peer->epoch_ctx); 621 622 if (! locked) 623 OVPN_WUNLOCK(sc); 624 } 625 626 static int 627 ovpn_new_peer(struct ifnet *ifp, const nvlist_t *nvl) 628 { 629 #ifdef INET6 630 struct epoch_tracker et; 631 #endif 632 struct sockaddr_storage local, remote; 633 struct ovpn_kpeer *peer = NULL; 634 struct file *fp = NULL; 635 struct ovpn_softc *sc = ifp->if_softc; 636 struct thread *td = curthread; 637 struct socket *so = NULL; 638 int fd; 639 uint32_t peerid; 640 int ret = 0; 641 bool setcb = false; 642 643 if (nvl == NULL) 644 return (EINVAL); 645 646 if (! nvlist_exists_number(nvl, "peerid")) 647 return (EINVAL); 648 649 if (! nvlist_exists_number(nvl, "fd")) 650 return (EINVAL); 651 652 if (! nvlist_exists_nvlist(nvl, "remote")) 653 return (EINVAL); 654 655 peerid = nvlist_get_number(nvl, "peerid"); 656 657 ret = ovpn_nvlist_to_sockaddr(nvlist_get_nvlist(nvl, "remote"), 658 &remote); 659 if (ret != 0) 660 return (ret); 661 662 fd = nvlist_get_number(nvl, "fd"); 663 664 /* Look up the userspace process and use the fd to find the socket. */ 665 ret = getsock(td, fd, &cap_connect_rights, &fp); 666 if (ret != 0) 667 return (ret); 668 669 so = fp->f_data; 670 671 peer = malloc(sizeof(*peer), M_OVPN, M_WAITOK | M_ZERO); 672 peer->peerid = peerid; 673 peer->sc = sc; 674 peer->refcount = 1; 675 peer->last_active = uma_zalloc_pcpu(pcpu_zone_4, M_WAITOK | M_ZERO); 676 COUNTER_ARRAY_ALLOC(peer->counters, OVPN_PEER_COUNTER_SIZE, M_WAITOK); 677 678 if (nvlist_exists_binary(nvl, "vpn_ipv4")) { 679 size_t len; 680 const void *addr = nvlist_get_binary(nvl, "vpn_ipv4", &len); 681 if (len != sizeof(peer->vpn4)) { 682 ret = EINVAL; 683 goto error; 684 } 685 memcpy(&peer->vpn4, addr, len); 686 } 687 688 if (nvlist_exists_binary(nvl, "vpn_ipv6")) { 689 size_t len; 690 const void *addr = nvlist_get_binary(nvl, "vpn_ipv6", &len); 691 if (len != sizeof(peer->vpn6)) { 692 ret = EINVAL; 693 goto error; 694 } 695 memcpy(&peer->vpn6, addr, len); 696 } 697 698 callout_init_rm(&peer->ping_send, &sc->lock, CALLOUT_SHAREDLOCK); 699 callout_init_rm(&peer->ping_rcv, &sc->lock, 0); 700 701 memset(&local, 0, sizeof(local)); 702 local.ss_len = sizeof(local); 703 ret = sosockaddr(so, (struct sockaddr *)&local); 704 if (ret != 0) 705 goto error; 706 if (nvlist_exists_nvlist(nvl, "local")) { 707 struct sockaddr_storage local1; 708 709 ret = ovpn_nvlist_to_sockaddr(nvlist_get_nvlist(nvl, "local"), 710 &local1); 711 if (ret != 0) 712 goto error; 713 714 /* 715 * openvpn doesn't provide a port here when in multihome mode, 716 * just steal the one the socket is bound to. 717 */ 718 if (ovpn_get_port(&local1) == 0) 719 ovpn_set_port(&local1, ovpn_get_port(&local)); 720 memcpy(&local, &local1, sizeof(local1)); 721 } 722 if (ovpn_get_port(&local) == 0) { 723 ret = EINVAL; 724 goto error; 725 } 726 if (local.ss_family != remote.ss_family) { 727 ret = EINVAL; 728 goto error; 729 } 730 731 memcpy(&peer->local, &local, sizeof(local)); 732 memcpy(&peer->remote, &remote, sizeof(remote)); 733 734 #ifdef INET6 735 if (peer->local.ss_family == AF_INET6 && 736 IN6_IS_ADDR_V4MAPPED(&TO_IN6(&peer->remote)->sin6_addr)) { 737 /* V4 mapped address, so treat this as v4, not v6. */ 738 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->local); 739 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->remote); 740 } 741 742 if (peer->local.ss_family == AF_INET6 && 743 IN6_IS_ADDR_UNSPECIFIED(&TO_IN6(&peer->local)->sin6_addr)) { 744 NET_EPOCH_ENTER(et); 745 ret = in6_selectsrc_addr(curthread->td_proc->p_fibnum, 746 &TO_IN6(&peer->remote)->sin6_addr, 747 TO_IN6(&peer->remote)->sin6_scope_id, NULL, 748 &TO_IN6(&peer->local)->sin6_addr, NULL); 749 NET_EPOCH_EXIT(et); 750 if (ret != 0) { 751 goto error; 752 } 753 } 754 #endif 755 OVPN_WLOCK(sc); 756 757 /* Disallow peer id re-use. */ 758 if (ovpn_find_peer(sc, peerid) != NULL) { 759 ret = EEXIST; 760 goto error_locked; 761 } 762 763 /* Make sure this is really a UDP socket. */ 764 if (so->so_type != SOCK_DGRAM || so->so_proto->pr_type != SOCK_DGRAM) { 765 ret = EPROTOTYPE; 766 goto error_locked; 767 } 768 769 /* Must be the same socket as for other peers on this interface. */ 770 if (sc->so != NULL && so != sc->so) { 771 if (! RB_EMPTY(&sc->peers)) { 772 ret = EBUSY; 773 goto error_locked; 774 } 775 776 /* 777 * If we have no peers we can safely release the socket and accept 778 * a new one. 779 */ 780 ret = udp_set_kernel_tunneling(sc->so, NULL, NULL, NULL); 781 MPASS(ret == 0); 782 sorele(sc->so); 783 sc->so = NULL; 784 } 785 786 if (sc->so == NULL) { 787 sc->so = so; 788 /* 789 * Maintain one extra ref so the socket doesn't go away until 790 * we're destroying the ifp. 791 */ 792 soref(sc->so); 793 setcb = true; 794 } 795 796 /* Insert the peer into the list. */ 797 RB_INSERT(ovpn_kpeers, &sc->peers, peer); 798 sc->peercount++; 799 800 OVPN_WUNLOCK(sc); 801 802 if (setcb) { 803 ret = udp_set_kernel_tunneling(sc->so, ovpn_udp_input, NULL, sc); 804 MPASS(ret == 0); 805 } 806 807 goto done; 808 809 error_locked: 810 OVPN_WUNLOCK(sc); 811 error: 812 COUNTER_ARRAY_FREE(peer->counters, OVPN_PEER_COUNTER_SIZE); 813 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 814 free(peer, M_OVPN); 815 done: 816 if (fp != NULL) 817 fdrop(fp, td); 818 819 return (ret); 820 } 821 822 static int 823 _ovpn_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 824 { 825 struct ovpn_kpeer *tmp __diagused; 826 827 OVPN_WASSERT(sc); 828 CURVNET_ASSERT_SET(); 829 830 MPASS(RB_FIND(ovpn_kpeers, &sc->peers, peer) == peer); 831 832 tmp = RB_REMOVE(ovpn_kpeers, &sc->peers, peer); 833 MPASS(tmp != NULL); 834 835 sc->peercount--; 836 837 ovpn_peer_release_ref(peer, true); 838 839 return (0); 840 } 841 842 static int 843 ovpn_del_peer(struct ifnet *ifp, nvlist_t *nvl) 844 { 845 struct ovpn_softc *sc = ifp->if_softc; 846 struct ovpn_kpeer *peer; 847 uint32_t peerid; 848 int ret; 849 850 OVPN_WASSERT(sc); 851 852 if (nvl == NULL) 853 return (EINVAL); 854 855 if (! nvlist_exists_number(nvl, "peerid")) 856 return (EINVAL); 857 858 peerid = nvlist_get_number(nvl, "peerid"); 859 860 peer = ovpn_find_peer(sc, peerid); 861 if (peer == NULL) 862 return (ENOENT); 863 864 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 865 ret = _ovpn_del_peer(sc, peer); 866 867 return (ret); 868 } 869 870 static int 871 ovpn_create_kkey_dir(struct ovpn_kkey_dir **kdirp, 872 const nvlist_t *nvl) 873 { 874 struct crypto_session_params csp; 875 struct ovpn_kkey_dir *kdir; 876 const char *ciphername; 877 enum ovpn_key_cipher cipher; 878 const void *key, *iv; 879 size_t keylen = 0, ivlen = 0; 880 int error; 881 882 if (! nvlist_exists_string(nvl, "cipher")) 883 return (EINVAL); 884 ciphername = nvlist_get_string(nvl, "cipher"); 885 886 if (strcmp(ciphername, "none") == 0) 887 cipher = OVPN_CIPHER_ALG_NONE; 888 else if (strcmp(ciphername, "AES-256-GCM") == 0 || 889 strcmp(ciphername, "AES-192-GCM") == 0 || 890 strcmp(ciphername, "AES-128-GCM") == 0) 891 cipher = OVPN_CIPHER_ALG_AES_GCM; 892 else if (strcmp(ciphername, "CHACHA20-POLY1305") == 0) 893 cipher = OVPN_CIPHER_ALG_CHACHA20_POLY1305; 894 else 895 return (EINVAL); 896 897 if (cipher != OVPN_CIPHER_ALG_NONE) { 898 if (! nvlist_exists_binary(nvl, "key")) 899 return (EINVAL); 900 key = nvlist_get_binary(nvl, "key", &keylen); 901 if (keylen > sizeof(kdir->key)) 902 return (E2BIG); 903 904 if (! nvlist_exists_binary(nvl, "iv")) 905 return (EINVAL); 906 iv = nvlist_get_binary(nvl, "iv", &ivlen); 907 if (ivlen != 8) 908 return (E2BIG); 909 } 910 911 kdir = malloc(sizeof(struct ovpn_kkey_dir), M_OVPN, 912 M_WAITOK | M_ZERO); 913 914 kdir->cipher = cipher; 915 kdir->keylen = keylen; 916 kdir->tx_seq = 1; 917 if (keylen != 0) 918 memcpy(kdir->key, key, keylen); 919 kdir->noncelen = ivlen; 920 if (ivlen != 0) 921 memcpy(kdir->nonce, iv, ivlen); 922 923 if (kdir->cipher != OVPN_CIPHER_ALG_NONE) { 924 /* Crypto init */ 925 bzero(&csp, sizeof(csp)); 926 csp.csp_mode = CSP_MODE_AEAD; 927 928 if (kdir->cipher == OVPN_CIPHER_ALG_CHACHA20_POLY1305) 929 csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305; 930 else 931 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 932 933 csp.csp_flags |= CSP_F_SEPARATE_AAD; 934 935 csp.csp_cipher_klen = kdir->keylen; 936 csp.csp_cipher_key = kdir->key; 937 csp.csp_ivlen = 96 / 8; 938 939 error = crypto_newsession(&kdir->cryptoid, &csp, 940 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE); 941 if (error) { 942 free(kdir, M_OVPN); 943 return (error); 944 } 945 } 946 947 mtx_init(&kdir->replay_mtx, "if_ovpn rx replay", NULL, MTX_DEF); 948 *kdirp = kdir; 949 950 return (0); 951 } 952 953 static void 954 ovpn_free_kkey_dir(struct ovpn_kkey_dir *kdir) 955 { 956 if (kdir == NULL) 957 return; 958 959 mtx_destroy(&kdir->replay_mtx); 960 961 crypto_freesession(kdir->cryptoid); 962 free(kdir, M_OVPN); 963 } 964 965 static int 966 ovpn_set_key(struct ifnet *ifp, const nvlist_t *nvl) 967 { 968 struct ovpn_softc *sc = ifp->if_softc; 969 struct ovpn_kkey_dir *enc, *dec; 970 struct ovpn_kpeer *peer; 971 int slot, keyid, peerid; 972 int error; 973 974 if (nvl == NULL) 975 return (EINVAL); 976 977 if (! nvlist_exists_number(nvl, "slot")) 978 return (EINVAL); 979 slot = nvlist_get_number(nvl, "slot"); 980 981 if (! nvlist_exists_number(nvl, "keyid")) 982 return (EINVAL); 983 keyid = nvlist_get_number(nvl, "keyid"); 984 985 if (! nvlist_exists_number(nvl, "peerid")) 986 return (EINVAL); 987 peerid = nvlist_get_number(nvl, "peerid"); 988 989 if (slot != OVPN_KEY_SLOT_PRIMARY && 990 slot != OVPN_KEY_SLOT_SECONDARY) 991 return (EINVAL); 992 993 if (! nvlist_exists_nvlist(nvl, "encrypt") || 994 ! nvlist_exists_nvlist(nvl, "decrypt")) 995 return (EINVAL); 996 997 error = ovpn_create_kkey_dir(&enc, nvlist_get_nvlist(nvl, "encrypt")); 998 if (error) 999 return (error); 1000 1001 error = ovpn_create_kkey_dir(&dec, nvlist_get_nvlist(nvl, "decrypt")); 1002 if (error) { 1003 ovpn_free_kkey_dir(enc); 1004 return (error); 1005 } 1006 1007 OVPN_WLOCK(sc); 1008 1009 peer = ovpn_find_peer(sc, peerid); 1010 if (peer == NULL) { 1011 ovpn_free_kkey_dir(dec); 1012 ovpn_free_kkey_dir(enc); 1013 OVPN_WUNLOCK(sc); 1014 return (ENOENT); 1015 } 1016 1017 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 1018 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 1019 1020 peer->keys[slot].encrypt = enc; 1021 peer->keys[slot].decrypt = dec; 1022 1023 peer->keys[slot].keyid = keyid; 1024 peer->keys[slot].peerid = peerid; 1025 1026 OVPN_WUNLOCK(sc); 1027 1028 return (0); 1029 } 1030 1031 static int 1032 ovpn_check_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, enum ovpn_key_slot slot) 1033 { 1034 OVPN_ASSERT(sc); 1035 1036 if (peer->keys[slot].encrypt == NULL) 1037 return (ENOLINK); 1038 1039 if (peer->keys[slot].decrypt == NULL) 1040 return (ENOLINK); 1041 1042 return (0); 1043 } 1044 1045 static int 1046 ovpn_start(struct ifnet *ifp) 1047 { 1048 struct ovpn_softc *sc = ifp->if_softc; 1049 1050 OVPN_WLOCK(sc); 1051 1052 ifp->if_flags |= IFF_UP; 1053 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1054 if_link_state_change(ifp, LINK_STATE_UP); 1055 1056 OVPN_WUNLOCK(sc); 1057 1058 return (0); 1059 } 1060 1061 static int 1062 ovpn_swap_keys(struct ifnet *ifp, nvlist_t *nvl) 1063 { 1064 struct ovpn_softc *sc = ifp->if_softc; 1065 struct ovpn_kpeer *peer; 1066 struct ovpn_kkey tmpkey; 1067 int error; 1068 1069 if (nvl == NULL) 1070 return (EINVAL); 1071 1072 if (! nvlist_exists_number(nvl, "peerid")) 1073 return (EINVAL); 1074 1075 OVPN_WLOCK(sc); 1076 1077 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 1078 if (peer == NULL) { 1079 OVPN_WUNLOCK(sc); 1080 return (ENOENT); 1081 } 1082 1083 /* Check that we have a second key to swap to. */ 1084 error = ovpn_check_key(sc, peer, OVPN_KEY_SLOT_SECONDARY); 1085 if (error) { 1086 OVPN_WUNLOCK(sc); 1087 return (error); 1088 } 1089 1090 tmpkey = peer->keys[0]; 1091 peer->keys[0] = peer->keys[1]; 1092 peer->keys[1] = tmpkey; 1093 1094 OVPN_WUNLOCK(sc); 1095 1096 return (0); 1097 } 1098 1099 static int 1100 ovpn_del_key(struct ifnet *ifp, const nvlist_t *nvl) 1101 { 1102 enum ovpn_key_slot slot; 1103 struct ovpn_kpeer *peer; 1104 struct ovpn_softc *sc = ifp->if_softc; 1105 1106 if (nvl == NULL) 1107 return (EINVAL); 1108 1109 if (! nvlist_exists_number(nvl, "peerid")) 1110 return (EINVAL); 1111 1112 if (! nvlist_exists_number(nvl, "slot")) 1113 return (EINVAL); 1114 slot = nvlist_get_number(nvl, "slot"); 1115 1116 if (slot != OVPN_KEY_SLOT_PRIMARY && 1117 slot != OVPN_KEY_SLOT_SECONDARY) 1118 return (EINVAL); 1119 1120 OVPN_WLOCK(sc); 1121 1122 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 1123 if (peer == NULL) { 1124 OVPN_WUNLOCK(sc); 1125 return (ENOENT); 1126 } 1127 1128 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 1129 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 1130 1131 peer->keys[slot].encrypt = NULL; 1132 peer->keys[slot].decrypt = NULL; 1133 1134 peer->keys[slot].keyid = 0; 1135 peer->keys[slot].peerid = 0; 1136 1137 OVPN_WUNLOCK(sc); 1138 1139 return (0); 1140 } 1141 1142 static void 1143 ovpn_send_ping(void *arg) 1144 { 1145 static const uint8_t ping_str[] = { 1146 0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb, 1147 0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48 1148 }; 1149 1150 struct epoch_tracker et; 1151 struct ovpn_kpeer *peer = arg; 1152 struct ovpn_softc *sc = peer->sc; 1153 struct mbuf *m; 1154 1155 OVPN_RASSERT(sc); 1156 1157 /* Ensure we repeat! */ 1158 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 1159 ovpn_send_ping, peer); 1160 1161 m = m_get2(sizeof(ping_str), M_NOWAIT, MT_DATA, M_PKTHDR); 1162 if (m == NULL) 1163 return; 1164 1165 m_copyback(m, 0, sizeof(ping_str), ping_str); 1166 m->m_len = m->m_pkthdr.len = sizeof(ping_str); 1167 1168 CURVNET_SET(sc->ifp->if_vnet); 1169 NET_EPOCH_ENTER(et); 1170 (void)ovpn_transmit_to_peer(sc->ifp, m, peer, NULL); 1171 NET_EPOCH_EXIT(et); 1172 CURVNET_RESTORE(); 1173 } 1174 1175 static void 1176 ovpn_timeout(void *arg) 1177 { 1178 struct ovpn_kpeer *peer = arg; 1179 struct ovpn_softc *sc = peer->sc; 1180 uint32_t last, _last_active; 1181 int ret __diagused; 1182 int cpu; 1183 1184 OVPN_WASSERT(sc); 1185 1186 last = 0; 1187 CPU_FOREACH(cpu) { 1188 _last_active = *zpcpu_get_cpu(peer->last_active, cpu); 1189 if (_last_active > last) 1190 last = _last_active; 1191 } 1192 1193 if (last + peer->keepalive.timeout > time_uptime) { 1194 callout_reset(&peer->ping_rcv, 1195 (peer->keepalive.timeout - (time_uptime - last)) * hz, 1196 ovpn_timeout, peer); 1197 return; 1198 } 1199 1200 CURVNET_SET(sc->ifp->if_vnet); 1201 peer->del_reason = OVPN_DEL_REASON_TIMEOUT; 1202 ret = _ovpn_del_peer(sc, peer); 1203 MPASS(ret == 0); 1204 CURVNET_RESTORE(); 1205 } 1206 1207 static int 1208 ovpn_set_peer(struct ifnet *ifp, const nvlist_t *nvl) 1209 { 1210 struct ovpn_softc *sc = ifp->if_softc; 1211 struct ovpn_kpeer *peer; 1212 1213 if (nvl == NULL) 1214 return (EINVAL); 1215 1216 if (! nvlist_exists_number(nvl, "interval") || 1217 ! nvlist_exists_number(nvl, "timeout") || 1218 ! nvlist_exists_number(nvl, "peerid")) 1219 return (EINVAL); 1220 1221 OVPN_WLOCK(sc); 1222 1223 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 1224 if (peer == NULL) { 1225 OVPN_WUNLOCK(sc); 1226 return (ENOENT); 1227 } 1228 1229 peer->keepalive.interval = nvlist_get_number(nvl, "interval"); 1230 peer->keepalive.timeout = nvlist_get_number(nvl, "timeout"); 1231 1232 if (peer->keepalive.interval > 0) 1233 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 1234 ovpn_send_ping, peer); 1235 if (peer->keepalive.timeout > 0) 1236 callout_reset(&peer->ping_rcv, peer->keepalive.timeout * hz, 1237 ovpn_timeout, peer); 1238 1239 OVPN_WUNLOCK(sc); 1240 1241 return (0); 1242 } 1243 1244 static int 1245 ovpn_set_ifmode(struct ifnet *ifp, const nvlist_t *nvl) 1246 { 1247 struct ovpn_softc *sc = ifp->if_softc; 1248 int ifmode; 1249 1250 if (nvl == NULL) 1251 return (EINVAL); 1252 1253 if (! nvlist_exists_number(nvl, "ifmode") ) 1254 return (EINVAL); 1255 1256 ifmode = nvlist_get_number(nvl, "ifmode"); 1257 1258 OVPN_WLOCK(sc); 1259 1260 /* deny this if UP */ 1261 if (ifp->if_flags & IFF_UP) { 1262 OVPN_WUNLOCK(sc); 1263 return (EBUSY); 1264 } 1265 1266 switch (ifmode & ~IFF_MULTICAST) { 1267 case IFF_POINTOPOINT: 1268 case IFF_BROADCAST: 1269 ifp->if_flags &= 1270 ~(IFF_BROADCAST|IFF_POINTOPOINT|IFF_MULTICAST); 1271 ifp->if_flags |= ifmode; 1272 break; 1273 default: 1274 OVPN_WUNLOCK(sc); 1275 return (EINVAL); 1276 } 1277 1278 OVPN_WUNLOCK(sc); 1279 1280 return (0); 1281 } 1282 1283 static int 1284 ovpn_ioctl_set(struct ifnet *ifp, struct ifdrv *ifd) 1285 { 1286 struct ovpn_softc *sc = ifp->if_softc; 1287 uint8_t *buf = NULL; 1288 nvlist_t *nvl = NULL; 1289 int ret; 1290 1291 if (ifd->ifd_len != 0) { 1292 if (ifd->ifd_len > OVPN_MAX_REQUEST_SIZE) 1293 return (E2BIG); 1294 1295 buf = malloc(ifd->ifd_len, M_OVPN, M_WAITOK); 1296 1297 ret = copyin(ifd->ifd_data, buf, ifd->ifd_len); 1298 if (ret != 0) { 1299 free(buf, M_OVPN); 1300 return (ret); 1301 } 1302 1303 nvl = nvlist_unpack(buf, ifd->ifd_len, 0); 1304 free(buf, M_OVPN); 1305 if (nvl == NULL) { 1306 return (EINVAL); 1307 } 1308 } 1309 1310 switch (ifd->ifd_cmd) { 1311 case OVPN_NEW_PEER: 1312 ret = ovpn_new_peer(ifp, nvl); 1313 break; 1314 case OVPN_DEL_PEER: 1315 OVPN_WLOCK(sc); 1316 ret = ovpn_del_peer(ifp, nvl); 1317 OVPN_WUNLOCK(sc); 1318 break; 1319 case OVPN_NEW_KEY: 1320 ret = ovpn_set_key(ifp, nvl); 1321 break; 1322 case OVPN_START_VPN: 1323 ret = ovpn_start(ifp); 1324 break; 1325 case OVPN_SWAP_KEYS: 1326 ret = ovpn_swap_keys(ifp, nvl); 1327 break; 1328 case OVPN_DEL_KEY: 1329 ret = ovpn_del_key(ifp, nvl); 1330 break; 1331 case OVPN_SET_PEER: 1332 ret = ovpn_set_peer(ifp, nvl); 1333 break; 1334 case OVPN_SET_IFMODE: 1335 ret = ovpn_set_ifmode(ifp, nvl); 1336 break; 1337 default: 1338 ret = ENOTSUP; 1339 } 1340 1341 nvlist_destroy(nvl); 1342 return (ret); 1343 } 1344 1345 static int 1346 ovpn_add_counters(nvlist_t *parent, const char *name, counter_u64_t in, 1347 counter_u64_t out) 1348 { 1349 nvlist_t *nvl; 1350 1351 nvl = nvlist_create(0); 1352 if (nvl == NULL) 1353 return (ENOMEM); 1354 1355 nvlist_add_number(nvl, "in", counter_u64_fetch(in)); 1356 nvlist_add_number(nvl, "out", counter_u64_fetch(out)); 1357 1358 nvlist_add_nvlist(parent, name, nvl); 1359 1360 nvlist_destroy(nvl); 1361 1362 return (0); 1363 } 1364 1365 static int 1366 ovpn_get_stats(struct ovpn_softc *sc, nvlist_t **onvl) 1367 { 1368 nvlist_t *nvl; 1369 int ret; 1370 1371 nvl = nvlist_create(0); 1372 if (nvl == NULL) 1373 return (ENOMEM); 1374 1375 #define OVPN_COUNTER_OUT(name, in, out) \ 1376 do { \ 1377 ret = ovpn_add_counters(nvl, name, OVPN_COUNTER(sc, in), \ 1378 OVPN_COUNTER(sc, out)); \ 1379 if (ret != 0) \ 1380 goto error; \ 1381 } while(0) 1382 1383 OVPN_COUNTER_OUT("lost_ctrl", lost_ctrl_pkts_in, lost_ctrl_pkts_out); 1384 OVPN_COUNTER_OUT("lost_data", lost_data_pkts_in, lost_data_pkts_out); 1385 OVPN_COUNTER_OUT("nomem_data", nomem_data_pkts_in, 1386 nomem_data_pkts_out); 1387 OVPN_COUNTER_OUT("data", received_data_pkts, sent_data_pkts); 1388 OVPN_COUNTER_OUT("ctrl", received_ctrl_pkts, sent_ctrl_pkts); 1389 OVPN_COUNTER_OUT("tunnel", tunnel_bytes_received, 1390 tunnel_bytes_received); 1391 OVPN_COUNTER_OUT("transport", transport_bytes_received, 1392 transport_bytes_received); 1393 #undef OVPN_COUNTER_OUT 1394 1395 *onvl = nvl; 1396 1397 return (0); 1398 1399 error: 1400 nvlist_destroy(nvl); 1401 return (ret); 1402 } 1403 1404 static int 1405 ovpn_get_peer_stats(struct ovpn_softc *sc, nvlist_t **nvl) 1406 { 1407 struct ovpn_kpeer *peer; 1408 nvlist_t *nvpeer = NULL; 1409 int ret; 1410 1411 OVPN_RLOCK_TRACKER; 1412 1413 *nvl = nvlist_create(0); 1414 if (*nvl == NULL) 1415 return (ENOMEM); 1416 1417 #define OVPN_PEER_COUNTER_OUT(name, in, out) \ 1418 do { \ 1419 ret = ovpn_add_counters(nvpeer, name, \ 1420 OVPN_PEER_COUNTER(peer, in), OVPN_PEER_COUNTER(peer, out)); \ 1421 if (ret != 0) \ 1422 goto error; \ 1423 } while(0) 1424 1425 OVPN_RLOCK(sc); 1426 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1427 nvpeer = nvlist_create(0); 1428 if (nvpeer == NULL) { 1429 OVPN_RUNLOCK(sc); 1430 nvlist_destroy(*nvl); 1431 *nvl = NULL; 1432 return (ENOMEM); 1433 } 1434 1435 nvlist_add_number(nvpeer, "peerid", peer->peerid); 1436 1437 OVPN_PEER_COUNTER_OUT("packets", pkt_in, pkt_out); 1438 OVPN_PEER_COUNTER_OUT("bytes", bytes_in, bytes_out); 1439 1440 nvlist_append_nvlist_array(*nvl, "peers", nvpeer); 1441 nvlist_destroy(nvpeer); 1442 } 1443 #undef OVPN_PEER_COUNTER_OUT 1444 OVPN_RUNLOCK(sc); 1445 1446 return (0); 1447 1448 error: 1449 nvlist_destroy(nvpeer); 1450 nvlist_destroy(*nvl); 1451 *nvl = NULL; 1452 return (ret); 1453 } 1454 1455 static int 1456 ovpn_poll_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1457 { 1458 nvlist_t *nvl; 1459 1460 nvl = nvlist_create(0); 1461 if (nvl == NULL) 1462 return (ENOMEM); 1463 1464 nvlist_add_number(nvl, "pending", buf_ring_count(sc->notifring)); 1465 1466 *onvl = nvl; 1467 1468 return (0); 1469 } 1470 1471 static void 1472 ovpn_notif_add_counters(nvlist_t *parent, struct ovpn_notification *n) 1473 { 1474 nvlist_t *nvl; 1475 1476 nvl = nvlist_create(0); 1477 if (nvl == NULL) 1478 return; 1479 1480 nvlist_add_number(nvl, "in", n->counters.pkt_in); 1481 nvlist_add_number(nvl, "out", n->counters.pkt_out); 1482 1483 nvlist_add_nvlist(parent, "packets", nvl); 1484 nvlist_destroy(nvl); 1485 1486 nvl = nvlist_create(0); 1487 if (nvl == NULL) 1488 return; 1489 1490 nvlist_add_number(nvl, "in", n->counters.bytes_in); 1491 nvlist_add_number(nvl, "out", n->counters.bytes_out); 1492 1493 nvlist_add_nvlist(parent, "bytes", nvl); 1494 nvlist_destroy(nvl); 1495 } 1496 1497 static int 1498 opvn_get_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1499 { 1500 struct ovpn_notification *n; 1501 nvlist_t *nvl; 1502 1503 /* Check if we have notifications pending. */ 1504 n = buf_ring_dequeue_mc(sc->notifring); 1505 if (n == NULL) 1506 return (ENOENT); 1507 1508 nvl = nvlist_create(0); 1509 if (nvl == NULL) { 1510 free(n, M_OVPN); 1511 return (ENOMEM); 1512 } 1513 nvlist_add_number(nvl, "peerid", n->peerid); 1514 nvlist_add_number(nvl, "notification", n->type); 1515 switch (n->type) { 1516 case OVPN_NOTIF_DEL_PEER: { 1517 nvlist_add_number(nvl, "del_reason", n->del_reason); 1518 1519 /* No error handling, because we want to send the notification 1520 * even if we can't attach the counters. */ 1521 ovpn_notif_add_counters(nvl, n); 1522 break; 1523 } 1524 case OVPN_NOTIF_FLOAT: { 1525 int ret; 1526 1527 ret = ovpn_add_sockaddr(nvl, "address", 1528 (struct sockaddr *)&n->address); 1529 1530 if (ret) { 1531 /* 1532 * Try to re-enqueue the notification. Maybe we'll 1533 * have better luck next time. No error handling, 1534 * because if we fail to re-enqueue there's nothing we can do. 1535 */ 1536 (void)ovpn_notify_float(sc, n->peerid, &n->address); 1537 nvlist_destroy(nvl); 1538 free(n, M_OVPN); 1539 return (ret); 1540 } 1541 break; 1542 } 1543 default: 1544 break; 1545 } 1546 free(n, M_OVPN); 1547 1548 *onvl = nvl; 1549 1550 return (0); 1551 } 1552 1553 static int 1554 ovpn_ioctl_get(struct ifnet *ifp, struct ifdrv *ifd) 1555 { 1556 struct ovpn_softc *sc = ifp->if_softc; 1557 nvlist_t *nvl = NULL; 1558 int error; 1559 1560 switch (ifd->ifd_cmd) { 1561 case OVPN_GET_STATS: 1562 error = ovpn_get_stats(sc, &nvl); 1563 break; 1564 case OVPN_GET_PEER_STATS: 1565 error = ovpn_get_peer_stats(sc, &nvl); 1566 break; 1567 case OVPN_POLL_PKT: 1568 error = ovpn_poll_pkt(sc, &nvl); 1569 break; 1570 case OVPN_GET_PKT: 1571 error = opvn_get_pkt(sc, &nvl); 1572 break; 1573 default: 1574 error = ENOTSUP; 1575 break; 1576 } 1577 1578 if (error == 0) { 1579 void *packed = NULL; 1580 size_t len; 1581 1582 MPASS(nvl != NULL); 1583 1584 packed = nvlist_pack(nvl, &len); 1585 if (! packed) { 1586 nvlist_destroy(nvl); 1587 return (ENOMEM); 1588 } 1589 1590 if (len > ifd->ifd_len) { 1591 free(packed, M_NVLIST); 1592 nvlist_destroy(nvl); 1593 return (ENOSPC); 1594 } 1595 1596 error = copyout(packed, ifd->ifd_data, len); 1597 ifd->ifd_len = len; 1598 1599 free(packed, M_NVLIST); 1600 nvlist_destroy(nvl); 1601 } 1602 1603 return (error); 1604 } 1605 1606 static int 1607 ovpn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 1608 { 1609 struct ifdrv *ifd; 1610 int error; 1611 1612 CURVNET_ASSERT_SET(); 1613 1614 switch (cmd) { 1615 case SIOCSDRVSPEC: 1616 case SIOCGDRVSPEC: 1617 error = priv_check(curthread, PRIV_NET_OVPN); 1618 if (error) 1619 return (error); 1620 break; 1621 } 1622 1623 switch (cmd) { 1624 case SIOCSDRVSPEC: 1625 ifd = (struct ifdrv *)data; 1626 error = ovpn_ioctl_set(ifp, ifd); 1627 break; 1628 case SIOCGDRVSPEC: 1629 ifd = (struct ifdrv *)data; 1630 error = ovpn_ioctl_get(ifp, ifd); 1631 break; 1632 case SIOCSIFMTU: { 1633 struct ifreq *ifr = (struct ifreq *)data; 1634 if (ifr->ifr_mtu < OVPN_MTU_MIN || ifr->ifr_mtu > OVPN_MTU_MAX) 1635 return (EINVAL); 1636 1637 ifp->if_mtu = ifr->ifr_mtu; 1638 return (0); 1639 } 1640 case SIOCSIFADDR: 1641 case SIOCADDMULTI: 1642 case SIOCDELMULTI: 1643 case SIOCGIFMTU: 1644 case SIOCSIFFLAGS: 1645 return (0); 1646 default: 1647 error = EINVAL; 1648 } 1649 1650 return (error); 1651 } 1652 1653 static int 1654 ovpn_encrypt_tx_cb(struct cryptop *crp) 1655 { 1656 struct epoch_tracker et; 1657 struct ovpn_kpeer *peer = crp->crp_opaque; 1658 struct ovpn_softc *sc = peer->sc; 1659 struct mbuf *m = crp->crp_buf.cb_mbuf; 1660 int tunnel_len; 1661 int ret; 1662 1663 CURVNET_SET(sc->ifp->if_vnet); 1664 NET_EPOCH_ENTER(et); 1665 1666 if (crp->crp_etype != 0) { 1667 crypto_freereq(crp); 1668 ovpn_peer_release_ref(peer, false); 1669 NET_EPOCH_EXIT(et); 1670 CURVNET_RESTORE(); 1671 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1672 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 1673 m_freem(m); 1674 return (0); 1675 } 1676 1677 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1678 1679 tunnel_len = m->m_pkthdr.len - sizeof(struct ovpn_wire_header); 1680 ret = ovpn_encap(sc, peer->peerid, m); 1681 if (ret == 0) { 1682 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1683 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 1684 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1); 1685 if_inc_counter(sc->ifp, IFCOUNTER_OBYTES, tunnel_len); 1686 } 1687 1688 crypto_freereq(crp); 1689 ovpn_peer_release_ref(peer, false); 1690 1691 NET_EPOCH_EXIT(et); 1692 CURVNET_RESTORE(); 1693 1694 return (0); 1695 } 1696 1697 static void 1698 ovpn_finish_rx(struct ovpn_softc *sc, struct mbuf *m, 1699 struct ovpn_kpeer *peer, struct ovpn_kkey *key, uint32_t seq, 1700 struct rm_priotracker *_ovpn_lock_trackerp) 1701 { 1702 uint32_t af; 1703 struct m_tag *mtag; 1704 1705 OVPN_RASSERT(sc); 1706 NET_EPOCH_ASSERT(); 1707 1708 /* Replay protection. */ 1709 if (V_replay_protection && ! ovpn_check_replay(key->decrypt, seq)) { 1710 OVPN_RUNLOCK(sc); 1711 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1712 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 1713 m_freem(m); 1714 return; 1715 } 1716 1717 critical_enter(); 1718 *zpcpu_get(peer->last_active) = time_uptime; 1719 critical_exit(); 1720 1721 OVPN_RUNLOCK(sc); 1722 1723 /* Check if the peer changed to a new source address. */ 1724 mtag = m_tag_find(m, PACKET_TAG_OVPN, NULL); 1725 if (mtag != NULL) { 1726 struct ovpn_mtag *ot = (struct ovpn_mtag *)(mtag + 1); 1727 1728 OVPN_WLOCK(sc); 1729 1730 /* 1731 * Check the address against the peer's remote again, because we may race 1732 * against ourselves (i.e. we may have tagged multiple packets to indicate we 1733 * floated). 1734 */ 1735 if (ovpn_sockaddr_compare((struct sockaddr *)&ot->addr, 1736 (struct sockaddr *)&peer->remote)) { 1737 OVPN_WUNLOCK(sc); 1738 goto skip_float; 1739 } 1740 1741 /* And notify userspace. */ 1742 if (ovpn_notify_float(sc, peer->peerid, &ot->addr) == 0) { 1743 /* 1744 * Update the 'remote' for this peer, but only if 1745 * we've actually enqueued the notification. 1746 * Otherwise we can try again later. 1747 */ 1748 memcpy(&peer->remote, &ot->addr, sizeof(peer->remote)); 1749 } 1750 1751 OVPN_WUNLOCK(sc); 1752 } 1753 1754 skip_float: 1755 OVPN_COUNTER_ADD(sc, received_data_pkts, 1); 1756 OVPN_COUNTER_ADD(sc, tunnel_bytes_received, m->m_pkthdr.len); 1757 OVPN_PEER_COUNTER_ADD(peer, pkt_in, 1); 1758 OVPN_PEER_COUNTER_ADD(peer, bytes_in, m->m_pkthdr.len); 1759 1760 /* Receive the packet on our interface. */ 1761 m->m_pkthdr.rcvif = sc->ifp; 1762 1763 /* Clear checksum flags in case the real hardware set them. */ 1764 m->m_pkthdr.csum_flags = 0; 1765 1766 /* Clear mbuf tags & flags */ 1767 m_tag_delete_nonpersistent(m); 1768 m_clrprotoflags(m); 1769 1770 /* Ensure we can read the first byte. */ 1771 m = m_pullup(m, 1); 1772 if (m == NULL) { 1773 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 1774 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 1775 return; 1776 } 1777 1778 /* 1779 * Check for address family, and disregard any control packets (e.g. 1780 * keepalive). 1781 */ 1782 af = ovpn_get_af(m); 1783 if (af != 0) { 1784 BPF_MTAP2(sc->ifp, &af, sizeof(af), m); 1785 if (V_async_netisr_queue) 1786 netisr_queue(af == AF_INET ? NETISR_IP : NETISR_IPV6, m); 1787 else 1788 netisr_dispatch(af == AF_INET ? NETISR_IP : NETISR_IPV6, m); 1789 } else { 1790 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1791 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 1792 m_freem(m); 1793 } 1794 } 1795 1796 static struct ovpn_kkey * 1797 ovpn_find_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, 1798 const struct ovpn_wire_header *ohdr) 1799 { 1800 struct ovpn_kkey *key = NULL; 1801 uint8_t keyid; 1802 1803 OVPN_RASSERT(sc); 1804 1805 keyid = (ntohl(ohdr->opcode) >> 24) & 0x07; 1806 1807 if (peer->keys[0].keyid == keyid) 1808 key = &peer->keys[0]; 1809 else if (peer->keys[1].keyid == keyid) 1810 key = &peer->keys[1]; 1811 1812 return (key); 1813 } 1814 1815 static int 1816 ovpn_decrypt_rx_cb(struct cryptop *crp) 1817 { 1818 struct epoch_tracker et; 1819 struct ovpn_softc *sc = crp->crp_opaque; 1820 struct mbuf *m = crp->crp_buf.cb_mbuf; 1821 struct ovpn_kkey *key; 1822 struct ovpn_kpeer *peer; 1823 struct ovpn_wire_header *ohdr; 1824 uint32_t peerid; 1825 1826 OVPN_RLOCK_TRACKER; 1827 1828 OVPN_RLOCK(sc); 1829 1830 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1831 1832 if (crp->crp_etype != 0) { 1833 crypto_freereq(crp); 1834 atomic_add_int(&sc->refcount, -1); 1835 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1836 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 1837 OVPN_RUNLOCK(sc); 1838 m_freem(m); 1839 return (0); 1840 } 1841 1842 CURVNET_SET(sc->ifp->if_vnet); 1843 1844 ohdr = mtodo(m, sizeof(struct udphdr)); 1845 1846 peerid = ntohl(ohdr->opcode) & 0x00ffffff; 1847 peer = ovpn_find_peer(sc, peerid); 1848 if (peer == NULL) { 1849 /* No such peer. Drop packet. */ 1850 crypto_freereq(crp); 1851 atomic_add_int(&sc->refcount, -1); 1852 OVPN_RUNLOCK(sc); 1853 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1854 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 1855 m_freem(m); 1856 CURVNET_RESTORE(); 1857 return (0); 1858 } 1859 1860 key = ovpn_find_key(sc, peer, ohdr); 1861 if (key == NULL) { 1862 crypto_freereq(crp); 1863 atomic_add_int(&sc->refcount, -1); 1864 /* 1865 * Has this key been removed between us starting the decrypt 1866 * and finishing it? 1867 */ 1868 OVPN_RUNLOCK(sc); 1869 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1870 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 1871 m_freem(m); 1872 CURVNET_RESTORE(); 1873 return (0); 1874 } 1875 1876 /* Now remove the outer headers */ 1877 m_adj_decap(m, sizeof(struct udphdr) + 1878 sizeof(struct ovpn_wire_header)); 1879 1880 NET_EPOCH_ENTER(et); 1881 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), _ovpn_lock_trackerp); 1882 NET_EPOCH_EXIT(et); 1883 OVPN_UNLOCK_ASSERT(sc); 1884 1885 CURVNET_RESTORE(); 1886 1887 crypto_freereq(crp); 1888 atomic_add_int(&sc->refcount, -1); 1889 1890 return (0); 1891 } 1892 1893 static int 1894 ovpn_get_af(struct mbuf *m) 1895 { 1896 struct ip *ip; 1897 struct ip6_hdr *ip6; 1898 1899 /* 1900 * We should pullup, but we're only interested in the first byte, so 1901 * that'll always be contiguous. 1902 */ 1903 ip = mtod(m, struct ip *); 1904 if (ip->ip_v == IPVERSION) 1905 return (AF_INET); 1906 1907 ip6 = mtod(m, struct ip6_hdr *); 1908 if ((ip6->ip6_vfc & IPV6_VERSION_MASK) == IPV6_VERSION) 1909 return (AF_INET6); 1910 1911 return (0); 1912 } 1913 1914 #ifdef INET 1915 static struct ovpn_kpeer * 1916 ovpn_find_peer_by_ip(struct ovpn_softc *sc, const struct in_addr addr) 1917 { 1918 struct ovpn_kpeer *peer = NULL; 1919 1920 OVPN_ASSERT(sc); 1921 1922 /* TODO: Add a second RB so we can look up by IP. */ 1923 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1924 if (addr.s_addr == peer->vpn4.s_addr) 1925 return (peer); 1926 } 1927 1928 return (peer); 1929 } 1930 #endif 1931 1932 #ifdef INET6 1933 static struct ovpn_kpeer * 1934 ovpn_find_peer_by_ip6(struct ovpn_softc *sc, const struct in6_addr *addr) 1935 { 1936 struct ovpn_kpeer *peer = NULL; 1937 1938 OVPN_ASSERT(sc); 1939 1940 /* TODO: Add a third RB so we can look up by IPv6 address. */ 1941 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1942 if (memcmp(addr, &peer->vpn6, sizeof(*addr)) == 0) 1943 return (peer); 1944 } 1945 1946 return (peer); 1947 } 1948 #endif 1949 1950 static struct ovpn_kpeer * 1951 ovpn_route_peer(struct ovpn_softc *sc, struct mbuf **m0, 1952 const struct sockaddr *dst) 1953 { 1954 struct ovpn_kpeer *peer = NULL; 1955 int af; 1956 1957 NET_EPOCH_ASSERT(); 1958 OVPN_ASSERT(sc); 1959 1960 /* Shortcut if we're a client (or are a server and have only one client). */ 1961 if (sc->peercount == 1) 1962 return (ovpn_find_only_peer(sc)); 1963 1964 if (dst != NULL) 1965 af = dst->sa_family; 1966 else 1967 af = ovpn_get_af(*m0); 1968 1969 switch (af) { 1970 #ifdef INET 1971 case AF_INET: { 1972 const struct sockaddr_in *sa = (const struct sockaddr_in *)dst; 1973 struct nhop_object *nh; 1974 const struct in_addr *ip_dst; 1975 1976 if (sa != NULL) { 1977 ip_dst = &sa->sin_addr; 1978 } else { 1979 struct ip *ip; 1980 1981 *m0 = m_pullup(*m0, sizeof(struct ip)); 1982 if (*m0 == NULL) 1983 return (NULL); 1984 ip = mtod(*m0, struct ip *); 1985 ip_dst = &ip->ip_dst; 1986 } 1987 1988 peer = ovpn_find_peer_by_ip(sc, *ip_dst); 1989 SDT_PROBE2(if_ovpn, tx, route, ip4, ip_dst, peer); 1990 if (peer == NULL) { 1991 nh = fib4_lookup(M_GETFIB(*m0), *ip_dst, 0, 1992 NHR_NONE, 0); 1993 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1994 peer = ovpn_find_peer_by_ip(sc, 1995 nh->gw4_sa.sin_addr); 1996 SDT_PROBE2(if_ovpn, tx, route, ip4, 1997 &nh->gw4_sa.sin_addr, peer); 1998 } 1999 } 2000 break; 2001 } 2002 #endif 2003 #ifdef INET6 2004 case AF_INET6: { 2005 const struct sockaddr_in6 *sa6 = 2006 (const struct sockaddr_in6 *)dst; 2007 struct nhop_object *nh; 2008 const struct in6_addr *ip6_dst; 2009 2010 if (sa6 != NULL) { 2011 ip6_dst = &sa6->sin6_addr; 2012 } else { 2013 struct ip6_hdr *ip6; 2014 2015 *m0 = m_pullup(*m0, sizeof(struct ip6_hdr)); 2016 if (*m0 == NULL) 2017 return (NULL); 2018 ip6 = mtod(*m0, struct ip6_hdr *); 2019 ip6_dst = &ip6->ip6_dst; 2020 } 2021 2022 peer = ovpn_find_peer_by_ip6(sc, ip6_dst); 2023 SDT_PROBE2(if_ovpn, tx, route, ip6, ip6_dst, peer); 2024 if (peer == NULL) { 2025 nh = fib6_lookup(M_GETFIB(*m0), ip6_dst, 0, 2026 NHR_NONE, 0); 2027 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 2028 peer = ovpn_find_peer_by_ip6(sc, 2029 &nh->gw6_sa.sin6_addr); 2030 SDT_PROBE2(if_ovpn, tx, route, ip6, 2031 &nh->gw6_sa.sin6_addr, peer); 2032 } 2033 } 2034 break; 2035 } 2036 #endif 2037 } 2038 2039 return (peer); 2040 } 2041 2042 static int 2043 ovpn_transmit(struct ifnet *ifp, struct mbuf *m) 2044 { 2045 return (ifp->if_output(ifp, m, NULL, NULL)); 2046 } 2047 2048 static int 2049 ovpn_transmit_to_peer(struct ifnet *ifp, struct mbuf *m, 2050 struct ovpn_kpeer *peer, struct rm_priotracker *_ovpn_lock_trackerp) 2051 { 2052 struct ovpn_wire_header *ohdr; 2053 struct ovpn_kkey *key; 2054 struct ovpn_softc *sc; 2055 struct cryptop *crp; 2056 uint32_t af, seq; 2057 uint64_t seq64; 2058 size_t len, ovpn_hdr_len; 2059 int tunnel_len; 2060 int ret; 2061 2062 sc = ifp->if_softc; 2063 2064 OVPN_RASSERT(sc); 2065 2066 tunnel_len = m->m_pkthdr.len; 2067 2068 key = &peer->keys[OVPN_KEY_SLOT_PRIMARY]; 2069 if (key->encrypt == NULL) { 2070 if (_ovpn_lock_trackerp != NULL) 2071 OVPN_RUNLOCK(sc); 2072 m_freem(m); 2073 return (ENOLINK); 2074 } 2075 2076 af = ovpn_get_af(m); 2077 /* Don't capture control packets. */ 2078 if (af != 0) 2079 BPF_MTAP2(ifp, &af, sizeof(af), m); 2080 2081 if (__predict_false(if_tunnel_check_nesting(ifp, m, MTAG_OVPN_LOOP, 3))) { 2082 if (_ovpn_lock_trackerp != NULL) 2083 OVPN_RUNLOCK(sc); 2084 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2085 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2086 m_freem(m); 2087 return (ELOOP); 2088 } 2089 2090 len = m->m_pkthdr.len; 2091 MPASS(len <= ifp->if_mtu); 2092 2093 ovpn_hdr_len = sizeof(struct ovpn_wire_header); 2094 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) 2095 ovpn_hdr_len -= 16; /* No auth tag. */ 2096 2097 M_PREPEND(m, ovpn_hdr_len, M_NOWAIT); 2098 if (m == NULL) { 2099 if (_ovpn_lock_trackerp != NULL) 2100 OVPN_RUNLOCK(sc); 2101 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2102 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 2103 return (ENOBUFS); 2104 } 2105 ohdr = mtod(m, struct ovpn_wire_header *); 2106 ohdr->opcode = (OVPN_OP_DATA_V2 << OVPN_OP_SHIFT) | key->keyid; 2107 ohdr->opcode <<= 24; 2108 ohdr->opcode |= key->peerid; 2109 ohdr->opcode = htonl(ohdr->opcode); 2110 2111 seq64 = atomic_fetchadd_64(&peer->keys[OVPN_KEY_SLOT_PRIMARY].encrypt->tx_seq, 1); 2112 if (seq64 == OVPN_SEQ_ROTATE) { 2113 ovpn_notify_key_rotation(sc, peer); 2114 } else if (seq64 > UINT32_MAX) { 2115 /* We've wrapped, give up on this packet. */ 2116 if (_ovpn_lock_trackerp != NULL) 2117 OVPN_RUNLOCK(sc); 2118 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2119 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 2120 2121 /* Let's avoid (very unlikely, but still) wraparounds of the 2122 * 64-bit counter taking us back to 0. */ 2123 atomic_store_64(&peer->keys[OVPN_KEY_SLOT_PRIMARY].encrypt->tx_seq, 2124 UINT32_MAX); 2125 2126 return (ENOBUFS); 2127 } 2128 2129 seq = htonl(seq64 & UINT32_MAX); 2130 ohdr->seq = seq; 2131 2132 OVPN_PEER_COUNTER_ADD(peer, pkt_out, 1); 2133 OVPN_PEER_COUNTER_ADD(peer, bytes_out, len); 2134 2135 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) { 2136 ret = ovpn_encap(sc, peer->peerid, m); 2137 if (_ovpn_lock_trackerp != NULL) 2138 OVPN_RUNLOCK(sc); 2139 if (ret == 0) { 2140 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 2141 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 2142 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1); 2143 if_inc_counter(sc->ifp, IFCOUNTER_OBYTES, tunnel_len); 2144 } 2145 return (ret); 2146 } 2147 2148 crp = crypto_getreq(key->encrypt->cryptoid, M_NOWAIT); 2149 if (crp == NULL) { 2150 if (_ovpn_lock_trackerp != NULL) 2151 OVPN_RUNLOCK(sc); 2152 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2153 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 2154 m_freem(m); 2155 return (ENOBUFS); 2156 } 2157 2158 /* Encryption covers only the payload, not the header. */ 2159 crp->crp_payload_start = sizeof(*ohdr); 2160 crp->crp_payload_length = len; 2161 crp->crp_op = CRYPTO_OP_ENCRYPT; 2162 2163 /* 2164 * AAD data covers the ovpn_wire_header minus the auth 2165 * tag. 2166 */ 2167 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2168 crp->crp_aad = ohdr; 2169 crp->crp_aad_start = 0; 2170 crp->crp_op |= CRYPTO_OP_COMPUTE_DIGEST; 2171 crp->crp_digest_start = offsetof(struct ovpn_wire_header, auth_tag); 2172 2173 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 2174 memcpy(crp->crp_iv, &seq, sizeof(seq)); 2175 memcpy(crp->crp_iv + sizeof(seq), key->encrypt->nonce, 2176 key->encrypt->noncelen); 2177 2178 crypto_use_mbuf(crp, m); 2179 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 2180 crp->crp_callback = ovpn_encrypt_tx_cb; 2181 crp->crp_opaque = peer; 2182 2183 atomic_add_int(&peer->refcount, 1); 2184 if (_ovpn_lock_trackerp != NULL) 2185 OVPN_RUNLOCK(sc); 2186 if (V_async_crypto) 2187 ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); 2188 else 2189 ret = crypto_dispatch(crp); 2190 if (ret) { 2191 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2192 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 2193 } 2194 2195 return (ret); 2196 } 2197 2198 /* 2199 * Note: Expects to hold the read lock on entry, and will release it itself. 2200 */ 2201 static int 2202 ovpn_encap(struct ovpn_softc *sc, uint32_t peerid, struct mbuf *m) 2203 { 2204 struct udphdr *udp; 2205 struct ovpn_kpeer *peer; 2206 int len; 2207 2208 OVPN_RLOCK_TRACKER; 2209 2210 OVPN_RLOCK(sc); 2211 NET_EPOCH_ASSERT(); 2212 2213 peer = ovpn_find_peer(sc, peerid); 2214 if (peer == NULL || sc->ifp->if_link_state != LINK_STATE_UP) { 2215 OVPN_RUNLOCK(sc); 2216 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2217 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 2218 m_freem(m); 2219 return (ENETDOWN); 2220 } 2221 2222 len = m->m_pkthdr.len; 2223 2224 M_PREPEND(m, sizeof(struct udphdr), M_NOWAIT); 2225 if (m == NULL) { 2226 OVPN_RUNLOCK(sc); 2227 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2228 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 2229 m_freem(m); 2230 return (ENOBUFS); 2231 } 2232 udp = mtod(m, struct udphdr *); 2233 2234 MPASS(peer->local.ss_family == peer->remote.ss_family); 2235 2236 udp->uh_sport = ovpn_get_port(&peer->local); 2237 udp->uh_dport = ovpn_get_port(&peer->remote); 2238 udp->uh_ulen = htons(sizeof(struct udphdr) + len); 2239 2240 switch (peer->remote.ss_family) { 2241 #ifdef INET 2242 case AF_INET: { 2243 struct sockaddr_in *in_local = TO_IN(&peer->local); 2244 struct sockaddr_in *in_remote = TO_IN(&peer->remote); 2245 struct ip *ip; 2246 2247 /* 2248 * This requires knowing the source IP, which we don't. Happily 2249 * we're allowed to keep this at 0, and the checksum won't do 2250 * anything the crypto won't already do. 2251 */ 2252 udp->uh_sum = 0; 2253 2254 /* Set the checksum flags so we recalculate checksums. */ 2255 m->m_pkthdr.csum_flags |= CSUM_IP; 2256 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 2257 2258 M_PREPEND(m, sizeof(struct ip), M_NOWAIT); 2259 if (m == NULL) { 2260 OVPN_RUNLOCK(sc); 2261 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2262 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 2263 return (ENOBUFS); 2264 } 2265 ip = mtod(m, struct ip *); 2266 2267 ip->ip_tos = 0; 2268 ip->ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) + 2269 len); 2270 ip->ip_off = 0; 2271 ip->ip_ttl = V_ip_defttl; 2272 ip->ip_p = IPPROTO_UDP; 2273 ip->ip_sum = 0; 2274 if (in_local->sin_port != 0) 2275 ip->ip_src = in_local->sin_addr; 2276 else 2277 ip->ip_src.s_addr = INADDR_ANY; 2278 ip->ip_dst = in_remote->sin_addr; 2279 2280 OVPN_RUNLOCK(sc); 2281 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 2282 2283 return (ip_output(m, NULL, NULL, 0, NULL, NULL)); 2284 } 2285 #endif 2286 #ifdef INET6 2287 case AF_INET6: { 2288 struct sockaddr_in6 *in6_local = TO_IN6(&peer->local); 2289 struct sockaddr_in6 *in6_remote = TO_IN6(&peer->remote); 2290 struct ip6_hdr *ip6; 2291 2292 M_PREPEND(m, sizeof(struct ip6_hdr), M_NOWAIT); 2293 if (m == NULL) { 2294 OVPN_RUNLOCK(sc); 2295 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2296 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 2297 return (ENOBUFS); 2298 } 2299 m = m_pullup(m, sizeof(*ip6) + sizeof(*udp)); 2300 if (m == NULL) { 2301 OVPN_RUNLOCK(sc); 2302 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2303 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 2304 return (ENOBUFS); 2305 } 2306 2307 ip6 = mtod(m, struct ip6_hdr *); 2308 2309 ip6->ip6_vfc = IPV6_VERSION; 2310 ip6->ip6_flow &= ~IPV6_FLOWINFO_MASK; 2311 ip6->ip6_plen = htons(sizeof(*ip6) + sizeof(struct udphdr) + 2312 len); 2313 ip6->ip6_nxt = IPPROTO_UDP; 2314 ip6->ip6_hlim = V_ip6_defhlim; 2315 2316 memcpy(&ip6->ip6_src, &in6_local->sin6_addr, 2317 sizeof(ip6->ip6_src)); 2318 memcpy(&ip6->ip6_dst, &in6_remote->sin6_addr, 2319 sizeof(ip6->ip6_dst)); 2320 2321 if (IN6_IS_ADDR_LINKLOCAL(&ip6->ip6_src)) { 2322 /* Local and remote must have the same scope. */ 2323 ip6->ip6_src.__u6_addr.__u6_addr16[1] = 2324 htons(in6_remote->sin6_scope_id & 0xffff); 2325 } 2326 if (IN6_IS_ADDR_LINKLOCAL(&ip6->ip6_dst)) 2327 ip6->ip6_dst.__u6_addr.__u6_addr16[1] = 2328 htons(in6_remote->sin6_scope_id & 0xffff); 2329 2330 udp = mtodo(m, sizeof(*ip6)); 2331 udp->uh_sum = in6_cksum_pseudo(ip6, 2332 m->m_pkthdr.len - sizeof(struct ip6_hdr), 2333 IPPROTO_UDP, 0); 2334 2335 m->m_pkthdr.csum_flags |= CSUM_UDP_IPV6; 2336 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 2337 2338 OVPN_RUNLOCK(sc); 2339 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 2340 2341 return (ip6_output(m, NULL, NULL, IPV6_UNSPECSRC, NULL, NULL, 2342 NULL)); 2343 } 2344 #endif 2345 default: 2346 panic("Unsupported address family %d", 2347 peer->remote.ss_family); 2348 } 2349 } 2350 2351 static int 2352 ovpn_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 2353 struct route *ro) 2354 { 2355 struct ovpn_softc *sc; 2356 struct ovpn_kpeer *peer; 2357 2358 OVPN_RLOCK_TRACKER; 2359 2360 sc = ifp->if_softc; 2361 2362 m = m_unshare(m, M_NOWAIT); 2363 if (m == NULL) { 2364 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2365 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 2366 return (ENOBUFS); 2367 } 2368 2369 OVPN_RLOCK(sc); 2370 2371 SDT_PROBE1(if_ovpn, tx, transmit, start, m); 2372 2373 if (__predict_false(ifp->if_link_state != LINK_STATE_UP)) { 2374 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2375 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 2376 OVPN_RUNLOCK(sc); 2377 m_freem(m); 2378 return (ENETDOWN); 2379 } 2380 2381 /** 2382 * Only obey 'dst' (i.e. the gateway) if no route is supplied. 2383 * That's our indication that we're being called through pf's route-to, 2384 * and we should route according to 'dst' instead. We can't do so 2385 * consistently, because the usual openvpn configuration sets the first 2386 * non-server IP in the subnet as the gateway. If we always use that 2387 * one we'd end up routing all traffic to the first client. 2388 * tl;dr: 'ro == NULL' tells us pf is doing a route-to, and then but 2389 * only then, we should treat 'dst' as the destination. */ 2390 peer = ovpn_route_peer(sc, &m, ro == NULL ? dst : NULL); 2391 if (peer == NULL) { 2392 /* No destination. */ 2393 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2394 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 2395 OVPN_RUNLOCK(sc); 2396 m_freem(m); 2397 return (ENETDOWN); 2398 } 2399 2400 return (ovpn_transmit_to_peer(ifp, m, peer, _ovpn_lock_trackerp)); 2401 } 2402 2403 static bool 2404 ovpn_check_replay(struct ovpn_kkey_dir *key, uint32_t seq) 2405 { 2406 uint32_t d; 2407 2408 mtx_lock(&key->replay_mtx); 2409 2410 /* Sequence number must be strictly greater than rx_seq */ 2411 if (seq <= key->rx_seq) { 2412 mtx_unlock(&key->replay_mtx); 2413 return (false); 2414 } 2415 2416 /* Large jump. The packet authenticated okay, so just accept that. */ 2417 if (seq > (key->rx_seq + (sizeof(key->rx_window) * 8))) { 2418 key->rx_seq = seq; 2419 key->rx_window = 0; 2420 mtx_unlock(&key->replay_mtx); 2421 return (true); 2422 } 2423 2424 /* Happy case. */ 2425 if ((seq == key->rx_seq + 1) && key->rx_window == 0) { 2426 key->rx_seq++; 2427 mtx_unlock(&key->replay_mtx); 2428 return (true); 2429 } 2430 2431 d = seq - key->rx_seq - 1; 2432 2433 if (key->rx_window & ((uint64_t)1 << d)) { 2434 /* Dupe! */ 2435 mtx_unlock(&key->replay_mtx); 2436 return (false); 2437 } 2438 2439 key->rx_window |= (uint64_t)1 << d; 2440 2441 while (key->rx_window & 1) { 2442 key->rx_seq++; 2443 key->rx_window >>= 1; 2444 } 2445 2446 mtx_unlock(&key->replay_mtx); 2447 2448 return (true); 2449 } 2450 2451 static struct ovpn_kpeer * 2452 ovpn_peer_from_mbuf(struct ovpn_softc *sc, struct mbuf *m, int off) 2453 { 2454 struct ovpn_wire_header ohdr; 2455 uint32_t peerid; 2456 const size_t hdrlen = sizeof(ohdr) - sizeof(ohdr.auth_tag); 2457 2458 OVPN_RASSERT(sc); 2459 2460 if (m_length(m, NULL) < (off + sizeof(struct udphdr) + hdrlen)) 2461 return (NULL); 2462 2463 m_copydata(m, off + sizeof(struct udphdr), hdrlen, (caddr_t)&ohdr); 2464 2465 peerid = ntohl(ohdr.opcode) & 0x00ffffff; 2466 2467 return (ovpn_find_peer(sc, peerid)); 2468 } 2469 2470 static bool 2471 ovpn_udp_input(struct mbuf *m, int off, struct inpcb *inp, 2472 const struct sockaddr *sa, void *ctx) 2473 { 2474 struct ovpn_softc *sc = ctx; 2475 struct ovpn_wire_header tmphdr; 2476 struct ovpn_wire_header *ohdr; 2477 struct udphdr *uhdr; 2478 struct ovpn_kkey *key; 2479 struct cryptop *crp; 2480 struct ovpn_kpeer *peer; 2481 size_t ohdrlen; 2482 int ret; 2483 uint8_t op; 2484 2485 OVPN_RLOCK_TRACKER; 2486 2487 M_ASSERTPKTHDR(m); 2488 2489 OVPN_COUNTER_ADD(sc, transport_bytes_received, m->m_pkthdr.len - off); 2490 if_inc_counter(sc->ifp, IFCOUNTER_IBYTES, m->m_pkthdr.len - off); 2491 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 2492 2493 ohdrlen = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2494 2495 OVPN_RLOCK(sc); 2496 2497 peer = ovpn_peer_from_mbuf(sc, m, off); 2498 if (peer == NULL) { 2499 OVPN_RUNLOCK(sc); 2500 return (false); 2501 } 2502 2503 if (m_length(m, NULL) < (off + sizeof(*uhdr) + ohdrlen)) { 2504 /* Short packet. */ 2505 OVPN_RUNLOCK(sc); 2506 return (false); 2507 } 2508 2509 m_copydata(m, off + sizeof(*uhdr), ohdrlen, (caddr_t)&tmphdr); 2510 2511 op = ntohl(tmphdr.opcode) >> 24 >> OVPN_OP_SHIFT; 2512 if (op != OVPN_OP_DATA_V2) { 2513 /* Control packet? */ 2514 OVPN_RUNLOCK(sc); 2515 return (false); 2516 } 2517 2518 m = m_unshare(m, M_NOWAIT); 2519 if (m == NULL) { 2520 OVPN_RUNLOCK(sc); 2521 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2522 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2523 return (true); 2524 } 2525 2526 m = m_pullup(m, off + sizeof(*uhdr) + ohdrlen); 2527 if (m == NULL) { 2528 OVPN_RUNLOCK(sc); 2529 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2530 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2531 return (true); 2532 } 2533 2534 /* 2535 * Simplify things by getting rid of the preceding headers, we don't 2536 * care about them. 2537 */ 2538 m_adj_decap(m, off); 2539 2540 uhdr = mtodo(m, 0); 2541 ohdr = mtodo(m, sizeof(*uhdr)); 2542 2543 key = ovpn_find_key(sc, peer, ohdr); 2544 if (key == NULL || key->decrypt == NULL) { 2545 OVPN_RUNLOCK(sc); 2546 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2547 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2548 m_freem(m); 2549 return (true); 2550 } 2551 2552 /* 2553 * If we got this from a different address than we expected tag the packet. 2554 * We'll deal with notifiying userspace later, after we've decrypted and 2555 * verified. 2556 */ 2557 if (! ovpn_sockaddr_compare((struct sockaddr *)&peer->remote, sa)) { 2558 struct m_tag *mt; 2559 struct ovpn_mtag *ot; 2560 2561 MPASS(sa->sa_len <= sizeof(ot->addr)); 2562 mt = m_tag_get(PACKET_TAG_OVPN, sizeof(*ot), M_NOWAIT); 2563 /* 2564 * If we fail to allocate here we'll just try again on the next 2565 * packet. 2566 */ 2567 if (mt != NULL) { 2568 ot = (struct ovpn_mtag *)(mt + 1); 2569 memcpy(&ot->addr, sa, sa->sa_len); 2570 2571 m_tag_prepend(m, mt); 2572 } 2573 } 2574 2575 if (key->decrypt->cipher == OVPN_CIPHER_ALG_NONE) { 2576 /* Now remove the outer headers */ 2577 m_adj_decap(m, sizeof(struct udphdr) + ohdrlen); 2578 2579 ohdr = mtodo(m, sizeof(*uhdr)); 2580 2581 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), 2582 _ovpn_lock_trackerp); 2583 OVPN_UNLOCK_ASSERT(sc); 2584 return (true); 2585 } 2586 2587 ohdrlen += sizeof(ohdr->auth_tag); 2588 2589 m = m_pullup(m, sizeof(*uhdr) + ohdrlen); 2590 if (m == NULL) { 2591 OVPN_RUNLOCK(sc); 2592 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2593 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2594 return (true); 2595 } 2596 uhdr = mtodo(m, 0); 2597 ohdr = mtodo(m, sizeof(*uhdr)); 2598 2599 /* Decrypt */ 2600 crp = crypto_getreq(key->decrypt->cryptoid, M_NOWAIT); 2601 if (crp == NULL) { 2602 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2603 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2604 OVPN_RUNLOCK(sc); 2605 m_freem(m); 2606 return (true); 2607 } 2608 2609 crp->crp_payload_start = sizeof(struct udphdr) + sizeof(*ohdr); 2610 crp->crp_payload_length = ntohs(uhdr->uh_ulen) - 2611 sizeof(*uhdr) - sizeof(*ohdr); 2612 crp->crp_op = CRYPTO_OP_DECRYPT; 2613 2614 /* AAD validation. */ 2615 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2616 crp->crp_aad = ohdr; 2617 crp->crp_aad_start = 0; 2618 crp->crp_op |= CRYPTO_OP_VERIFY_DIGEST; 2619 crp->crp_digest_start = sizeof(struct udphdr) + 2620 offsetof(struct ovpn_wire_header, auth_tag); 2621 2622 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 2623 memcpy(crp->crp_iv, &ohdr->seq, sizeof(ohdr->seq)); 2624 memcpy(crp->crp_iv + sizeof(ohdr->seq), key->decrypt->nonce, 2625 key->decrypt->noncelen); 2626 2627 crypto_use_mbuf(crp, m); 2628 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 2629 crp->crp_callback = ovpn_decrypt_rx_cb; 2630 crp->crp_opaque = sc; 2631 2632 atomic_add_int(&sc->refcount, 1); 2633 OVPN_RUNLOCK(sc); 2634 if (V_async_crypto) 2635 ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); 2636 else 2637 ret = crypto_dispatch(crp); 2638 if (ret != 0) { 2639 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2640 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2641 } 2642 2643 return (true); 2644 } 2645 2646 static void 2647 ovpn_qflush(struct ifnet *ifp __unused) 2648 { 2649 2650 } 2651 2652 static void 2653 ovpn_flush_rxring(struct ovpn_softc *sc) 2654 { 2655 struct ovpn_notification *n; 2656 2657 OVPN_WASSERT(sc); 2658 2659 while (! buf_ring_empty(sc->notifring)) { 2660 n = buf_ring_dequeue_sc(sc->notifring); 2661 free(n, M_OVPN); 2662 } 2663 } 2664 2665 #ifdef VIMAGE 2666 static void 2667 ovpn_reassign(struct ifnet *ifp, struct vnet *new_vnet __unused, 2668 char *unused __unused) 2669 { 2670 struct ovpn_softc *sc = ifp->if_softc; 2671 struct ovpn_kpeer *peer, *tmppeer; 2672 int ret __diagused; 2673 2674 OVPN_WLOCK(sc); 2675 2676 /* Flush keys & configuration. */ 2677 RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) { 2678 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 2679 ret = _ovpn_del_peer(sc, peer); 2680 MPASS(ret == 0); 2681 } 2682 2683 ovpn_flush_rxring(sc); 2684 2685 OVPN_WUNLOCK(sc); 2686 } 2687 #endif 2688 2689 static int 2690 ovpn_clone_match(struct if_clone *ifc, const char *name) 2691 { 2692 /* 2693 * Allow all names that start with 'ovpn', specifically because pfSense 2694 * uses ovpnc1 / ovpns2 2695 */ 2696 return (strncmp(ovpnname, name, strlen(ovpnname)) == 0); 2697 } 2698 2699 static int 2700 ovpn_clone_create(struct if_clone *ifc, char *name, size_t len, 2701 struct ifc_data *ifd, struct ifnet **ifpp) 2702 { 2703 struct ovpn_softc *sc; 2704 struct ifnet *ifp; 2705 char *dp; 2706 int error, unit, wildcard; 2707 2708 /* Try to see if a special unit was requested. */ 2709 error = ifc_name2unit(name, &unit); 2710 if (error != 0) 2711 return (error); 2712 wildcard = (unit < 0); 2713 2714 error = ifc_alloc_unit(ifc, &unit); 2715 if (error != 0) 2716 return (error); 2717 2718 /* 2719 * If no unit had been given, we need to adjust the ifName. 2720 */ 2721 for (dp = name; *dp != '\0'; dp++); 2722 if (wildcard) { 2723 error = snprintf(dp, len - (dp - name), "%d", unit); 2724 if (error > len - (dp - name)) { 2725 /* ifName too long. */ 2726 ifc_free_unit(ifc, unit); 2727 return (ENOSPC); 2728 } 2729 dp += error; 2730 } 2731 2732 /* Make sure it doesn't already exist. */ 2733 if (ifunit(name) != NULL) 2734 return (EEXIST); 2735 2736 sc = malloc(sizeof(struct ovpn_softc), M_OVPN, M_WAITOK | M_ZERO); 2737 sc->ifp = if_alloc(IFT_TUNNEL); 2738 rm_init_flags(&sc->lock, "if_ovpn_lock", RM_RECURSE); 2739 sc->refcount = 0; 2740 2741 sc->notifring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL); 2742 2743 COUNTER_ARRAY_ALLOC(sc->counters, OVPN_COUNTER_SIZE, M_WAITOK); 2744 2745 ifp = sc->ifp; 2746 ifp->if_softc = sc; 2747 strlcpy(ifp->if_xname, name, IFNAMSIZ); 2748 ifp->if_dname = ovpngroupname; 2749 ifp->if_dunit = unit; 2750 2751 ifp->if_addrlen = 0; 2752 ifp->if_mtu = 1428; 2753 ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST; 2754 ifp->if_ioctl = ovpn_ioctl; 2755 ifp->if_transmit = ovpn_transmit; 2756 ifp->if_output = ovpn_output; 2757 ifp->if_qflush = ovpn_qflush; 2758 #ifdef VIMAGE 2759 ifp->if_reassign = ovpn_reassign; 2760 #endif 2761 ifp->if_capabilities |= IFCAP_LINKSTATE; 2762 ifp->if_capenable |= IFCAP_LINKSTATE; 2763 2764 if_attach(ifp); 2765 bpfattach(ifp, DLT_NULL, sizeof(uint32_t)); 2766 *ifpp = ifp; 2767 2768 return (0); 2769 } 2770 2771 static void 2772 ovpn_clone_destroy_cb(struct epoch_context *ctx) 2773 { 2774 struct ovpn_softc *sc; 2775 int ret __diagused; 2776 2777 sc = __containerof(ctx, struct ovpn_softc, epoch_ctx); 2778 2779 MPASS(sc->peercount == 0); 2780 MPASS(RB_EMPTY(&sc->peers)); 2781 2782 if (sc->so != NULL) { 2783 CURVNET_SET(sc->ifp->if_vnet); 2784 ret = udp_set_kernel_tunneling(sc->so, NULL, NULL, NULL); 2785 MPASS(ret == 0); 2786 sorele(sc->so); 2787 CURVNET_RESTORE(); 2788 } 2789 2790 COUNTER_ARRAY_FREE(sc->counters, OVPN_COUNTER_SIZE); 2791 2792 rm_destroy(&sc->lock); 2793 if_free(sc->ifp); 2794 free(sc, M_OVPN); 2795 } 2796 2797 static int 2798 ovpn_clone_destroy(struct if_clone *ifc, struct ifnet *ifp, uint32_t flags) 2799 { 2800 struct ovpn_softc *sc; 2801 struct ovpn_kpeer *peer, *tmppeer; 2802 int unit; 2803 int ret __diagused; 2804 2805 sc = ifp->if_softc; 2806 unit = ifp->if_dunit; 2807 2808 OVPN_WLOCK(sc); 2809 2810 if (atomic_load_int(&sc->refcount) > 0) { 2811 OVPN_WUNLOCK(sc); 2812 return (EBUSY); 2813 } 2814 2815 RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) { 2816 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 2817 ret = _ovpn_del_peer(sc, peer); 2818 MPASS(ret == 0); 2819 } 2820 2821 ovpn_flush_rxring(sc); 2822 buf_ring_free(sc->notifring, M_OVPN); 2823 2824 OVPN_WUNLOCK(sc); 2825 2826 bpfdetach(ifp); 2827 if_detach(ifp); 2828 ifp->if_softc = NULL; 2829 2830 NET_EPOCH_CALL(ovpn_clone_destroy_cb, &sc->epoch_ctx); 2831 2832 if (unit != IF_DUNIT_NONE) 2833 ifc_free_unit(ifc, unit); 2834 2835 NET_EPOCH_DRAIN_CALLBACKS(); 2836 2837 return (0); 2838 } 2839 2840 static void 2841 vnet_ovpn_init(const void *unused __unused) 2842 { 2843 struct if_clone_addreq req = { 2844 .match_f = ovpn_clone_match, 2845 .create_f = ovpn_clone_create, 2846 .destroy_f = ovpn_clone_destroy, 2847 }; 2848 V_ovpn_cloner = ifc_attach_cloner(ovpngroupname, &req); 2849 } 2850 VNET_SYSINIT(vnet_ovpn_init, SI_SUB_PSEUDO, SI_ORDER_ANY, 2851 vnet_ovpn_init, NULL); 2852 2853 static int 2854 ovpn_prison_remove(void *obj, void *data __unused) 2855 { 2856 #ifdef VIMAGE 2857 struct prison *pr; 2858 2859 pr = obj; 2860 if (prison_owns_vnet(pr)) { 2861 CURVNET_SET(pr->pr_vnet); 2862 if (V_ovpn_cloner != NULL) { 2863 ifc_detach_cloner(V_ovpn_cloner); 2864 V_ovpn_cloner = NULL; 2865 } 2866 CURVNET_RESTORE(); 2867 } 2868 #endif 2869 return (0); 2870 } 2871 2872 static int 2873 ovpnmodevent(module_t mod, int type, void *data) 2874 { 2875 static int ovpn_osd_jail_slot; 2876 2877 switch (type) { 2878 case MOD_LOAD: { 2879 /* 2880 * Registration is handled in vnet_ovpn_init(), but cloned 2881 * interfaces must be destroyed via PR_METHOD_REMOVE since they 2882 * hold a reference to the prison via the UDP socket, which 2883 * prevents the prison from being destroyed. 2884 */ 2885 osd_method_t methods[PR_MAXMETHOD] = { 2886 [PR_METHOD_REMOVE] = ovpn_prison_remove, 2887 }; 2888 ovpn_osd_jail_slot = osd_jail_register(NULL, methods); 2889 break; 2890 } 2891 case MOD_UNLOAD: 2892 if (ovpn_osd_jail_slot != 0) 2893 osd_jail_deregister(ovpn_osd_jail_slot); 2894 CURVNET_SET(vnet0); 2895 if (V_ovpn_cloner != NULL) { 2896 ifc_detach_cloner(V_ovpn_cloner); 2897 V_ovpn_cloner = NULL; 2898 } 2899 CURVNET_RESTORE(); 2900 break; 2901 default: 2902 return (EOPNOTSUPP); 2903 } 2904 2905 return (0); 2906 } 2907 2908 static moduledata_t ovpn_mod = { 2909 "if_ovpn", 2910 ovpnmodevent, 2911 0 2912 }; 2913 2914 DECLARE_MODULE(if_ovpn, ovpn_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 2915 MODULE_VERSION(if_ovpn, 1); 2916 MODULE_DEPEND(if_ovpn, crypto, 1, 1, 1); 2917