1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2021-2022 Rubicon Communications, LLC (Netgate) 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 */ 28 #include "opt_inet.h" 29 #include "opt_inet6.h" 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/buf_ring.h> 34 #include <sys/epoch.h> 35 #include <sys/file.h> 36 #include <sys/filedesc.h> 37 #include <sys/kernel.h> 38 #include <sys/malloc.h> 39 #include <sys/mbuf.h> 40 #include <sys/module.h> 41 #include <sys/nv.h> 42 #include <sys/priv.h> 43 #include <sys/protosw.h> 44 #include <sys/rmlock.h> 45 #include <sys/sdt.h> 46 #include <sys/smp.h> 47 #include <sys/socket.h> 48 #include <sys/socketvar.h> 49 #include <sys/sockio.h> 50 #include <sys/sysctl.h> 51 #include <sys/time.h> 52 53 #include <machine/atomic.h> 54 55 #include <net/bpf.h> 56 #include <net/if.h> 57 #include <net/if_clone.h> 58 #include <net/if_types.h> 59 #include <net/if_var.h> 60 #include <net/netisr.h> 61 #include <net/route/nhop.h> 62 63 #include <netinet/in.h> 64 #include <netinet/in_fib.h> 65 #include <netinet/ip.h> 66 #include <netinet/ip6.h> 67 #include <netinet/ip_var.h> 68 #include <netinet/udp.h> 69 #include <netinet/udp_var.h> 70 71 #include <netinet6/ip6_var.h> 72 #include <netinet6/in6_fib.h> 73 74 #include <machine/in_cksum.h> 75 76 #include <opencrypto/cryptodev.h> 77 78 #include "if_ovpn.h" 79 80 struct ovpn_kkey_dir { 81 int refcount; 82 uint8_t key[32]; 83 uint8_t keylen; 84 uint8_t nonce[8]; 85 uint8_t noncelen; 86 enum ovpn_key_cipher cipher; 87 crypto_session_t cryptoid; 88 89 struct mtx replay_mtx; 90 /* 91 * Last seen gapless sequence number. New rx seq numbers must be 92 * strictly higher than this. 93 */ 94 uint32_t rx_seq; 95 /* Seen packets, relative to rx_seq. bit(0) will always be 0. */ 96 uint64_t rx_window; 97 }; 98 99 struct ovpn_kkey { 100 struct ovpn_kkey_dir *encrypt; 101 struct ovpn_kkey_dir *decrypt; 102 uint8_t keyid; 103 uint32_t peerid; 104 }; 105 106 struct ovpn_keepalive { 107 uint32_t interval; 108 uint32_t timeout; 109 }; 110 111 struct ovpn_wire_header { 112 uint32_t opcode; /* opcode, key id, peer id */ 113 uint32_t seq; 114 uint8_t auth_tag[16]; 115 }; 116 117 struct ovpn_peer_counters { 118 uint64_t pkt_in; 119 uint64_t pkt_out; 120 uint64_t bytes_in; 121 uint64_t bytes_out; 122 }; 123 #define OVPN_PEER_COUNTER_SIZE (sizeof(struct ovpn_peer_counters)/sizeof(uint64_t)) 124 125 struct ovpn_notification { 126 enum ovpn_notif_type type; 127 uint32_t peerid; 128 129 /* Delete notification */ 130 enum ovpn_del_reason del_reason; 131 struct ovpn_peer_counters counters; 132 }; 133 134 struct ovpn_softc; 135 136 struct ovpn_kpeer { 137 RB_ENTRY(ovpn_kpeer) tree; 138 int refcount; 139 uint32_t peerid; 140 141 struct ovpn_softc *sc; 142 struct sockaddr_storage local; 143 struct sockaddr_storage remote; 144 145 struct in_addr vpn4; 146 struct in6_addr vpn6; 147 148 struct ovpn_kkey keys[2]; 149 uint32_t tx_seq; 150 151 enum ovpn_del_reason del_reason; 152 struct ovpn_keepalive keepalive; 153 uint32_t *last_active; 154 struct callout ping_send; 155 struct callout ping_rcv; 156 157 counter_u64_t counters[OVPN_PEER_COUNTER_SIZE]; 158 }; 159 160 struct ovpn_counters { 161 uint64_t lost_ctrl_pkts_in; 162 uint64_t lost_ctrl_pkts_out; 163 uint64_t lost_data_pkts_in; 164 uint64_t lost_data_pkts_out; 165 uint64_t nomem_data_pkts_in; 166 uint64_t nomem_data_pkts_out; 167 uint64_t received_ctrl_pkts; 168 uint64_t received_data_pkts; 169 uint64_t sent_ctrl_pkts; 170 uint64_t sent_data_pkts; 171 172 uint64_t transport_bytes_sent; 173 uint64_t transport_bytes_received; 174 uint64_t tunnel_bytes_sent; 175 uint64_t tunnel_bytes_received; 176 }; 177 #define OVPN_COUNTER_SIZE (sizeof(struct ovpn_counters)/sizeof(uint64_t)) 178 179 RB_HEAD(ovpn_kpeers, ovpn_kpeer); 180 RB_HEAD(ovpn_kpeers_by_ip, ovpn_kpeer); 181 RB_HEAD(ovpn_kpeers_by_ip6, ovpn_kpeer); 182 183 struct ovpn_softc { 184 int refcount; 185 struct rmlock lock; 186 struct ifnet *ifp; 187 struct socket *so; 188 int peercount; 189 struct ovpn_kpeers peers; 190 struct ovpn_kpeers_by_ip peers_by_ip; 191 struct ovpn_kpeers_by_ip6 peers_by_ip6; 192 193 /* Pending notification */ 194 struct buf_ring *notifring; 195 196 counter_u64_t counters[OVPN_COUNTER_SIZE]; 197 198 struct epoch_context epoch_ctx; 199 }; 200 201 static struct ovpn_kpeer *ovpn_find_peer(struct ovpn_softc *, uint32_t); 202 static struct ovpn_kpeer *ovpn_find_peer_by_ip(struct ovpn_softc *, 203 const struct in_addr); 204 static struct ovpn_kpeer *ovpn_find_peer_by_ip6(struct ovpn_softc *, 205 const struct in6_addr *); 206 static bool ovpn_udp_input(struct mbuf *, int, struct inpcb *, 207 const struct sockaddr *, void *); 208 static int ovpn_transmit_to_peer(struct ifnet *, struct mbuf *, 209 struct ovpn_kpeer *, struct rm_priotracker *); 210 static int ovpn_encap(struct ovpn_softc *, uint32_t, struct mbuf *); 211 static int ovpn_get_af(struct mbuf *); 212 static void ovpn_free_kkey_dir(struct ovpn_kkey_dir *); 213 static bool ovpn_check_replay(struct ovpn_kkey_dir *, uint32_t); 214 static int ovpn_peer_compare(const struct ovpn_kpeer *, 215 const struct ovpn_kpeer *); 216 static int ovpn_peer_compare_by_ip(const struct ovpn_kpeer *, 217 const struct ovpn_kpeer *); 218 static int ovpn_peer_compare_by_ip6(const struct ovpn_kpeer *, 219 const struct ovpn_kpeer *); 220 221 static RB_PROTOTYPE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare); 222 static RB_GENERATE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare); 223 static RB_PROTOTYPE(ovpn_kpeers_by_ip, ovpn_kpeer, tree, 224 ovpn_peer_compare_by_ip); 225 static RB_GENERATE(ovpn_kpeers_by_ip, ovpn_kpeer, tree, 226 ovpn_peer_compare_by_ip); 227 static RB_PROTOTYPE(ovpn_kpeers_by_ip6, ovpn_kpeer, tree, 228 ovpn_peer_compare_by_ip6); 229 static RB_GENERATE(ovpn_kpeers_by_ip6, ovpn_kpeer, tree, 230 ovpn_peer_compare_by_ip6); 231 232 #define OVPN_MTU_MIN 576 233 #define OVPN_MTU_MAX (IP_MAXPACKET - sizeof(struct ip) - \ 234 sizeof(struct udphdr) - sizeof(struct ovpn_wire_header)) 235 236 #define OVPN_OP_DATA_V2 0x09 237 #define OVPN_OP_SHIFT 3 238 239 VNET_DEFINE_STATIC(struct if_clone *, ovpn_cloner); 240 #define V_ovpn_cloner VNET(ovpn_cloner) 241 242 #define OVPN_RLOCK_TRACKER struct rm_priotracker _ovpn_lock_tracker; \ 243 struct rm_priotracker *_ovpn_lock_trackerp = &_ovpn_lock_tracker 244 #define OVPN_RLOCK(sc) rm_rlock(&(sc)->lock, _ovpn_lock_trackerp) 245 #define OVPN_RUNLOCK(sc) rm_runlock(&(sc)->lock, _ovpn_lock_trackerp) 246 #define OVPN_WLOCK(sc) rm_wlock(&(sc)->lock) 247 #define OVPN_WUNLOCK(sc) rm_wunlock(&(sc)->lock) 248 #define OVPN_ASSERT(sc) rm_assert(&(sc)->lock, RA_LOCKED) 249 #define OVPN_RASSERT(sc) rm_assert(&(sc)->lock, RA_RLOCKED) 250 #define OVPN_WASSERT(sc) rm_assert(&(sc)->lock, RA_WLOCKED) 251 #define OVPN_UNLOCK_ASSERT(sc) rm_assert(&(sc)->lock, RA_UNLOCKED) 252 253 #define OVPN_COUNTER(sc, name) \ 254 ((sc)->counters[offsetof(struct ovpn_counters, name)/sizeof(uint64_t)]) 255 #define OVPN_PEER_COUNTER(peer, name) \ 256 ((peer)->counters[offsetof(struct ovpn_peer_counters, name) / \ 257 sizeof(uint64_t)]) 258 259 #define OVPN_COUNTER_ADD(sc, name, val) \ 260 counter_u64_add(OVPN_COUNTER(sc, name), val) 261 #define OVPN_PEER_COUNTER_ADD(p, name, val) \ 262 counter_u64_add(OVPN_PEER_COUNTER(p, name), val) 263 264 #define TO_IN(x) ((struct sockaddr_in *)(x)) 265 #define TO_IN6(x) ((struct sockaddr_in6 *)(x)) 266 267 SDT_PROVIDER_DEFINE(if_ovpn); 268 SDT_PROBE_DEFINE1(if_ovpn, tx, transmit, start, "struct mbuf *"); 269 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip4, "struct in_addr *", "struct ovpn_kpeer *"); 270 SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip6, "struct in6_addr *", "struct ovpn_kpeer *"); 271 272 static const char ovpnname[] = "ovpn"; 273 static const char ovpngroupname[] = "openvpn"; 274 275 static MALLOC_DEFINE(M_OVPN, ovpnname, "OpenVPN DCO Interface"); 276 277 SYSCTL_DECL(_net_link); 278 static SYSCTL_NODE(_net_link, IFT_OTHER, openvpn, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 279 "OpenVPN DCO Interface"); 280 VNET_DEFINE_STATIC(int, replay_protection) = 0; 281 #define V_replay_protection VNET(replay_protection) 282 SYSCTL_INT(_net_link_openvpn, OID_AUTO, replay_protection, CTLFLAG_VNET | CTLFLAG_RW, 283 &VNET_NAME(replay_protection), 0, "Validate sequence numbers"); 284 285 VNET_DEFINE_STATIC(int, async_crypto); 286 #define V_async_crypto VNET(async_crypto) 287 SYSCTL_INT(_net_link_openvpn, OID_AUTO, async_crypto, 288 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_crypto), 0, 289 "Use asynchronous mode to parallelize crypto jobs."); 290 291 VNET_DEFINE_STATIC(int, async_netisr_queue); 292 #define V_async_netisr_queue VNET(async_netisr_queue) 293 SYSCTL_INT(_net_link_openvpn, OID_AUTO, netisr_queue, 294 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_netisr_queue), 0, 295 "Use netisr_queue() rather than netisr_dispatch()."); 296 297 static int 298 ovpn_peer_compare(const struct ovpn_kpeer *a, const struct ovpn_kpeer *b) 299 { 300 return (a->peerid - b->peerid); 301 } 302 303 static int 304 ovpn_peer_compare_by_ip(const struct ovpn_kpeer *a, const struct ovpn_kpeer *b) 305 { 306 return (memcmp(&a->vpn4, &b->vpn4, sizeof(a->vpn4))); 307 } 308 309 static int 310 ovpn_peer_compare_by_ip6(const struct ovpn_kpeer *a, 311 const struct ovpn_kpeer *b) 312 { 313 return (memcmp(&a->vpn6, &b->vpn6, sizeof(a->vpn6))); 314 } 315 316 static struct ovpn_kpeer * 317 ovpn_find_peer(struct ovpn_softc *sc, uint32_t peerid) 318 { 319 struct ovpn_kpeer p; 320 321 OVPN_ASSERT(sc); 322 323 p.peerid = peerid; 324 325 return (RB_FIND(ovpn_kpeers, &sc->peers, &p)); 326 } 327 328 static struct ovpn_kpeer * 329 ovpn_find_only_peer(struct ovpn_softc *sc) 330 { 331 OVPN_ASSERT(sc); 332 333 return (RB_ROOT(&sc->peers)); 334 } 335 336 static uint16_t 337 ovpn_get_port(struct sockaddr_storage *s) 338 { 339 switch (s->ss_family) { 340 case AF_INET: { 341 struct sockaddr_in *in = (struct sockaddr_in *)s; 342 return (in->sin_port); 343 } 344 case AF_INET6: { 345 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)s; 346 return (in6->sin6_port); 347 } 348 default: 349 panic("Unsupported address family %d", s->ss_family); 350 } 351 } 352 353 static int 354 ovpn_nvlist_to_sockaddr(const nvlist_t *nvl, struct sockaddr_storage *sa) 355 { 356 int af; 357 358 if (! nvlist_exists_number(nvl, "af")) 359 return (EINVAL); 360 if (! nvlist_exists_binary(nvl, "address")) 361 return (EINVAL); 362 if (! nvlist_exists_number(nvl, "port")) 363 return (EINVAL); 364 365 af = nvlist_get_number(nvl, "af"); 366 367 switch (af) { 368 #ifdef INET 369 case AF_INET: { 370 struct sockaddr_in *in = (struct sockaddr_in *)sa; 371 size_t len; 372 const void *addr = nvlist_get_binary(nvl, "address", &len); 373 in->sin_family = af; 374 if (len != sizeof(in->sin_addr)) 375 return (EINVAL); 376 377 memcpy(&in->sin_addr, addr, sizeof(in->sin_addr)); 378 in->sin_port = nvlist_get_number(nvl, "port"); 379 break; 380 } 381 #endif 382 #ifdef INET6 383 case AF_INET6: { 384 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)sa; 385 size_t len; 386 const void *addr = nvlist_get_binary(nvl, "address", &len); 387 in6->sin6_family = af; 388 if (len != sizeof(in6->sin6_addr)) 389 return (EINVAL); 390 391 memcpy(&in6->sin6_addr, addr, sizeof(in6->sin6_addr)); 392 in6->sin6_port = nvlist_get_number(nvl, "port"); 393 break; 394 } 395 #endif 396 default: 397 return (EINVAL); 398 } 399 400 return (0); 401 } 402 403 static bool 404 ovpn_has_peers(struct ovpn_softc *sc) 405 { 406 OVPN_ASSERT(sc); 407 408 return (sc->peercount > 0); 409 } 410 411 static void 412 ovpn_rele_so(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 413 { 414 bool has_peers; 415 416 OVPN_WASSERT(sc); 417 418 if (sc->so == NULL) 419 return; 420 421 has_peers = ovpn_has_peers(sc); 422 423 /* Only remove the tunnel function if we're releasing the socket for 424 * the last peer. */ 425 if (! has_peers) 426 (void)udp_set_kernel_tunneling(sc->so, NULL, NULL, NULL); 427 428 sorele(sc->so); 429 430 if (! has_peers) 431 sc->so = NULL; 432 } 433 434 static void 435 ovpn_notify_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 436 { 437 struct ovpn_notification *n; 438 439 OVPN_WASSERT(sc); 440 441 n = malloc(sizeof(*n), M_OVPN, M_NOWAIT); 442 if (n == NULL) 443 return; 444 445 n->peerid = peer->peerid; 446 n->type = OVPN_NOTIF_DEL_PEER; 447 n->del_reason = peer->del_reason; 448 449 n->counters.pkt_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_in)); 450 n->counters.pkt_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_out)); 451 n->counters.bytes_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_in)); 452 n->counters.bytes_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_out)); 453 454 if (buf_ring_enqueue(sc->notifring, n) != 0) { 455 free(n, M_OVPN); 456 } else if (sc->so != NULL) { 457 /* Wake up userspace */ 458 sc->so->so_error = EAGAIN; 459 sorwakeup(sc->so); 460 sowwakeup(sc->so); 461 } 462 } 463 464 static void 465 ovpn_peer_release_ref(struct ovpn_kpeer *peer, bool locked) 466 { 467 struct ovpn_softc *sc; 468 469 CURVNET_ASSERT_SET(); 470 471 atomic_add_int(&peer->refcount, -1); 472 473 if (atomic_load_int(&peer->refcount) > 0) 474 return; 475 476 sc = peer->sc; 477 478 if (! locked) { 479 OVPN_WLOCK(sc); 480 481 /* Might have changed before we acquired the lock. */ 482 if (atomic_load_int(&peer->refcount) > 0) { 483 OVPN_WUNLOCK(sc); 484 return; 485 } 486 } 487 488 OVPN_ASSERT(sc); 489 490 /* The peer should have been removed from the list already. */ 491 MPASS(ovpn_find_peer(sc, peer->peerid) == NULL); 492 493 ovpn_notify_del_peer(sc, peer); 494 495 for (int i = 0; i < 2; i++) { 496 ovpn_free_kkey_dir(peer->keys[i].encrypt); 497 ovpn_free_kkey_dir(peer->keys[i].decrypt); 498 } 499 500 ovpn_rele_so(sc, peer); 501 502 callout_stop(&peer->ping_send); 503 callout_stop(&peer->ping_rcv); 504 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 505 free(peer, M_OVPN); 506 507 if (! locked) 508 OVPN_WUNLOCK(sc); 509 } 510 511 static int 512 ovpn_new_peer(struct ifnet *ifp, const nvlist_t *nvl) 513 { 514 #ifdef INET6 515 struct epoch_tracker et; 516 #endif 517 struct sockaddr_storage remote; 518 struct ovpn_kpeer *peer = NULL; 519 struct file *fp = NULL; 520 struct sockaddr *name = NULL; 521 struct ovpn_softc *sc = ifp->if_softc; 522 struct thread *td = curthread; 523 struct socket *so = NULL; 524 int fd; 525 uint32_t peerid; 526 int ret = 0; 527 528 if (nvl == NULL) 529 return (EINVAL); 530 531 if (! nvlist_exists_number(nvl, "peerid")) 532 return (EINVAL); 533 534 if (! nvlist_exists_number(nvl, "fd")) 535 return (EINVAL); 536 537 if (! nvlist_exists_nvlist(nvl, "remote")) 538 return (EINVAL); 539 540 peerid = nvlist_get_number(nvl, "peerid"); 541 542 ret = ovpn_nvlist_to_sockaddr(nvlist_get_nvlist(nvl, "remote"), 543 &remote); 544 if (ret != 0) 545 return (ret); 546 547 fd = nvlist_get_number(nvl, "fd"); 548 549 /* Look up the userspace process and use the fd to find the socket. */ 550 ret = getsock(td, fd, &cap_connect_rights, &fp); 551 if (ret != 0) 552 return (ret); 553 554 so = fp->f_data; 555 556 peer = malloc(sizeof(*peer), M_OVPN, M_WAITOK | M_ZERO); 557 peer->peerid = peerid; 558 peer->sc = sc; 559 peer->tx_seq = 1; 560 peer->refcount = 1; 561 peer->last_active = uma_zalloc_pcpu(pcpu_zone_4, M_WAITOK | M_ZERO); 562 COUNTER_ARRAY_ALLOC(peer->counters, OVPN_PEER_COUNTER_SIZE, M_WAITOK); 563 564 if (nvlist_exists_binary(nvl, "vpn_ipv4")) { 565 size_t len; 566 const void *addr = nvlist_get_binary(nvl, "vpn_ipv4", &len); 567 if (len != sizeof(peer->vpn4)) { 568 ret = EINVAL; 569 goto error; 570 } 571 memcpy(&peer->vpn4, addr, len); 572 } 573 574 if (nvlist_exists_binary(nvl, "vpn_ipv6")) { 575 size_t len; 576 const void *addr = nvlist_get_binary(nvl, "vpn_ipv6", &len); 577 if (len != sizeof(peer->vpn6)) { 578 ret = EINVAL; 579 goto error; 580 } 581 memcpy(&peer->vpn6, addr, len); 582 } 583 584 callout_init_rm(&peer->ping_send, &sc->lock, CALLOUT_SHAREDLOCK); 585 callout_init_rm(&peer->ping_rcv, &sc->lock, 0); 586 587 ret = so->so_proto->pr_sockaddr(so, &name); 588 if (ret) 589 goto error; 590 591 if (ovpn_get_port((struct sockaddr_storage *)name) == 0) { 592 ret = EINVAL; 593 goto error; 594 } 595 if (name->sa_family != remote.ss_family) { 596 ret = EINVAL; 597 goto error; 598 } 599 600 memcpy(&peer->local, name, name->sa_len); 601 memcpy(&peer->remote, &remote, sizeof(remote)); 602 free(name, M_SONAME); 603 name = NULL; 604 605 if (peer->local.ss_family == AF_INET6 && 606 IN6_IS_ADDR_V4MAPPED(&TO_IN6(&peer->remote)->sin6_addr)) { 607 /* V4 mapped address, so treat this as v4, not v6. */ 608 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->local); 609 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->remote); 610 } 611 612 #ifdef INET6 613 if (peer->local.ss_family == AF_INET6 && 614 IN6_IS_ADDR_UNSPECIFIED(&TO_IN6(&peer->local)->sin6_addr)) { 615 NET_EPOCH_ENTER(et); 616 ret = in6_selectsrc_addr(curthread->td_proc->p_fibnum, 617 &TO_IN6(&peer->remote)->sin6_addr, 618 0, NULL, &TO_IN6(&peer->local)->sin6_addr, NULL); 619 NET_EPOCH_EXIT(et); 620 if (ret != 0) { 621 goto error; 622 } 623 } 624 #endif 625 OVPN_WLOCK(sc); 626 627 /* Disallow peer id re-use. */ 628 if (ovpn_find_peer(sc, peerid) != NULL) { 629 ret = EEXIST; 630 goto error_locked; 631 } 632 633 /* Make sure this is really a UDP socket. */ 634 if (so->so_type != SOCK_DGRAM || so->so_proto->pr_type != SOCK_DGRAM) { 635 ret = EPROTOTYPE; 636 goto error_locked; 637 } 638 639 /* Must be the same socket as for other peers on this interface. */ 640 if (sc->so != NULL && so != sc->so) 641 goto error_locked; 642 643 if (sc->so == NULL) 644 sc->so = so; 645 646 /* Insert the peer into the lists. */ 647 RB_INSERT(ovpn_kpeers, &sc->peers, peer); 648 if (nvlist_exists_binary(nvl, "vpn_ipv4")) { 649 RB_INSERT(ovpn_kpeers_by_ip, &sc->peers_by_ip, peer); 650 } 651 if (nvlist_exists_binary(nvl, "vpn_ipv6")) { 652 RB_INSERT(ovpn_kpeers_by_ip6, &sc->peers_by_ip6, peer); 653 } 654 sc->peercount++; 655 soref(sc->so); 656 657 ret = udp_set_kernel_tunneling(sc->so, ovpn_udp_input, NULL, sc); 658 if (ret == EBUSY) { 659 /* Fine, another peer already set the input function. */ 660 ret = 0; 661 } 662 if (ret != 0) { 663 RB_REMOVE(ovpn_kpeers, &sc->peers, peer); 664 if (nvlist_exists_binary(nvl, "vpn_ipv4")) { 665 RB_REMOVE(ovpn_kpeers_by_ip, &sc->peers_by_ip, peer); 666 } 667 if (nvlist_exists_binary(nvl, "vpn_ipv6")) { 668 RB_REMOVE(ovpn_kpeers_by_ip6, &sc->peers_by_ip6, peer); 669 } 670 sc->peercount--; 671 goto error_locked; 672 } 673 674 OVPN_WUNLOCK(sc); 675 676 goto done; 677 678 error_locked: 679 OVPN_WUNLOCK(sc); 680 error: 681 free(name, M_SONAME); 682 COUNTER_ARRAY_FREE(peer->counters, OVPN_PEER_COUNTER_SIZE); 683 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 684 free(peer, M_OVPN); 685 done: 686 if (fp != NULL) 687 fdrop(fp, td); 688 689 return (ret); 690 } 691 692 static int 693 _ovpn_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 694 { 695 struct ovpn_kpeer *tmp; 696 697 OVPN_WASSERT(sc); 698 CURVNET_ASSERT_SET(); 699 700 MPASS(RB_FIND(ovpn_kpeers, &sc->peers, peer) == peer); 701 702 tmp = RB_REMOVE(ovpn_kpeers, &sc->peers, peer); 703 MPASS(tmp != NULL); 704 705 tmp = ovpn_find_peer_by_ip(sc, peer->vpn4); 706 if (tmp) 707 RB_REMOVE(ovpn_kpeers_by_ip, &sc->peers_by_ip, tmp); 708 tmp = ovpn_find_peer_by_ip6(sc, &peer->vpn6); 709 if (tmp) 710 RB_REMOVE(ovpn_kpeers_by_ip6, &sc->peers_by_ip6, tmp); 711 712 sc->peercount--; 713 714 ovpn_peer_release_ref(peer, true); 715 716 return (0); 717 } 718 719 static int 720 ovpn_del_peer(struct ifnet *ifp, nvlist_t *nvl) 721 { 722 struct ovpn_softc *sc = ifp->if_softc; 723 struct ovpn_kpeer *peer; 724 uint32_t peerid; 725 int ret; 726 727 OVPN_WASSERT(sc); 728 729 if (nvl == NULL) 730 return (EINVAL); 731 732 if (! nvlist_exists_number(nvl, "peerid")) 733 return (EINVAL); 734 735 peerid = nvlist_get_number(nvl, "peerid"); 736 737 peer = ovpn_find_peer(sc, peerid); 738 if (peer == NULL) 739 return (ENOENT); 740 741 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 742 ret = _ovpn_del_peer(sc, peer); 743 744 return (ret); 745 } 746 747 static int 748 ovpn_create_kkey_dir(struct ovpn_kkey_dir **kdirp, 749 const nvlist_t *nvl) 750 { 751 struct crypto_session_params csp; 752 struct ovpn_kkey_dir *kdir; 753 const char *ciphername; 754 enum ovpn_key_cipher cipher; 755 const void *key, *iv; 756 size_t keylen = 0, ivlen = 0; 757 int error; 758 759 if (! nvlist_exists_string(nvl, "cipher")) 760 return (EINVAL); 761 ciphername = nvlist_get_string(nvl, "cipher"); 762 763 if (strcmp(ciphername, "none") == 0) 764 cipher = OVPN_CIPHER_ALG_NONE; 765 else if (strcmp(ciphername, "AES-256-GCM") == 0 || 766 strcmp(ciphername, "AES-192-GCM") == 0 || 767 strcmp(ciphername, "AES-128-GCM") == 0) 768 cipher = OVPN_CIPHER_ALG_AES_GCM; 769 else if (strcmp(ciphername, "CHACHA20-POLY1305") == 0) 770 cipher = OVPN_CIPHER_ALG_CHACHA20_POLY1305; 771 else 772 return (EINVAL); 773 774 if (cipher != OVPN_CIPHER_ALG_NONE) { 775 if (! nvlist_exists_binary(nvl, "key")) 776 return (EINVAL); 777 key = nvlist_get_binary(nvl, "key", &keylen); 778 if (keylen > sizeof(kdir->key)) 779 return (E2BIG); 780 781 if (! nvlist_exists_binary(nvl, "iv")) 782 return (EINVAL); 783 iv = nvlist_get_binary(nvl, "iv", &ivlen); 784 if (ivlen != 8) 785 return (E2BIG); 786 } 787 788 kdir = malloc(sizeof(struct ovpn_kkey_dir), M_OVPN, 789 M_WAITOK | M_ZERO); 790 791 kdir->cipher = cipher; 792 kdir->keylen = keylen; 793 memcpy(kdir->key, key, keylen); 794 kdir->noncelen = ivlen; 795 memcpy(kdir->nonce, iv, ivlen); 796 797 if (kdir->cipher != OVPN_CIPHER_ALG_NONE) { 798 /* Crypto init */ 799 bzero(&csp, sizeof(csp)); 800 csp.csp_mode = CSP_MODE_AEAD; 801 802 if (kdir->cipher == OVPN_CIPHER_ALG_CHACHA20_POLY1305) 803 csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305; 804 else 805 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 806 807 csp.csp_flags |= CSP_F_SEPARATE_AAD; 808 809 csp.csp_cipher_klen = kdir->keylen; 810 csp.csp_cipher_key = kdir->key; 811 csp.csp_ivlen = 96 / 8; 812 813 error = crypto_newsession(&kdir->cryptoid, &csp, 814 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE); 815 if (error) { 816 free(kdir, M_OVPN); 817 return (error); 818 } 819 } 820 821 mtx_init(&kdir->replay_mtx, "if_ovpn rx replay", NULL, MTX_DEF); 822 *kdirp = kdir; 823 824 return (0); 825 } 826 827 static void 828 ovpn_free_kkey_dir(struct ovpn_kkey_dir *kdir) 829 { 830 if (kdir == NULL) 831 return; 832 833 mtx_destroy(&kdir->replay_mtx); 834 835 crypto_freesession(kdir->cryptoid); 836 free(kdir, M_OVPN); 837 } 838 839 static int 840 ovpn_set_key(struct ifnet *ifp, const nvlist_t *nvl) 841 { 842 struct ovpn_softc *sc = ifp->if_softc; 843 struct ovpn_kkey_dir *enc, *dec; 844 struct ovpn_kpeer *peer; 845 int slot, keyid, peerid; 846 int error; 847 848 if (nvl == NULL) 849 return (EINVAL); 850 851 if (! nvlist_exists_number(nvl, "slot")) 852 return (EINVAL); 853 slot = nvlist_get_number(nvl, "slot"); 854 855 if (! nvlist_exists_number(nvl, "keyid")) 856 return (EINVAL); 857 keyid = nvlist_get_number(nvl, "keyid"); 858 859 if (! nvlist_exists_number(nvl, "peerid")) 860 return (EINVAL); 861 peerid = nvlist_get_number(nvl, "peerid"); 862 863 if (slot != OVPN_KEY_SLOT_PRIMARY && 864 slot != OVPN_KEY_SLOT_SECONDARY) 865 return (EINVAL); 866 867 if (! nvlist_exists_nvlist(nvl, "encrypt") || 868 ! nvlist_exists_nvlist(nvl, "decrypt")) 869 return (EINVAL); 870 871 error = ovpn_create_kkey_dir(&enc, nvlist_get_nvlist(nvl, "encrypt")); 872 if (error) 873 return (error); 874 875 error = ovpn_create_kkey_dir(&dec, nvlist_get_nvlist(nvl, "decrypt")); 876 if (error) { 877 ovpn_free_kkey_dir(enc); 878 return (error); 879 } 880 881 OVPN_WLOCK(sc); 882 883 peer = ovpn_find_peer(sc, peerid); 884 if (peer == NULL) { 885 ovpn_free_kkey_dir(dec); 886 ovpn_free_kkey_dir(enc); 887 OVPN_WUNLOCK(sc); 888 return (ENOENT); 889 } 890 891 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 892 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 893 894 peer->keys[slot].encrypt = enc; 895 peer->keys[slot].decrypt = dec; 896 897 peer->keys[slot].keyid = keyid; 898 peer->keys[slot].peerid = peerid; 899 900 OVPN_WUNLOCK(sc); 901 902 return (0); 903 } 904 905 static int 906 ovpn_check_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, enum ovpn_key_slot slot) 907 { 908 OVPN_ASSERT(sc); 909 910 if (peer->keys[slot].encrypt == NULL) 911 return (ENOLINK); 912 913 if (peer->keys[slot].decrypt == NULL) 914 return (ENOLINK); 915 916 return (0); 917 } 918 919 static int 920 ovpn_start(struct ifnet *ifp) 921 { 922 struct ovpn_softc *sc = ifp->if_softc; 923 924 OVPN_WLOCK(sc); 925 926 ifp->if_flags |= IFF_UP; 927 ifp->if_drv_flags |= IFF_DRV_RUNNING; 928 if_link_state_change(ifp, LINK_STATE_UP); 929 930 OVPN_WUNLOCK(sc); 931 932 return (0); 933 } 934 935 static int 936 ovpn_swap_keys(struct ifnet *ifp, nvlist_t *nvl) 937 { 938 struct ovpn_softc *sc = ifp->if_softc; 939 struct ovpn_kpeer *peer; 940 struct ovpn_kkey tmpkey; 941 int error; 942 943 if (nvl == NULL) 944 return (EINVAL); 945 946 if (! nvlist_exists_number(nvl, "peerid")) 947 return (EINVAL); 948 949 OVPN_WLOCK(sc); 950 951 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 952 if (peer == NULL) { 953 OVPN_WUNLOCK(sc); 954 return (ENOENT); 955 } 956 957 /* Check that we have a second key to swap to. */ 958 error = ovpn_check_key(sc, peer, OVPN_KEY_SLOT_SECONDARY); 959 if (error) { 960 OVPN_WUNLOCK(sc); 961 return (error); 962 } 963 964 tmpkey = peer->keys[0]; 965 peer->keys[0] = peer->keys[1]; 966 peer->keys[1] = tmpkey; 967 968 OVPN_WUNLOCK(sc); 969 970 return (0); 971 } 972 973 static int 974 ovpn_del_key(struct ifnet *ifp, const nvlist_t *nvl) 975 { 976 enum ovpn_key_slot slot; 977 struct ovpn_kpeer *peer; 978 struct ovpn_softc *sc = ifp->if_softc; 979 980 if (nvl == NULL) 981 return (EINVAL); 982 983 if (! nvlist_exists_number(nvl, "peerid")) 984 return (EINVAL); 985 986 if (! nvlist_exists_number(nvl, "slot")) 987 return (EINVAL); 988 slot = nvlist_get_number(nvl, "slot"); 989 990 if (slot != OVPN_KEY_SLOT_PRIMARY && 991 slot != OVPN_KEY_SLOT_SECONDARY) 992 return (EINVAL); 993 994 OVPN_WLOCK(sc); 995 996 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 997 if (peer == NULL) { 998 OVPN_WUNLOCK(sc); 999 return (ENOENT); 1000 } 1001 1002 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 1003 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 1004 1005 peer->keys[slot].encrypt = NULL; 1006 peer->keys[slot].decrypt = NULL; 1007 1008 peer->keys[slot].keyid = 0; 1009 peer->keys[slot].peerid = 0; 1010 1011 OVPN_WUNLOCK(sc); 1012 1013 return (0); 1014 } 1015 1016 static void 1017 ovpn_send_ping(void *arg) 1018 { 1019 static const uint8_t ping_str[] = { 1020 0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb, 1021 0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48 1022 }; 1023 1024 struct epoch_tracker et; 1025 struct ovpn_kpeer *peer = arg; 1026 struct ovpn_softc *sc = peer->sc; 1027 struct mbuf *m; 1028 1029 OVPN_RASSERT(sc); 1030 1031 /* Ensure we repeat! */ 1032 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 1033 ovpn_send_ping, peer); 1034 1035 m = m_get2(sizeof(ping_str), M_NOWAIT, MT_DATA, M_PKTHDR); 1036 if (m == NULL) 1037 return; 1038 1039 m_copyback(m, 0, sizeof(ping_str), ping_str); 1040 m->m_len = m->m_pkthdr.len = sizeof(ping_str); 1041 1042 CURVNET_SET(sc->ifp->if_vnet); 1043 NET_EPOCH_ENTER(et); 1044 (void)ovpn_transmit_to_peer(sc->ifp, m, peer, NULL); 1045 NET_EPOCH_EXIT(et); 1046 CURVNET_RESTORE(); 1047 } 1048 1049 static void 1050 ovpn_timeout(void *arg) 1051 { 1052 struct ovpn_kpeer *peer = arg; 1053 struct ovpn_softc *sc = peer->sc; 1054 uint32_t last, _last_active; 1055 int ret __diagused; 1056 int cpu; 1057 1058 OVPN_WASSERT(sc); 1059 1060 last = 0; 1061 CPU_FOREACH(cpu) { 1062 _last_active = *zpcpu_get_cpu(peer->last_active, cpu); 1063 if (_last_active > last) 1064 last = _last_active; 1065 } 1066 1067 if (last + peer->keepalive.timeout > time_uptime) { 1068 callout_reset(&peer->ping_rcv, 1069 (peer->keepalive.timeout - (time_uptime - last)) * hz, 1070 ovpn_timeout, peer); 1071 return; 1072 } 1073 1074 CURVNET_SET(sc->ifp->if_vnet); 1075 peer->del_reason = OVPN_DEL_REASON_TIMEOUT; 1076 ret = _ovpn_del_peer(sc, peer); 1077 MPASS(ret == 0); 1078 CURVNET_RESTORE(); 1079 } 1080 1081 static int 1082 ovpn_set_peer(struct ifnet *ifp, const nvlist_t *nvl) 1083 { 1084 struct ovpn_softc *sc = ifp->if_softc; 1085 struct ovpn_kpeer *peer; 1086 1087 if (nvl == NULL) 1088 return (EINVAL); 1089 1090 if (! nvlist_exists_number(nvl, "interval") || 1091 ! nvlist_exists_number(nvl, "timeout") || 1092 ! nvlist_exists_number(nvl, "peerid")) 1093 return (EINVAL); 1094 1095 OVPN_WLOCK(sc); 1096 1097 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 1098 if (peer == NULL) { 1099 OVPN_WUNLOCK(sc); 1100 return (ENOENT); 1101 } 1102 1103 peer->keepalive.interval = nvlist_get_number(nvl, "interval"); 1104 peer->keepalive.timeout = nvlist_get_number(nvl, "timeout"); 1105 1106 if (peer->keepalive.interval > 0) 1107 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 1108 ovpn_send_ping, peer); 1109 if (peer->keepalive.timeout > 0) 1110 callout_reset(&peer->ping_rcv, peer->keepalive.timeout * hz, 1111 ovpn_timeout, peer); 1112 1113 OVPN_WUNLOCK(sc); 1114 1115 return (0); 1116 } 1117 1118 static int 1119 ovpn_set_ifmode(struct ifnet *ifp, const nvlist_t *nvl) 1120 { 1121 struct ovpn_softc *sc = ifp->if_softc; 1122 int ifmode; 1123 1124 if (nvl == NULL) 1125 return (EINVAL); 1126 1127 if (! nvlist_exists_number(nvl, "ifmode") ) 1128 return (EINVAL); 1129 1130 ifmode = nvlist_get_number(nvl, "ifmode"); 1131 1132 OVPN_WLOCK(sc); 1133 1134 /* deny this if UP */ 1135 if (ifp->if_flags & IFF_UP) { 1136 OVPN_WUNLOCK(sc); 1137 return (EBUSY); 1138 } 1139 1140 switch (ifmode & ~IFF_MULTICAST) { 1141 case IFF_POINTOPOINT: 1142 case IFF_BROADCAST: 1143 ifp->if_flags &= 1144 ~(IFF_BROADCAST|IFF_POINTOPOINT|IFF_MULTICAST); 1145 ifp->if_flags |= ifmode; 1146 break; 1147 default: 1148 OVPN_WUNLOCK(sc); 1149 return (EINVAL); 1150 } 1151 1152 OVPN_WUNLOCK(sc); 1153 1154 return (0); 1155 } 1156 1157 static int 1158 ovpn_ioctl_set(struct ifnet *ifp, struct ifdrv *ifd) 1159 { 1160 struct ovpn_softc *sc = ifp->if_softc; 1161 uint8_t *buf = NULL; 1162 nvlist_t *nvl = NULL; 1163 int ret; 1164 1165 if (ifd->ifd_len != 0) { 1166 if (ifd->ifd_len > OVPN_MAX_REQUEST_SIZE) 1167 return (E2BIG); 1168 1169 buf = malloc(ifd->ifd_len, M_OVPN, M_WAITOK); 1170 1171 ret = copyin(ifd->ifd_data, buf, ifd->ifd_len); 1172 if (ret != 0) { 1173 free(buf, M_OVPN); 1174 return (ret); 1175 } 1176 1177 nvl = nvlist_unpack(buf, ifd->ifd_len, 0); 1178 free(buf, M_OVPN); 1179 if (nvl == NULL) { 1180 return (EINVAL); 1181 } 1182 } 1183 1184 switch (ifd->ifd_cmd) { 1185 case OVPN_NEW_PEER: 1186 ret = ovpn_new_peer(ifp, nvl); 1187 break; 1188 case OVPN_DEL_PEER: 1189 OVPN_WLOCK(sc); 1190 ret = ovpn_del_peer(ifp, nvl); 1191 OVPN_WUNLOCK(sc); 1192 break; 1193 case OVPN_NEW_KEY: 1194 ret = ovpn_set_key(ifp, nvl); 1195 break; 1196 case OVPN_START_VPN: 1197 ret = ovpn_start(ifp); 1198 break; 1199 case OVPN_SWAP_KEYS: 1200 ret = ovpn_swap_keys(ifp, nvl); 1201 break; 1202 case OVPN_DEL_KEY: 1203 ret = ovpn_del_key(ifp, nvl); 1204 break; 1205 case OVPN_SET_PEER: 1206 ret = ovpn_set_peer(ifp, nvl); 1207 break; 1208 case OVPN_SET_IFMODE: 1209 ret = ovpn_set_ifmode(ifp, nvl); 1210 break; 1211 default: 1212 ret = ENOTSUP; 1213 } 1214 1215 nvlist_destroy(nvl); 1216 return (ret); 1217 } 1218 1219 static int 1220 ovpn_add_counters(nvlist_t *parent, const char *name, counter_u64_t in, 1221 counter_u64_t out) 1222 { 1223 nvlist_t *nvl; 1224 1225 nvl = nvlist_create(0); 1226 if (nvl == NULL) 1227 return (ENOMEM); 1228 1229 nvlist_add_number(nvl, "in", counter_u64_fetch(in)); 1230 nvlist_add_number(nvl, "out", counter_u64_fetch(out)); 1231 1232 nvlist_add_nvlist(parent, name, nvl); 1233 1234 nvlist_destroy(nvl); 1235 1236 return (0); 1237 } 1238 1239 static int 1240 ovpn_get_stats(struct ovpn_softc *sc, nvlist_t **onvl) 1241 { 1242 nvlist_t *nvl; 1243 int ret; 1244 1245 nvl = nvlist_create(0); 1246 if (nvl == NULL) 1247 return (ENOMEM); 1248 1249 #define OVPN_COUNTER_OUT(name, in, out) \ 1250 do { \ 1251 ret = ovpn_add_counters(nvl, name, OVPN_COUNTER(sc, in), \ 1252 OVPN_COUNTER(sc, out)); \ 1253 if (ret != 0) \ 1254 goto error; \ 1255 } while(0) 1256 1257 OVPN_COUNTER_OUT("lost_ctrl", lost_ctrl_pkts_in, lost_ctrl_pkts_out); 1258 OVPN_COUNTER_OUT("lost_data", lost_data_pkts_in, lost_data_pkts_out); 1259 OVPN_COUNTER_OUT("nomem_data", nomem_data_pkts_in, 1260 nomem_data_pkts_out); 1261 OVPN_COUNTER_OUT("data", received_data_pkts, sent_data_pkts); 1262 OVPN_COUNTER_OUT("ctrl", received_ctrl_pkts, sent_ctrl_pkts); 1263 OVPN_COUNTER_OUT("tunnel", tunnel_bytes_received, 1264 tunnel_bytes_received); 1265 OVPN_COUNTER_OUT("transport", transport_bytes_received, 1266 transport_bytes_received); 1267 #undef OVPN_COUNTER_OUT 1268 1269 *onvl = nvl; 1270 1271 return (0); 1272 1273 error: 1274 nvlist_destroy(nvl); 1275 return (ret); 1276 } 1277 1278 static int 1279 ovpn_get_peer_stats(struct ovpn_softc *sc, nvlist_t **nvl) 1280 { 1281 struct ovpn_kpeer *peer; 1282 nvlist_t *nvpeer = NULL; 1283 int ret; 1284 1285 OVPN_RLOCK_TRACKER; 1286 1287 *nvl = nvlist_create(0); 1288 if (*nvl == NULL) 1289 return (ENOMEM); 1290 1291 #define OVPN_PEER_COUNTER_OUT(name, in, out) \ 1292 do { \ 1293 ret = ovpn_add_counters(nvpeer, name, \ 1294 OVPN_PEER_COUNTER(peer, in), OVPN_PEER_COUNTER(peer, out)); \ 1295 if (ret != 0) \ 1296 goto error; \ 1297 } while(0) 1298 1299 OVPN_RLOCK(sc); 1300 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1301 nvpeer = nvlist_create(0); 1302 if (nvpeer == NULL) { 1303 OVPN_RUNLOCK(sc); 1304 nvlist_destroy(*nvl); 1305 *nvl = NULL; 1306 return (ENOMEM); 1307 } 1308 1309 nvlist_add_number(nvpeer, "peerid", peer->peerid); 1310 1311 OVPN_PEER_COUNTER_OUT("packets", pkt_in, pkt_out); 1312 OVPN_PEER_COUNTER_OUT("bytes", bytes_in, bytes_out); 1313 1314 nvlist_append_nvlist_array(*nvl, "peers", nvpeer); 1315 nvlist_destroy(nvpeer); 1316 } 1317 #undef OVPN_PEER_COUNTER_OUT 1318 OVPN_RUNLOCK(sc); 1319 1320 return (0); 1321 1322 error: 1323 nvlist_destroy(nvpeer); 1324 nvlist_destroy(*nvl); 1325 *nvl = NULL; 1326 return (ret); 1327 } 1328 1329 static int 1330 ovpn_poll_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1331 { 1332 nvlist_t *nvl; 1333 1334 nvl = nvlist_create(0); 1335 if (nvl == NULL) 1336 return (ENOMEM); 1337 1338 nvlist_add_number(nvl, "pending", buf_ring_count(sc->notifring)); 1339 1340 *onvl = nvl; 1341 1342 return (0); 1343 } 1344 1345 static void 1346 ovpn_notif_add_counters(nvlist_t *parent, struct ovpn_notification *n) 1347 { 1348 nvlist_t *nvl; 1349 1350 nvl = nvlist_create(0); 1351 if (nvl == NULL) 1352 return; 1353 1354 nvlist_add_number(nvl, "in", n->counters.pkt_in); 1355 nvlist_add_number(nvl, "out", n->counters.pkt_out); 1356 1357 nvlist_add_nvlist(parent, "packets", nvl); 1358 nvlist_destroy(nvl); 1359 1360 nvl = nvlist_create(0); 1361 if (nvl == NULL) 1362 return; 1363 1364 nvlist_add_number(nvl, "in", n->counters.bytes_in); 1365 nvlist_add_number(nvl, "out", n->counters.bytes_out); 1366 1367 nvlist_add_nvlist(parent, "bytes", nvl); 1368 nvlist_destroy(nvl); 1369 } 1370 1371 static int 1372 opvn_get_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1373 { 1374 struct ovpn_notification *n; 1375 nvlist_t *nvl; 1376 1377 /* Check if we have notifications pending. */ 1378 n = buf_ring_dequeue_mc(sc->notifring); 1379 if (n == NULL) 1380 return (ENOENT); 1381 1382 nvl = nvlist_create(0); 1383 if (nvl == NULL) { 1384 free(n, M_OVPN); 1385 return (ENOMEM); 1386 } 1387 nvlist_add_number(nvl, "peerid", n->peerid); 1388 nvlist_add_number(nvl, "notification", n->type); 1389 if (n->type == OVPN_NOTIF_DEL_PEER) { 1390 nvlist_add_number(nvl, "del_reason", n->del_reason); 1391 1392 /* No error handling, because we want to send the notification 1393 * even if we can't attach the counters. */ 1394 ovpn_notif_add_counters(nvl, n); 1395 } 1396 free(n, M_OVPN); 1397 1398 *onvl = nvl; 1399 1400 return (0); 1401 } 1402 1403 static int 1404 ovpn_ioctl_get(struct ifnet *ifp, struct ifdrv *ifd) 1405 { 1406 struct ovpn_softc *sc = ifp->if_softc; 1407 nvlist_t *nvl = NULL; 1408 int error; 1409 1410 switch (ifd->ifd_cmd) { 1411 case OVPN_GET_STATS: 1412 error = ovpn_get_stats(sc, &nvl); 1413 break; 1414 case OVPN_GET_PEER_STATS: 1415 error = ovpn_get_peer_stats(sc, &nvl); 1416 break; 1417 case OVPN_POLL_PKT: 1418 error = ovpn_poll_pkt(sc, &nvl); 1419 break; 1420 case OVPN_GET_PKT: 1421 error = opvn_get_pkt(sc, &nvl); 1422 break; 1423 default: 1424 error = ENOTSUP; 1425 break; 1426 } 1427 1428 if (error == 0) { 1429 void *packed = NULL; 1430 size_t len; 1431 1432 MPASS(nvl != NULL); 1433 1434 packed = nvlist_pack(nvl, &len); 1435 if (! packed) { 1436 nvlist_destroy(nvl); 1437 return (ENOMEM); 1438 } 1439 1440 if (len > ifd->ifd_len) { 1441 free(packed, M_NVLIST); 1442 nvlist_destroy(nvl); 1443 return (ENOSPC); 1444 } 1445 1446 error = copyout(packed, ifd->ifd_data, len); 1447 ifd->ifd_len = len; 1448 1449 free(packed, M_NVLIST); 1450 nvlist_destroy(nvl); 1451 } 1452 1453 return (error); 1454 } 1455 1456 static int 1457 ovpn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 1458 { 1459 struct ifdrv *ifd; 1460 int error; 1461 1462 CURVNET_ASSERT_SET(); 1463 1464 switch (cmd) { 1465 case SIOCSDRVSPEC: 1466 case SIOCGDRVSPEC: 1467 error = priv_check(curthread, PRIV_NET_OVPN); 1468 if (error) 1469 return (error); 1470 break; 1471 } 1472 1473 switch (cmd) { 1474 case SIOCSDRVSPEC: 1475 ifd = (struct ifdrv *)data; 1476 error = ovpn_ioctl_set(ifp, ifd); 1477 break; 1478 case SIOCGDRVSPEC: 1479 ifd = (struct ifdrv *)data; 1480 error = ovpn_ioctl_get(ifp, ifd); 1481 break; 1482 case SIOCSIFMTU: { 1483 struct ifreq *ifr = (struct ifreq *)data; 1484 if (ifr->ifr_mtu < OVPN_MTU_MIN || ifr->ifr_mtu > OVPN_MTU_MAX) 1485 return (EINVAL); 1486 1487 ifp->if_mtu = ifr->ifr_mtu; 1488 return (0); 1489 } 1490 case SIOCSIFADDR: 1491 case SIOCADDMULTI: 1492 case SIOCDELMULTI: 1493 case SIOCGIFMTU: 1494 case SIOCSIFFLAGS: 1495 return (0); 1496 default: 1497 error = EINVAL; 1498 } 1499 1500 return (error); 1501 } 1502 1503 static int 1504 ovpn_encrypt_tx_cb(struct cryptop *crp) 1505 { 1506 struct epoch_tracker et; 1507 struct ovpn_kpeer *peer = crp->crp_opaque; 1508 struct ovpn_softc *sc = peer->sc; 1509 struct mbuf *m = crp->crp_buf.cb_mbuf; 1510 int tunnel_len; 1511 int ret; 1512 1513 CURVNET_SET(sc->ifp->if_vnet); 1514 NET_EPOCH_ENTER(et); 1515 1516 if (crp->crp_etype != 0) { 1517 crypto_freereq(crp); 1518 ovpn_peer_release_ref(peer, false); 1519 NET_EPOCH_EXIT(et); 1520 CURVNET_RESTORE(); 1521 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1522 m_freem(m); 1523 return (0); 1524 } 1525 1526 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1527 1528 tunnel_len = m->m_pkthdr.len - sizeof(struct ovpn_wire_header); 1529 ret = ovpn_encap(sc, peer->peerid, m); 1530 if (ret == 0) { 1531 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1532 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 1533 } 1534 1535 crypto_freereq(crp); 1536 ovpn_peer_release_ref(peer, false); 1537 1538 NET_EPOCH_EXIT(et); 1539 CURVNET_RESTORE(); 1540 1541 return (0); 1542 } 1543 1544 static void 1545 ovpn_finish_rx(struct ovpn_softc *sc, struct mbuf *m, 1546 struct ovpn_kpeer *peer, struct ovpn_kkey *key, uint32_t seq, 1547 struct rm_priotracker *_ovpn_lock_trackerp) 1548 { 1549 uint32_t af; 1550 1551 OVPN_RASSERT(sc); 1552 NET_EPOCH_ASSERT(); 1553 1554 /* Replay protection. */ 1555 if (V_replay_protection && ! ovpn_check_replay(key->decrypt, seq)) { 1556 OVPN_RUNLOCK(sc); 1557 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1558 m_freem(m); 1559 return; 1560 } 1561 1562 critical_enter(); 1563 *zpcpu_get(peer->last_active) = time_uptime; 1564 critical_exit(); 1565 1566 OVPN_RUNLOCK(sc); 1567 1568 OVPN_COUNTER_ADD(sc, received_data_pkts, 1); 1569 OVPN_COUNTER_ADD(sc, tunnel_bytes_received, m->m_pkthdr.len); 1570 OVPN_PEER_COUNTER_ADD(peer, pkt_in, 1); 1571 OVPN_PEER_COUNTER_ADD(peer, bytes_in, m->m_pkthdr.len); 1572 1573 /* Receive the packet on our interface. */ 1574 m->m_pkthdr.rcvif = sc->ifp; 1575 1576 /* Clear checksum flags in case the real hardware set them. */ 1577 m->m_pkthdr.csum_flags = 0; 1578 1579 /* Ensure we can read the first byte. */ 1580 m = m_pullup(m, 1); 1581 if (m == NULL) { 1582 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 1583 return; 1584 } 1585 1586 /* 1587 * Check for address family, and disregard any control packets (e.g. 1588 * keepalive). 1589 */ 1590 af = ovpn_get_af(m); 1591 if (af != 0) { 1592 BPF_MTAP2(sc->ifp, &af, sizeof(af), m); 1593 if (V_async_netisr_queue) 1594 netisr_queue(af == AF_INET ? NETISR_IP : NETISR_IPV6, m); 1595 else 1596 netisr_dispatch(af == AF_INET ? NETISR_IP : NETISR_IPV6, m); 1597 } else { 1598 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1599 m_freem(m); 1600 } 1601 } 1602 1603 static struct ovpn_kkey * 1604 ovpn_find_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, 1605 const struct ovpn_wire_header *ohdr) 1606 { 1607 struct ovpn_kkey *key = NULL; 1608 uint8_t keyid; 1609 1610 OVPN_RASSERT(sc); 1611 1612 keyid = (ntohl(ohdr->opcode) >> 24) & 0x07; 1613 1614 if (peer->keys[0].keyid == keyid) 1615 key = &peer->keys[0]; 1616 else if (peer->keys[1].keyid == keyid) 1617 key = &peer->keys[1]; 1618 1619 return (key); 1620 } 1621 1622 static int 1623 ovpn_decrypt_rx_cb(struct cryptop *crp) 1624 { 1625 struct epoch_tracker et; 1626 struct ovpn_softc *sc = crp->crp_opaque; 1627 struct mbuf *m = crp->crp_buf.cb_mbuf; 1628 struct ovpn_kkey *key; 1629 struct ovpn_kpeer *peer; 1630 struct ovpn_wire_header *ohdr; 1631 uint32_t peerid; 1632 1633 OVPN_RLOCK_TRACKER; 1634 1635 OVPN_RLOCK(sc); 1636 1637 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1638 1639 if (crp->crp_etype != 0) { 1640 crypto_freereq(crp); 1641 atomic_add_int(&sc->refcount, -1); 1642 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1643 OVPN_RUNLOCK(sc); 1644 m_freem(m); 1645 return (0); 1646 } 1647 1648 CURVNET_SET(sc->ifp->if_vnet); 1649 1650 ohdr = mtodo(m, sizeof(struct udphdr)); 1651 1652 peerid = ntohl(ohdr->opcode) & 0x00ffffff; 1653 peer = ovpn_find_peer(sc, peerid); 1654 if (peer == NULL) { 1655 /* No such peer. Drop packet. */ 1656 crypto_freereq(crp); 1657 atomic_add_int(&sc->refcount, -1); 1658 OVPN_RUNLOCK(sc); 1659 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1660 m_freem(m); 1661 CURVNET_RESTORE(); 1662 return (0); 1663 } 1664 1665 key = ovpn_find_key(sc, peer, ohdr); 1666 if (key == NULL) { 1667 crypto_freereq(crp); 1668 atomic_add_int(&sc->refcount, -1); 1669 /* 1670 * Has this key been removed between us starting the decrypt 1671 * and finishing it? 1672 */ 1673 OVPN_RUNLOCK(sc); 1674 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1675 m_freem(m); 1676 CURVNET_RESTORE(); 1677 return (0); 1678 } 1679 1680 /* Now remove the outer headers */ 1681 m_adj_decap(m, sizeof(struct udphdr) + 1682 sizeof(struct ovpn_wire_header)); 1683 1684 NET_EPOCH_ENTER(et); 1685 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), _ovpn_lock_trackerp); 1686 NET_EPOCH_EXIT(et); 1687 OVPN_UNLOCK_ASSERT(sc); 1688 1689 CURVNET_RESTORE(); 1690 1691 crypto_freereq(crp); 1692 atomic_add_int(&sc->refcount, -1); 1693 1694 return (0); 1695 } 1696 1697 static int 1698 ovpn_get_af(struct mbuf *m) 1699 { 1700 struct ip *ip; 1701 struct ip6_hdr *ip6; 1702 1703 /* 1704 * We should pullup, but we're only interested in the first byte, so 1705 * that'll always be contiguous. 1706 */ 1707 ip = mtod(m, struct ip *); 1708 if (ip->ip_v == IPVERSION) 1709 return (AF_INET); 1710 1711 ip6 = mtod(m, struct ip6_hdr *); 1712 if ((ip6->ip6_vfc & IPV6_VERSION_MASK) == IPV6_VERSION) 1713 return (AF_INET6); 1714 1715 return (0); 1716 } 1717 1718 static struct ovpn_kpeer * 1719 ovpn_find_peer_by_ip(struct ovpn_softc *sc, const struct in_addr addr) 1720 { 1721 struct ovpn_kpeer peer; 1722 1723 OVPN_ASSERT(sc); 1724 1725 peer.vpn4 = addr; 1726 1727 return (RB_FIND(ovpn_kpeers_by_ip, &sc->peers_by_ip, &peer)); 1728 } 1729 1730 static struct ovpn_kpeer * 1731 ovpn_find_peer_by_ip6(struct ovpn_softc *sc, const struct in6_addr *addr) 1732 { 1733 struct ovpn_kpeer peer; 1734 1735 OVPN_ASSERT(sc); 1736 1737 peer.vpn6 = *addr; 1738 1739 return (RB_FIND(ovpn_kpeers_by_ip6, &sc->peers_by_ip6, &peer)); 1740 } 1741 1742 static struct ovpn_kpeer * 1743 ovpn_route_peer(struct ovpn_softc *sc, struct mbuf **m0, 1744 const struct sockaddr *dst) 1745 { 1746 struct ovpn_kpeer *peer = NULL; 1747 int af; 1748 1749 NET_EPOCH_ASSERT(); 1750 OVPN_ASSERT(sc); 1751 1752 /* Shortcut if we're a client (or are a server and have only one client). */ 1753 if (sc->peercount == 1) 1754 return (ovpn_find_only_peer(sc)); 1755 1756 if (dst != NULL) 1757 af = dst->sa_family; 1758 else 1759 af = ovpn_get_af(*m0); 1760 1761 switch (af) { 1762 #ifdef INET 1763 case AF_INET: { 1764 const struct sockaddr_in *sa = (const struct sockaddr_in *)dst; 1765 struct nhop_object *nh; 1766 const struct in_addr *ip_dst; 1767 1768 if (sa != NULL) { 1769 ip_dst = &sa->sin_addr; 1770 } else { 1771 struct ip *ip; 1772 1773 *m0 = m_pullup(*m0, sizeof(struct ip)); 1774 if (*m0 == NULL) 1775 return (NULL); 1776 ip = mtod(*m0, struct ip *); 1777 ip_dst = &ip->ip_dst; 1778 } 1779 1780 peer = ovpn_find_peer_by_ip(sc, *ip_dst); 1781 SDT_PROBE2(if_ovpn, tx, route, ip4, ip_dst, peer); 1782 if (peer == NULL) { 1783 nh = fib4_lookup(M_GETFIB(*m0), *ip_dst, 0, 1784 NHR_NONE, 0); 1785 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1786 peer = ovpn_find_peer_by_ip(sc, 1787 nh->gw4_sa.sin_addr); 1788 SDT_PROBE2(if_ovpn, tx, route, ip4, 1789 &nh->gw4_sa.sin_addr, peer); 1790 } 1791 } 1792 break; 1793 } 1794 #endif 1795 #ifdef INET6 1796 case AF_INET6: { 1797 const struct sockaddr_in6 *sa6 = 1798 (const struct sockaddr_in6 *)dst; 1799 struct nhop_object *nh; 1800 const struct in6_addr *ip6_dst; 1801 1802 if (sa6 != NULL) { 1803 ip6_dst = &sa6->sin6_addr; 1804 } else { 1805 struct ip6_hdr *ip6; 1806 1807 *m0 = m_pullup(*m0, sizeof(struct ip6_hdr)); 1808 if (*m0 == NULL) 1809 return (NULL); 1810 ip6 = mtod(*m0, struct ip6_hdr *); 1811 ip6_dst = &ip6->ip6_dst; 1812 } 1813 1814 peer = ovpn_find_peer_by_ip6(sc, ip6_dst); 1815 SDT_PROBE2(if_ovpn, tx, route, ip6, ip6_dst, peer); 1816 if (peer == NULL) { 1817 nh = fib6_lookup(M_GETFIB(*m0), ip6_dst, 0, 1818 NHR_NONE, 0); 1819 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1820 peer = ovpn_find_peer_by_ip6(sc, 1821 &nh->gw6_sa.sin6_addr); 1822 SDT_PROBE2(if_ovpn, tx, route, ip6, 1823 &nh->gw6_sa.sin6_addr, peer); 1824 } 1825 } 1826 break; 1827 } 1828 #endif 1829 } 1830 1831 return (peer); 1832 } 1833 1834 static int 1835 ovpn_transmit(struct ifnet *ifp, struct mbuf *m) 1836 { 1837 return (ifp->if_output(ifp, m, NULL, NULL)); 1838 } 1839 1840 static int 1841 ovpn_transmit_to_peer(struct ifnet *ifp, struct mbuf *m, 1842 struct ovpn_kpeer *peer, struct rm_priotracker *_ovpn_lock_trackerp) 1843 { 1844 struct ovpn_wire_header *ohdr; 1845 struct ovpn_kkey *key; 1846 struct ovpn_softc *sc; 1847 struct cryptop *crp; 1848 uint32_t af, seq; 1849 size_t len, ovpn_hdr_len; 1850 int tunnel_len; 1851 int ret; 1852 1853 sc = ifp->if_softc; 1854 1855 OVPN_RASSERT(sc); 1856 1857 tunnel_len = m->m_pkthdr.len; 1858 1859 key = &peer->keys[OVPN_KEY_SLOT_PRIMARY]; 1860 if (key->encrypt == NULL) { 1861 if (_ovpn_lock_trackerp != NULL) 1862 OVPN_RUNLOCK(sc); 1863 m_freem(m); 1864 return (ENOLINK); 1865 } 1866 1867 af = ovpn_get_af(m); 1868 /* Don't capture control packets. */ 1869 if (af != 0) 1870 BPF_MTAP2(ifp, &af, sizeof(af), m); 1871 1872 len = m->m_pkthdr.len; 1873 MPASS(len <= ifp->if_mtu); 1874 1875 ovpn_hdr_len = sizeof(struct ovpn_wire_header); 1876 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) 1877 ovpn_hdr_len -= 16; /* No auth tag. */ 1878 1879 M_PREPEND(m, ovpn_hdr_len, M_NOWAIT); 1880 if (m == NULL) { 1881 if (_ovpn_lock_trackerp != NULL) 1882 OVPN_RUNLOCK(sc); 1883 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1884 return (ENOBUFS); 1885 } 1886 ohdr = mtod(m, struct ovpn_wire_header *); 1887 ohdr->opcode = (OVPN_OP_DATA_V2 << OVPN_OP_SHIFT) | key->keyid; 1888 ohdr->opcode <<= 24; 1889 ohdr->opcode |= key->peerid; 1890 ohdr->opcode = htonl(ohdr->opcode); 1891 1892 seq = atomic_fetchadd_32(&peer->tx_seq, 1); 1893 seq = htonl(seq); 1894 ohdr->seq = seq; 1895 1896 OVPN_PEER_COUNTER_ADD(peer, pkt_out, 1); 1897 OVPN_PEER_COUNTER_ADD(peer, bytes_out, len); 1898 1899 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) { 1900 ret = ovpn_encap(sc, peer->peerid, m); 1901 if (_ovpn_lock_trackerp != NULL) 1902 OVPN_RUNLOCK(sc); 1903 if (ret == 0) { 1904 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1905 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 1906 } 1907 return (ret); 1908 } 1909 1910 crp = crypto_getreq(key->encrypt->cryptoid, M_NOWAIT); 1911 if (crp == NULL) { 1912 if (_ovpn_lock_trackerp != NULL) 1913 OVPN_RUNLOCK(sc); 1914 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1915 m_freem(m); 1916 return (ENOBUFS); 1917 } 1918 1919 /* Encryption covers only the payload, not the header. */ 1920 crp->crp_payload_start = sizeof(*ohdr); 1921 crp->crp_payload_length = len; 1922 crp->crp_op = CRYPTO_OP_ENCRYPT; 1923 1924 /* 1925 * AAD data covers the ovpn_wire_header minus the auth 1926 * tag. 1927 */ 1928 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 1929 crp->crp_aad = ohdr; 1930 crp->crp_aad_start = 0; 1931 crp->crp_op |= CRYPTO_OP_COMPUTE_DIGEST; 1932 crp->crp_digest_start = offsetof(struct ovpn_wire_header, auth_tag); 1933 1934 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 1935 memcpy(crp->crp_iv, &seq, sizeof(seq)); 1936 memcpy(crp->crp_iv + sizeof(seq), key->encrypt->nonce, 1937 key->encrypt->noncelen); 1938 1939 crypto_use_mbuf(crp, m); 1940 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 1941 crp->crp_callback = ovpn_encrypt_tx_cb; 1942 crp->crp_opaque = peer; 1943 1944 atomic_add_int(&peer->refcount, 1); 1945 if (_ovpn_lock_trackerp != NULL) 1946 OVPN_RUNLOCK(sc); 1947 if (V_async_crypto) 1948 ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); 1949 else 1950 ret = crypto_dispatch(crp); 1951 if (ret) { 1952 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1953 } 1954 1955 return (ret); 1956 } 1957 1958 /* 1959 * Note: Expects to hold the read lock on entry, and will release it itself. 1960 */ 1961 static int 1962 ovpn_encap(struct ovpn_softc *sc, uint32_t peerid, struct mbuf *m) 1963 { 1964 struct udphdr *udp; 1965 struct ovpn_kpeer *peer; 1966 int len; 1967 1968 OVPN_RLOCK_TRACKER; 1969 1970 OVPN_RLOCK(sc); 1971 NET_EPOCH_ASSERT(); 1972 1973 peer = ovpn_find_peer(sc, peerid); 1974 if (peer == NULL || sc->ifp->if_link_state != LINK_STATE_UP) { 1975 OVPN_RUNLOCK(sc); 1976 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1977 m_freem(m); 1978 return (ENETDOWN); 1979 } 1980 1981 len = m->m_pkthdr.len; 1982 1983 M_PREPEND(m, sizeof(struct udphdr), M_NOWAIT); 1984 if (m == NULL) { 1985 OVPN_RUNLOCK(sc); 1986 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1987 m_freem(m); 1988 return (ENOBUFS); 1989 } 1990 udp = mtod(m, struct udphdr *); 1991 1992 MPASS(peer->local.ss_family == peer->remote.ss_family); 1993 1994 udp->uh_sport = ovpn_get_port(&peer->local); 1995 udp->uh_dport = ovpn_get_port(&peer->remote); 1996 udp->uh_ulen = htons(sizeof(struct udphdr) + len); 1997 1998 switch (peer->remote.ss_family) { 1999 #ifdef INET 2000 case AF_INET: { 2001 struct sockaddr_in *in_local = TO_IN(&peer->local); 2002 struct sockaddr_in *in_remote = TO_IN(&peer->remote); 2003 struct ip *ip; 2004 2005 /* 2006 * This requires knowing the source IP, which we don't. Happily 2007 * we're allowed to keep this at 0, and the checksum won't do 2008 * anything the crypto won't already do. 2009 */ 2010 udp->uh_sum = 0; 2011 2012 /* Set the checksum flags so we recalculate checksums. */ 2013 m->m_pkthdr.csum_flags |= CSUM_IP; 2014 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 2015 2016 M_PREPEND(m, sizeof(struct ip), M_NOWAIT); 2017 if (m == NULL) { 2018 OVPN_RUNLOCK(sc); 2019 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2020 return (ENOBUFS); 2021 } 2022 ip = mtod(m, struct ip *); 2023 2024 ip->ip_tos = 0; 2025 ip->ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) + 2026 len); 2027 ip->ip_off = 0; 2028 ip->ip_ttl = V_ip_defttl; 2029 ip->ip_p = IPPROTO_UDP; 2030 ip->ip_sum = 0; 2031 if (in_local->sin_port != 0) 2032 ip->ip_src = in_local->sin_addr; 2033 else 2034 ip->ip_src.s_addr = INADDR_ANY; 2035 ip->ip_dst = in_remote->sin_addr; 2036 2037 OVPN_RUNLOCK(sc); 2038 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 2039 2040 return (ip_output(m, NULL, NULL, 0, NULL, NULL)); 2041 } 2042 #endif 2043 #ifdef INET6 2044 case AF_INET6: { 2045 struct sockaddr_in6 *in6_local = TO_IN6(&peer->local); 2046 struct sockaddr_in6 *in6_remote = TO_IN6(&peer->remote); 2047 struct ip6_hdr *ip6; 2048 2049 M_PREPEND(m, sizeof(struct ip6_hdr), M_NOWAIT); 2050 if (m == NULL) { 2051 OVPN_RUNLOCK(sc); 2052 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2053 return (ENOBUFS); 2054 } 2055 m = m_pullup(m, sizeof(*ip6) + sizeof(*udp)); 2056 if (m == NULL) { 2057 OVPN_RUNLOCK(sc); 2058 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2059 return (ENOBUFS); 2060 } 2061 2062 ip6 = mtod(m, struct ip6_hdr *); 2063 2064 ip6->ip6_vfc = IPV6_VERSION; 2065 ip6->ip6_flow &= ~IPV6_FLOWINFO_MASK; 2066 ip6->ip6_plen = htons(sizeof(*ip6) + sizeof(struct udphdr) + 2067 len); 2068 ip6->ip6_nxt = IPPROTO_UDP; 2069 ip6->ip6_hlim = V_ip6_defhlim; 2070 2071 memcpy(&ip6->ip6_src, &in6_local->sin6_addr, 2072 sizeof(ip6->ip6_src)); 2073 memcpy(&ip6->ip6_dst, &in6_remote->sin6_addr, 2074 sizeof(ip6->ip6_dst)); 2075 2076 udp = mtodo(m, sizeof(*ip6)); 2077 udp->uh_sum = in6_cksum_pseudo(ip6, 2078 m->m_pkthdr.len - sizeof(struct ip6_hdr), 2079 IPPROTO_UDP, 0); 2080 2081 m->m_pkthdr.csum_flags |= CSUM_UDP_IPV6; 2082 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 2083 2084 OVPN_RUNLOCK(sc); 2085 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 2086 2087 return (ip6_output(m, NULL, NULL, IPV6_UNSPECSRC, NULL, NULL, 2088 NULL)); 2089 } 2090 #endif 2091 default: 2092 panic("Unsupported address family %d", 2093 peer->remote.ss_family); 2094 } 2095 } 2096 2097 static int 2098 ovpn_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 2099 struct route *ro) 2100 { 2101 struct ovpn_softc *sc; 2102 struct ovpn_kpeer *peer; 2103 2104 OVPN_RLOCK_TRACKER; 2105 2106 sc = ifp->if_softc; 2107 2108 OVPN_RLOCK(sc); 2109 2110 SDT_PROBE1(if_ovpn, tx, transmit, start, m); 2111 2112 if (__predict_false(ifp->if_link_state != LINK_STATE_UP)) { 2113 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2114 OVPN_RUNLOCK(sc); 2115 m_freem(m); 2116 return (ENETDOWN); 2117 } 2118 2119 /** 2120 * Only obey 'dst' (i.e. the gateway) if no route is supplied. 2121 * That's our indication that we're being called through pf's route-to, 2122 * and we should route according to 'dst' instead. We can't do so 2123 * consistently, because the usual openvpn configuration sets the first 2124 * non-server IP in the subnet as the gateway. If we always use that 2125 * one we'd end up routing all traffic to the first client. 2126 * tl;dr: 'ro == NULL' tells us pf is doing a route-to, and then but 2127 * only then, we should treat 'dst' as the destination. */ 2128 peer = ovpn_route_peer(sc, &m, ro == NULL ? dst : NULL); 2129 if (peer == NULL) { 2130 /* No destination. */ 2131 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2132 OVPN_RUNLOCK(sc); 2133 m_freem(m); 2134 return (ENETDOWN); 2135 } 2136 2137 return (ovpn_transmit_to_peer(ifp, m, peer, _ovpn_lock_trackerp)); 2138 } 2139 2140 static bool 2141 ovpn_check_replay(struct ovpn_kkey_dir *key, uint32_t seq) 2142 { 2143 uint32_t d; 2144 2145 mtx_lock(&key->replay_mtx); 2146 2147 /* Sequence number must be strictly greater than rx_seq */ 2148 if (seq <= key->rx_seq) { 2149 mtx_unlock(&key->replay_mtx); 2150 return (false); 2151 } 2152 2153 /* Large jump. The packet authenticated okay, so just accept that. */ 2154 if (seq > (key->rx_seq + (sizeof(key->rx_window) * 8))) { 2155 key->rx_seq = seq; 2156 key->rx_window = 0; 2157 mtx_unlock(&key->replay_mtx); 2158 return (true); 2159 } 2160 2161 /* Happy case. */ 2162 if ((seq == key->rx_seq + 1) && key->rx_window == 0) { 2163 key->rx_seq++; 2164 mtx_unlock(&key->replay_mtx); 2165 return (true); 2166 } 2167 2168 d = seq - key->rx_seq - 1; 2169 2170 if (key->rx_window & ((uint64_t)1 << d)) { 2171 /* Dupe! */ 2172 mtx_unlock(&key->replay_mtx); 2173 return (false); 2174 } 2175 2176 key->rx_window |= (uint64_t)1 << d; 2177 2178 while (key->rx_window & 1) { 2179 key->rx_seq++; 2180 key->rx_window >>= 1; 2181 } 2182 2183 mtx_unlock(&key->replay_mtx); 2184 2185 return (true); 2186 } 2187 2188 static struct ovpn_kpeer * 2189 ovpn_peer_from_mbuf(struct ovpn_softc *sc, struct mbuf *m, int off) 2190 { 2191 struct ovpn_wire_header ohdr; 2192 uint32_t peerid; 2193 const size_t hdrlen = sizeof(ohdr) - sizeof(ohdr.auth_tag); 2194 2195 OVPN_RASSERT(sc); 2196 2197 if (m_length(m, NULL) < (off + sizeof(struct udphdr) + hdrlen)) 2198 return (NULL); 2199 2200 m_copydata(m, off + sizeof(struct udphdr), hdrlen, (caddr_t)&ohdr); 2201 2202 peerid = ntohl(ohdr.opcode) & 0x00ffffff; 2203 2204 return (ovpn_find_peer(sc, peerid)); 2205 } 2206 2207 static bool 2208 ovpn_udp_input(struct mbuf *m, int off, struct inpcb *inp, 2209 const struct sockaddr *sa, void *ctx) 2210 { 2211 struct ovpn_softc *sc = ctx; 2212 struct ovpn_wire_header tmphdr; 2213 struct ovpn_wire_header *ohdr; 2214 struct udphdr *uhdr; 2215 struct ovpn_kkey *key; 2216 struct cryptop *crp; 2217 struct ovpn_kpeer *peer; 2218 size_t ohdrlen; 2219 int ret; 2220 uint8_t op; 2221 2222 OVPN_RLOCK_TRACKER; 2223 2224 M_ASSERTPKTHDR(m); 2225 2226 OVPN_COUNTER_ADD(sc, transport_bytes_received, m->m_pkthdr.len - off); 2227 2228 ohdrlen = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2229 2230 OVPN_RLOCK(sc); 2231 2232 peer = ovpn_peer_from_mbuf(sc, m, off); 2233 if (peer == NULL) { 2234 OVPN_RUNLOCK(sc); 2235 return (false); 2236 } 2237 2238 if (m_length(m, NULL) < (off + sizeof(*uhdr) + ohdrlen)) { 2239 /* Short packet. */ 2240 OVPN_RUNLOCK(sc); 2241 return (false); 2242 } 2243 2244 m_copydata(m, off + sizeof(*uhdr), ohdrlen, (caddr_t)&tmphdr); 2245 2246 op = ntohl(tmphdr.opcode) >> 24 >> OVPN_OP_SHIFT; 2247 if (op != OVPN_OP_DATA_V2) { 2248 /* Control packet? */ 2249 OVPN_RUNLOCK(sc); 2250 return (false); 2251 } 2252 2253 m = m_pullup(m, off + sizeof(*uhdr) + ohdrlen); 2254 if (m == NULL) { 2255 OVPN_RUNLOCK(sc); 2256 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2257 return (true); 2258 } 2259 2260 /* 2261 * Simplify things by getting rid of the preceding headers, we don't 2262 * care about them. 2263 */ 2264 m_adj_decap(m, off); 2265 2266 uhdr = mtodo(m, 0); 2267 ohdr = mtodo(m, sizeof(*uhdr)); 2268 2269 key = ovpn_find_key(sc, peer, ohdr); 2270 if (key == NULL || key->decrypt == NULL) { 2271 OVPN_RUNLOCK(sc); 2272 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2273 m_freem(m); 2274 return (true); 2275 } 2276 2277 if (key->decrypt->cipher == OVPN_CIPHER_ALG_NONE) { 2278 /* Now remove the outer headers */ 2279 m_adj_decap(m, sizeof(struct udphdr) + ohdrlen); 2280 2281 ohdr = mtodo(m, sizeof(*uhdr)); 2282 2283 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), 2284 _ovpn_lock_trackerp); 2285 OVPN_UNLOCK_ASSERT(sc); 2286 return (true); 2287 } 2288 2289 ohdrlen += sizeof(ohdr->auth_tag); 2290 2291 m = m_pullup(m, sizeof(*uhdr) + ohdrlen); 2292 if (m == NULL) { 2293 OVPN_RUNLOCK(sc); 2294 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2295 return (true); 2296 } 2297 uhdr = mtodo(m, 0); 2298 ohdr = mtodo(m, sizeof(*uhdr)); 2299 2300 /* Decrypt */ 2301 crp = crypto_getreq(key->decrypt->cryptoid, M_NOWAIT); 2302 if (crp == NULL) { 2303 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2304 OVPN_RUNLOCK(sc); 2305 m_freem(m); 2306 return (true); 2307 } 2308 2309 crp->crp_payload_start = sizeof(struct udphdr) + sizeof(*ohdr); 2310 crp->crp_payload_length = ntohs(uhdr->uh_ulen) - 2311 sizeof(*uhdr) - sizeof(*ohdr); 2312 crp->crp_op = CRYPTO_OP_DECRYPT; 2313 2314 /* AAD validation. */ 2315 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2316 crp->crp_aad = ohdr; 2317 crp->crp_aad_start = 0; 2318 crp->crp_op |= CRYPTO_OP_VERIFY_DIGEST; 2319 crp->crp_digest_start = sizeof(struct udphdr) + 2320 offsetof(struct ovpn_wire_header, auth_tag); 2321 2322 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 2323 memcpy(crp->crp_iv, &ohdr->seq, sizeof(ohdr->seq)); 2324 memcpy(crp->crp_iv + sizeof(ohdr->seq), key->decrypt->nonce, 2325 key->decrypt->noncelen); 2326 2327 crypto_use_mbuf(crp, m); 2328 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 2329 crp->crp_callback = ovpn_decrypt_rx_cb; 2330 crp->crp_opaque = sc; 2331 2332 atomic_add_int(&sc->refcount, 1); 2333 OVPN_RUNLOCK(sc); 2334 if (V_async_crypto) 2335 ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); 2336 else 2337 ret = crypto_dispatch(crp); 2338 if (ret != 0) { 2339 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2340 } 2341 2342 return (true); 2343 } 2344 2345 static void 2346 ovpn_qflush(struct ifnet *ifp __unused) 2347 { 2348 2349 } 2350 2351 static void 2352 ovpn_flush_rxring(struct ovpn_softc *sc) 2353 { 2354 struct ovpn_notification *n; 2355 2356 OVPN_WASSERT(sc); 2357 2358 while (! buf_ring_empty(sc->notifring)) { 2359 n = buf_ring_dequeue_sc(sc->notifring); 2360 free(n, M_OVPN); 2361 } 2362 } 2363 2364 #ifdef VIMAGE 2365 static void 2366 ovpn_reassign(struct ifnet *ifp, struct vnet *new_vnet __unused, 2367 char *unused __unused) 2368 { 2369 struct ovpn_softc *sc = ifp->if_softc; 2370 struct ovpn_kpeer *peer, *tmppeer; 2371 int ret __diagused; 2372 2373 OVPN_WLOCK(sc); 2374 2375 /* Flush keys & configuration. */ 2376 RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) { 2377 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 2378 ret = _ovpn_del_peer(sc, peer); 2379 MPASS(ret == 0); 2380 } 2381 2382 ovpn_flush_rxring(sc); 2383 2384 OVPN_WUNLOCK(sc); 2385 } 2386 #endif 2387 2388 static int 2389 ovpn_clone_match(struct if_clone *ifc, const char *name) 2390 { 2391 /* 2392 * Allow all names that start with 'ovpn', specifically because pfSense 2393 * uses ovpnc1 / ovpns2 2394 */ 2395 return (strncmp(ovpnname, name, strlen(ovpnname)) == 0); 2396 } 2397 2398 static int 2399 ovpn_clone_create(struct if_clone *ifc, char *name, size_t len, 2400 struct ifc_data *ifd, struct ifnet **ifpp) 2401 { 2402 struct ovpn_softc *sc; 2403 struct ifnet *ifp; 2404 char *dp; 2405 int error, unit, wildcard; 2406 2407 /* Try to see if a special unit was requested. */ 2408 error = ifc_name2unit(name, &unit); 2409 if (error != 0) 2410 return (error); 2411 wildcard = (unit < 0); 2412 2413 error = ifc_alloc_unit(ifc, &unit); 2414 if (error != 0) 2415 return (error); 2416 2417 /* 2418 * If no unit had been given, we need to adjust the ifName. 2419 */ 2420 for (dp = name; *dp != '\0'; dp++); 2421 if (wildcard) { 2422 error = snprintf(dp, len - (dp - name), "%d", unit); 2423 if (error > len - (dp - name)) { 2424 /* ifName too long. */ 2425 ifc_free_unit(ifc, unit); 2426 return (ENOSPC); 2427 } 2428 dp += error; 2429 } 2430 2431 /* Make sure it doesn't already exist. */ 2432 if (ifunit(name) != NULL) 2433 return (EEXIST); 2434 2435 sc = malloc(sizeof(struct ovpn_softc), M_OVPN, M_WAITOK | M_ZERO); 2436 sc->ifp = if_alloc(IFT_ENC); 2437 rm_init_flags(&sc->lock, "if_ovpn_lock", RM_RECURSE); 2438 sc->refcount = 0; 2439 2440 sc->notifring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL); 2441 2442 COUNTER_ARRAY_ALLOC(sc->counters, OVPN_COUNTER_SIZE, M_WAITOK); 2443 2444 ifp = sc->ifp; 2445 ifp->if_softc = sc; 2446 strlcpy(ifp->if_xname, name, IFNAMSIZ); 2447 ifp->if_dname = ovpngroupname; 2448 ifp->if_dunit = unit; 2449 2450 ifp->if_addrlen = 0; 2451 ifp->if_mtu = 1428; 2452 ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST; 2453 ifp->if_ioctl = ovpn_ioctl; 2454 ifp->if_transmit = ovpn_transmit; 2455 ifp->if_output = ovpn_output; 2456 ifp->if_qflush = ovpn_qflush; 2457 #ifdef VIMAGE 2458 ifp->if_reassign = ovpn_reassign; 2459 #endif 2460 ifp->if_capabilities |= IFCAP_LINKSTATE; 2461 ifp->if_capenable |= IFCAP_LINKSTATE; 2462 2463 if_attach(ifp); 2464 bpfattach(ifp, DLT_NULL, sizeof(uint32_t)); 2465 *ifpp = ifp; 2466 2467 return (0); 2468 } 2469 2470 static void 2471 ovpn_clone_destroy_cb(struct epoch_context *ctx) 2472 { 2473 struct ovpn_softc *sc; 2474 2475 sc = __containerof(ctx, struct ovpn_softc, epoch_ctx); 2476 2477 MPASS(sc->peercount == 0); 2478 MPASS(RB_EMPTY(&sc->peers)); 2479 2480 COUNTER_ARRAY_FREE(sc->counters, OVPN_COUNTER_SIZE); 2481 2482 if_free(sc->ifp); 2483 free(sc, M_OVPN); 2484 } 2485 2486 static int 2487 ovpn_clone_destroy(struct if_clone *ifc, struct ifnet *ifp, uint32_t flags) 2488 { 2489 struct ovpn_softc *sc; 2490 struct ovpn_kpeer *peer, *tmppeer; 2491 int unit; 2492 int ret __diagused; 2493 2494 sc = ifp->if_softc; 2495 unit = ifp->if_dunit; 2496 2497 OVPN_WLOCK(sc); 2498 2499 if (atomic_load_int(&sc->refcount) > 0) { 2500 OVPN_WUNLOCK(sc); 2501 return (EBUSY); 2502 } 2503 2504 RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) { 2505 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 2506 ret = _ovpn_del_peer(sc, peer); 2507 MPASS(ret == 0); 2508 } 2509 2510 ovpn_flush_rxring(sc); 2511 buf_ring_free(sc->notifring, M_OVPN); 2512 2513 OVPN_WUNLOCK(sc); 2514 2515 bpfdetach(ifp); 2516 if_detach(ifp); 2517 ifp->if_softc = NULL; 2518 2519 NET_EPOCH_CALL(ovpn_clone_destroy_cb, &sc->epoch_ctx); 2520 2521 if (unit != IF_DUNIT_NONE) 2522 ifc_free_unit(ifc, unit); 2523 2524 NET_EPOCH_DRAIN_CALLBACKS(); 2525 2526 return (0); 2527 } 2528 2529 static void 2530 vnet_ovpn_init(const void *unused __unused) 2531 { 2532 struct if_clone_addreq req = { 2533 .match_f = ovpn_clone_match, 2534 .create_f = ovpn_clone_create, 2535 .destroy_f = ovpn_clone_destroy, 2536 }; 2537 V_ovpn_cloner = ifc_attach_cloner(ovpngroupname, &req); 2538 } 2539 VNET_SYSINIT(vnet_ovpn_init, SI_SUB_PSEUDO, SI_ORDER_ANY, 2540 vnet_ovpn_init, NULL); 2541 2542 static void 2543 vnet_ovpn_uninit(const void *unused __unused) 2544 { 2545 if_clone_detach(V_ovpn_cloner); 2546 } 2547 VNET_SYSUNINIT(vnet_ovpn_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY, 2548 vnet_ovpn_uninit, NULL); 2549 2550 static int 2551 ovpnmodevent(module_t mod, int type, void *data) 2552 { 2553 switch (type) { 2554 case MOD_LOAD: 2555 /* Done in vnet_ovpn_init() */ 2556 break; 2557 case MOD_UNLOAD: 2558 /* Done in vnet_ovpn_uninit() */ 2559 break; 2560 default: 2561 return (EOPNOTSUPP); 2562 } 2563 2564 return (0); 2565 } 2566 2567 static moduledata_t ovpn_mod = { 2568 "if_ovpn", 2569 ovpnmodevent, 2570 0 2571 }; 2572 2573 DECLARE_MODULE(if_ovpn, ovpn_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 2574 MODULE_VERSION(if_ovpn, 1); 2575