1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 4 */ 5 6 #include "queueing.h" 7 #include "socket.h" 8 #include "timers.h" 9 #include "device.h" 10 #include "ratelimiter.h" 11 #include "peer.h" 12 #include "messages.h" 13 14 #include <linux/module.h> 15 #include <linux/rtnetlink.h> 16 #include <linux/inet.h> 17 #include <linux/netdevice.h> 18 #include <linux/inetdevice.h> 19 #include <linux/if_arp.h> 20 #include <linux/icmp.h> 21 #include <linux/suspend.h> 22 #include <net/icmp.h> 23 #include <net/rtnetlink.h> 24 #include <net/ip_tunnels.h> 25 #include <net/addrconf.h> 26 27 static LIST_HEAD(device_list); 28 29 static int wg_open(struct net_device *dev) 30 { 31 struct in_device *dev_v4 = __in_dev_get_rtnl(dev); 32 struct inet6_dev *dev_v6 = __in6_dev_get(dev); 33 struct wg_device *wg = netdev_priv(dev); 34 struct wg_peer *peer; 35 int ret; 36 37 if (dev_v4) { 38 /* At some point we might put this check near the ip_rt_send_ 39 * redirect call of ip_forward in net/ipv4/ip_forward.c, similar 40 * to the current secpath check. 41 */ 42 IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false); 43 IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false; 44 } 45 if (dev_v6) 46 dev_v6->cnf.addr_gen_mode = IN6_ADDR_GEN_MODE_NONE; 47 48 mutex_lock(&wg->device_update_lock); 49 ret = wg_socket_init(wg, wg->incoming_port); 50 if (ret < 0) 51 goto out; 52 list_for_each_entry(peer, &wg->peer_list, peer_list) { 53 wg_packet_send_staged_packets(peer); 54 if (peer->persistent_keepalive_interval) 55 wg_packet_send_keepalive(peer); 56 } 57 out: 58 mutex_unlock(&wg->device_update_lock); 59 return ret; 60 } 61 62 #ifdef CONFIG_PM_SLEEP 63 static int wg_pm_notification(struct notifier_block *nb, unsigned long action, 64 void *data) 65 { 66 struct wg_device *wg; 67 struct wg_peer *peer; 68 69 /* If the machine is constantly suspending and resuming, as part of 70 * its normal operation rather than as a somewhat rare event, then we 71 * don't actually want to clear keys. 72 */ 73 if (IS_ENABLED(CONFIG_PM_AUTOSLEEP) || IS_ENABLED(CONFIG_ANDROID)) 74 return 0; 75 76 if (action != PM_HIBERNATION_PREPARE && action != PM_SUSPEND_PREPARE) 77 return 0; 78 79 rtnl_lock(); 80 list_for_each_entry(wg, &device_list, device_list) { 81 mutex_lock(&wg->device_update_lock); 82 list_for_each_entry(peer, &wg->peer_list, peer_list) { 83 del_timer(&peer->timer_zero_key_material); 84 wg_noise_handshake_clear(&peer->handshake); 85 wg_noise_keypairs_clear(&peer->keypairs); 86 } 87 mutex_unlock(&wg->device_update_lock); 88 } 89 rtnl_unlock(); 90 rcu_barrier(); 91 return 0; 92 } 93 94 static struct notifier_block pm_notifier = { .notifier_call = wg_pm_notification }; 95 #endif 96 97 static int wg_stop(struct net_device *dev) 98 { 99 struct wg_device *wg = netdev_priv(dev); 100 struct wg_peer *peer; 101 102 mutex_lock(&wg->device_update_lock); 103 list_for_each_entry(peer, &wg->peer_list, peer_list) { 104 wg_packet_purge_staged_packets(peer); 105 wg_timers_stop(peer); 106 wg_noise_handshake_clear(&peer->handshake); 107 wg_noise_keypairs_clear(&peer->keypairs); 108 wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); 109 } 110 mutex_unlock(&wg->device_update_lock); 111 skb_queue_purge(&wg->incoming_handshakes); 112 wg_socket_reinit(wg, NULL, NULL); 113 return 0; 114 } 115 116 static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev) 117 { 118 struct wg_device *wg = netdev_priv(dev); 119 struct sk_buff_head packets; 120 struct wg_peer *peer; 121 struct sk_buff *next; 122 sa_family_t family; 123 u32 mtu; 124 int ret; 125 126 if (unlikely(!wg_check_packet_protocol(skb))) { 127 ret = -EPROTONOSUPPORT; 128 net_dbg_ratelimited("%s: Invalid IP packet\n", dev->name); 129 goto err; 130 } 131 132 peer = wg_allowedips_lookup_dst(&wg->peer_allowedips, skb); 133 if (unlikely(!peer)) { 134 ret = -ENOKEY; 135 if (skb->protocol == htons(ETH_P_IP)) 136 net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI4\n", 137 dev->name, &ip_hdr(skb)->daddr); 138 else if (skb->protocol == htons(ETH_P_IPV6)) 139 net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI6\n", 140 dev->name, &ipv6_hdr(skb)->daddr); 141 goto err_icmp; 142 } 143 144 family = READ_ONCE(peer->endpoint.addr.sa_family); 145 if (unlikely(family != AF_INET && family != AF_INET6)) { 146 ret = -EDESTADDRREQ; 147 net_dbg_ratelimited("%s: No valid endpoint has been configured or discovered for peer %llu\n", 148 dev->name, peer->internal_id); 149 goto err_peer; 150 } 151 152 mtu = skb_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu; 153 154 __skb_queue_head_init(&packets); 155 if (!skb_is_gso(skb)) { 156 skb_mark_not_on_list(skb); 157 } else { 158 struct sk_buff *segs = skb_gso_segment(skb, 0); 159 160 if (IS_ERR(segs)) { 161 ret = PTR_ERR(segs); 162 goto err_peer; 163 } 164 dev_kfree_skb(skb); 165 skb = segs; 166 } 167 168 skb_list_walk_safe(skb, skb, next) { 169 skb_mark_not_on_list(skb); 170 171 skb = skb_share_check(skb, GFP_ATOMIC); 172 if (unlikely(!skb)) 173 continue; 174 175 /* We only need to keep the original dst around for icmp, 176 * so at this point we're in a position to drop it. 177 */ 178 skb_dst_drop(skb); 179 180 PACKET_CB(skb)->mtu = mtu; 181 182 __skb_queue_tail(&packets, skb); 183 } 184 185 spin_lock_bh(&peer->staged_packet_queue.lock); 186 /* If the queue is getting too big, we start removing the oldest packets 187 * until it's small again. We do this before adding the new packet, so 188 * we don't remove GSO segments that are in excess. 189 */ 190 while (skb_queue_len(&peer->staged_packet_queue) > MAX_STAGED_PACKETS) { 191 dev_kfree_skb(__skb_dequeue(&peer->staged_packet_queue)); 192 ++dev->stats.tx_dropped; 193 } 194 skb_queue_splice_tail(&packets, &peer->staged_packet_queue); 195 spin_unlock_bh(&peer->staged_packet_queue.lock); 196 197 wg_packet_send_staged_packets(peer); 198 199 wg_peer_put(peer); 200 return NETDEV_TX_OK; 201 202 err_peer: 203 wg_peer_put(peer); 204 err_icmp: 205 if (skb->protocol == htons(ETH_P_IP)) 206 icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0); 207 else if (skb->protocol == htons(ETH_P_IPV6)) 208 icmpv6_ndo_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0); 209 err: 210 ++dev->stats.tx_errors; 211 kfree_skb(skb); 212 return ret; 213 } 214 215 static const struct net_device_ops netdev_ops = { 216 .ndo_open = wg_open, 217 .ndo_stop = wg_stop, 218 .ndo_start_xmit = wg_xmit, 219 .ndo_get_stats64 = dev_get_tstats64 220 }; 221 222 static void wg_destruct(struct net_device *dev) 223 { 224 struct wg_device *wg = netdev_priv(dev); 225 226 rtnl_lock(); 227 list_del(&wg->device_list); 228 rtnl_unlock(); 229 mutex_lock(&wg->device_update_lock); 230 rcu_assign_pointer(wg->creating_net, NULL); 231 wg->incoming_port = 0; 232 wg_socket_reinit(wg, NULL, NULL); 233 /* The final references are cleared in the below calls to destroy_workqueue. */ 234 wg_peer_remove_all(wg); 235 destroy_workqueue(wg->handshake_receive_wq); 236 destroy_workqueue(wg->handshake_send_wq); 237 destroy_workqueue(wg->packet_crypt_wq); 238 wg_packet_queue_free(&wg->decrypt_queue); 239 wg_packet_queue_free(&wg->encrypt_queue); 240 rcu_barrier(); /* Wait for all the peers to be actually freed. */ 241 wg_ratelimiter_uninit(); 242 memzero_explicit(&wg->static_identity, sizeof(wg->static_identity)); 243 skb_queue_purge(&wg->incoming_handshakes); 244 free_percpu(dev->tstats); 245 free_percpu(wg->incoming_handshakes_worker); 246 kvfree(wg->index_hashtable); 247 kvfree(wg->peer_hashtable); 248 mutex_unlock(&wg->device_update_lock); 249 250 pr_debug("%s: Interface destroyed\n", dev->name); 251 free_netdev(dev); 252 } 253 254 static const struct device_type device_type = { .name = KBUILD_MODNAME }; 255 256 static void wg_setup(struct net_device *dev) 257 { 258 struct wg_device *wg = netdev_priv(dev); 259 enum { WG_NETDEV_FEATURES = NETIF_F_HW_CSUM | NETIF_F_RXCSUM | 260 NETIF_F_SG | NETIF_F_GSO | 261 NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA }; 262 const int overhead = MESSAGE_MINIMUM_LENGTH + sizeof(struct udphdr) + 263 max(sizeof(struct ipv6hdr), sizeof(struct iphdr)); 264 265 dev->netdev_ops = &netdev_ops; 266 dev->header_ops = &ip_tunnel_header_ops; 267 dev->hard_header_len = 0; 268 dev->addr_len = 0; 269 dev->needed_headroom = DATA_PACKET_HEAD_ROOM; 270 dev->needed_tailroom = noise_encrypted_len(MESSAGE_PADDING_MULTIPLE); 271 dev->type = ARPHRD_NONE; 272 dev->flags = IFF_POINTOPOINT | IFF_NOARP; 273 dev->priv_flags |= IFF_NO_QUEUE; 274 dev->features |= NETIF_F_LLTX; 275 dev->features |= WG_NETDEV_FEATURES; 276 dev->hw_features |= WG_NETDEV_FEATURES; 277 dev->hw_enc_features |= WG_NETDEV_FEATURES; 278 dev->mtu = ETH_DATA_LEN - overhead; 279 dev->max_mtu = round_down(INT_MAX, MESSAGE_PADDING_MULTIPLE) - overhead; 280 281 SET_NETDEV_DEVTYPE(dev, &device_type); 282 283 /* We need to keep the dst around in case of icmp replies. */ 284 netif_keep_dst(dev); 285 286 memset(wg, 0, sizeof(*wg)); 287 wg->dev = dev; 288 } 289 290 static int wg_newlink(struct net *src_net, struct net_device *dev, 291 struct nlattr *tb[], struct nlattr *data[], 292 struct netlink_ext_ack *extack) 293 { 294 struct wg_device *wg = netdev_priv(dev); 295 int ret = -ENOMEM; 296 297 rcu_assign_pointer(wg->creating_net, src_net); 298 init_rwsem(&wg->static_identity.lock); 299 mutex_init(&wg->socket_update_lock); 300 mutex_init(&wg->device_update_lock); 301 skb_queue_head_init(&wg->incoming_handshakes); 302 wg_allowedips_init(&wg->peer_allowedips); 303 wg_cookie_checker_init(&wg->cookie_checker, wg); 304 INIT_LIST_HEAD(&wg->peer_list); 305 wg->device_update_gen = 1; 306 307 wg->peer_hashtable = wg_pubkey_hashtable_alloc(); 308 if (!wg->peer_hashtable) 309 return ret; 310 311 wg->index_hashtable = wg_index_hashtable_alloc(); 312 if (!wg->index_hashtable) 313 goto err_free_peer_hashtable; 314 315 dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); 316 if (!dev->tstats) 317 goto err_free_index_hashtable; 318 319 wg->incoming_handshakes_worker = 320 wg_packet_percpu_multicore_worker_alloc( 321 wg_packet_handshake_receive_worker, wg); 322 if (!wg->incoming_handshakes_worker) 323 goto err_free_tstats; 324 325 wg->handshake_receive_wq = alloc_workqueue("wg-kex-%s", 326 WQ_CPU_INTENSIVE | WQ_FREEZABLE, 0, dev->name); 327 if (!wg->handshake_receive_wq) 328 goto err_free_incoming_handshakes; 329 330 wg->handshake_send_wq = alloc_workqueue("wg-kex-%s", 331 WQ_UNBOUND | WQ_FREEZABLE, 0, dev->name); 332 if (!wg->handshake_send_wq) 333 goto err_destroy_handshake_receive; 334 335 wg->packet_crypt_wq = alloc_workqueue("wg-crypt-%s", 336 WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, dev->name); 337 if (!wg->packet_crypt_wq) 338 goto err_destroy_handshake_send; 339 340 ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker, 341 MAX_QUEUED_PACKETS); 342 if (ret < 0) 343 goto err_destroy_packet_crypt; 344 345 ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker, 346 MAX_QUEUED_PACKETS); 347 if (ret < 0) 348 goto err_free_encrypt_queue; 349 350 ret = wg_ratelimiter_init(); 351 if (ret < 0) 352 goto err_free_decrypt_queue; 353 354 ret = register_netdevice(dev); 355 if (ret < 0) 356 goto err_uninit_ratelimiter; 357 358 list_add(&wg->device_list, &device_list); 359 360 /* We wait until the end to assign priv_destructor, so that 361 * register_netdevice doesn't call it for us if it fails. 362 */ 363 dev->priv_destructor = wg_destruct; 364 365 pr_debug("%s: Interface created\n", dev->name); 366 return ret; 367 368 err_uninit_ratelimiter: 369 wg_ratelimiter_uninit(); 370 err_free_decrypt_queue: 371 wg_packet_queue_free(&wg->decrypt_queue); 372 err_free_encrypt_queue: 373 wg_packet_queue_free(&wg->encrypt_queue); 374 err_destroy_packet_crypt: 375 destroy_workqueue(wg->packet_crypt_wq); 376 err_destroy_handshake_send: 377 destroy_workqueue(wg->handshake_send_wq); 378 err_destroy_handshake_receive: 379 destroy_workqueue(wg->handshake_receive_wq); 380 err_free_incoming_handshakes: 381 free_percpu(wg->incoming_handshakes_worker); 382 err_free_tstats: 383 free_percpu(dev->tstats); 384 err_free_index_hashtable: 385 kvfree(wg->index_hashtable); 386 err_free_peer_hashtable: 387 kvfree(wg->peer_hashtable); 388 return ret; 389 } 390 391 static struct rtnl_link_ops link_ops __read_mostly = { 392 .kind = KBUILD_MODNAME, 393 .priv_size = sizeof(struct wg_device), 394 .setup = wg_setup, 395 .newlink = wg_newlink, 396 }; 397 398 static void wg_netns_pre_exit(struct net *net) 399 { 400 struct wg_device *wg; 401 402 rtnl_lock(); 403 list_for_each_entry(wg, &device_list, device_list) { 404 if (rcu_access_pointer(wg->creating_net) == net) { 405 pr_debug("%s: Creating namespace exiting\n", wg->dev->name); 406 netif_carrier_off(wg->dev); 407 mutex_lock(&wg->device_update_lock); 408 rcu_assign_pointer(wg->creating_net, NULL); 409 wg_socket_reinit(wg, NULL, NULL); 410 mutex_unlock(&wg->device_update_lock); 411 } 412 } 413 rtnl_unlock(); 414 } 415 416 static struct pernet_operations pernet_ops = { 417 .pre_exit = wg_netns_pre_exit 418 }; 419 420 int __init wg_device_init(void) 421 { 422 int ret; 423 424 #ifdef CONFIG_PM_SLEEP 425 ret = register_pm_notifier(&pm_notifier); 426 if (ret) 427 return ret; 428 #endif 429 430 ret = register_pernet_device(&pernet_ops); 431 if (ret) 432 goto error_pm; 433 434 ret = rtnl_link_register(&link_ops); 435 if (ret) 436 goto error_pernet; 437 438 return 0; 439 440 error_pernet: 441 unregister_pernet_device(&pernet_ops); 442 error_pm: 443 #ifdef CONFIG_PM_SLEEP 444 unregister_pm_notifier(&pm_notifier); 445 #endif 446 return ret; 447 } 448 449 void wg_device_uninit(void) 450 { 451 rtnl_link_unregister(&link_ops); 452 unregister_pernet_device(&pernet_ops); 453 #ifdef CONFIG_PM_SLEEP 454 unregister_pm_notifier(&pm_notifier); 455 #endif 456 rcu_barrier(); 457 } 458