1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 4 */ 5 6 #include "queueing.h" 7 #include "device.h" 8 #include "peer.h" 9 #include "timers.h" 10 #include "messages.h" 11 #include "cookie.h" 12 #include "socket.h" 13 14 #include <linux/ip.h> 15 #include <linux/ipv6.h> 16 #include <linux/udp.h> 17 #include <net/ip_tunnels.h> 18 19 /* Must be called with bh disabled. */ 20 static void update_rx_stats(struct wg_peer *peer, size_t len) 21 { 22 dev_sw_netstats_rx_add(peer->device->dev, len); 23 peer->rx_bytes += len; 24 } 25 26 #define SKB_TYPE_LE32(skb) (((struct message_header *)(skb)->data)->type) 27 28 static size_t validate_header_len(struct sk_buff *skb) 29 { 30 if (unlikely(skb->len < sizeof(struct message_header))) 31 return 0; 32 if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_DATA) && 33 skb->len >= MESSAGE_MINIMUM_LENGTH) 34 return sizeof(struct message_data); 35 if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION) && 36 skb->len == sizeof(struct message_handshake_initiation)) 37 return sizeof(struct message_handshake_initiation); 38 if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE) && 39 skb->len == sizeof(struct message_handshake_response)) 40 return sizeof(struct message_handshake_response); 41 if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE) && 42 skb->len == sizeof(struct message_handshake_cookie)) 43 return sizeof(struct message_handshake_cookie); 44 return 0; 45 } 46 47 static int prepare_skb_header(struct sk_buff *skb, struct wg_device *wg) 48 { 49 size_t data_offset, data_len, header_len; 50 struct udphdr *udp; 51 52 if (unlikely(!wg_check_packet_protocol(skb) || 53 skb_transport_header(skb) < skb->head || 54 (skb_transport_header(skb) + sizeof(struct udphdr)) > 55 skb_tail_pointer(skb))) 56 return -EINVAL; /* Bogus IP header */ 57 udp = udp_hdr(skb); 58 data_offset = (u8 *)udp - skb->data; 59 if (unlikely(data_offset > U16_MAX || 60 data_offset + sizeof(struct udphdr) > skb->len)) 61 /* Packet has offset at impossible location or isn't big enough 62 * to have UDP fields. 63 */ 64 return -EINVAL; 65 data_len = ntohs(udp->len); 66 if (unlikely(data_len < sizeof(struct udphdr) || 67 data_len > skb->len - data_offset)) 68 /* UDP packet is reporting too small of a size or lying about 69 * its size. 70 */ 71 return -EINVAL; 72 data_len -= sizeof(struct udphdr); 73 data_offset = (u8 *)udp + sizeof(struct udphdr) - skb->data; 74 if (unlikely(!pskb_may_pull(skb, 75 data_offset + sizeof(struct message_header)) || 76 pskb_trim(skb, data_len + data_offset) < 0)) 77 return -EINVAL; 78 skb_pull(skb, data_offset); 79 if (unlikely(skb->len != data_len)) 80 /* Final len does not agree with calculated len */ 81 return -EINVAL; 82 header_len = validate_header_len(skb); 83 if (unlikely(!header_len)) 84 return -EINVAL; 85 __skb_push(skb, data_offset); 86 if (unlikely(!pskb_may_pull(skb, data_offset + header_len))) 87 return -EINVAL; 88 __skb_pull(skb, data_offset); 89 return 0; 90 } 91 92 static void wg_receive_handshake_packet(struct wg_device *wg, 93 struct sk_buff *skb) 94 { 95 enum cookie_mac_state mac_state; 96 struct wg_peer *peer = NULL; 97 /* This is global, so that our load calculation applies to the whole 98 * system. We don't care about races with it at all. 99 */ 100 static u64 last_under_load; 101 bool packet_needs_cookie; 102 bool under_load; 103 104 if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE)) { 105 net_dbg_skb_ratelimited("%s: Receiving cookie response from %pISpfsc\n", 106 wg->dev->name, skb); 107 wg_cookie_message_consume( 108 (struct message_handshake_cookie *)skb->data, wg); 109 return; 110 } 111 112 under_load = atomic_read(&wg->handshake_queue_len) >= 113 MAX_QUEUED_INCOMING_HANDSHAKES / 8; 114 if (under_load) { 115 last_under_load = ktime_get_coarse_boottime_ns(); 116 } else if (last_under_load) { 117 under_load = !wg_birthdate_has_expired(last_under_load, 1); 118 if (!under_load) 119 last_under_load = 0; 120 } 121 mac_state = wg_cookie_validate_packet(&wg->cookie_checker, skb, 122 under_load); 123 if ((under_load && mac_state == VALID_MAC_WITH_COOKIE) || 124 (!under_load && mac_state == VALID_MAC_BUT_NO_COOKIE)) { 125 packet_needs_cookie = false; 126 } else if (under_load && mac_state == VALID_MAC_BUT_NO_COOKIE) { 127 packet_needs_cookie = true; 128 } else { 129 net_dbg_skb_ratelimited("%s: Invalid MAC of handshake, dropping packet from %pISpfsc\n", 130 wg->dev->name, skb); 131 return; 132 } 133 134 switch (SKB_TYPE_LE32(skb)) { 135 case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): { 136 struct message_handshake_initiation *message = 137 (struct message_handshake_initiation *)skb->data; 138 139 if (packet_needs_cookie) { 140 wg_packet_send_handshake_cookie(wg, skb, 141 message->sender_index); 142 return; 143 } 144 peer = wg_noise_handshake_consume_initiation(message, wg); 145 if (unlikely(!peer)) { 146 net_dbg_skb_ratelimited("%s: Invalid handshake initiation from %pISpfsc\n", 147 wg->dev->name, skb); 148 return; 149 } 150 wg_socket_set_peer_endpoint_from_skb(peer, skb); 151 net_dbg_ratelimited("%s: Receiving handshake initiation from peer %llu (%pISpfsc)\n", 152 wg->dev->name, peer->internal_id, 153 &peer->endpoint.addr); 154 wg_packet_send_handshake_response(peer); 155 break; 156 } 157 case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): { 158 struct message_handshake_response *message = 159 (struct message_handshake_response *)skb->data; 160 161 if (packet_needs_cookie) { 162 wg_packet_send_handshake_cookie(wg, skb, 163 message->sender_index); 164 return; 165 } 166 peer = wg_noise_handshake_consume_response(message, wg); 167 if (unlikely(!peer)) { 168 net_dbg_skb_ratelimited("%s: Invalid handshake response from %pISpfsc\n", 169 wg->dev->name, skb); 170 return; 171 } 172 wg_socket_set_peer_endpoint_from_skb(peer, skb); 173 net_dbg_ratelimited("%s: Receiving handshake response from peer %llu (%pISpfsc)\n", 174 wg->dev->name, peer->internal_id, 175 &peer->endpoint.addr); 176 if (wg_noise_handshake_begin_session(&peer->handshake, 177 &peer->keypairs)) { 178 wg_timers_session_derived(peer); 179 wg_timers_handshake_complete(peer); 180 /* Calling this function will either send any existing 181 * packets in the queue and not send a keepalive, which 182 * is the best case, Or, if there's nothing in the 183 * queue, it will send a keepalive, in order to give 184 * immediate confirmation of the session. 185 */ 186 wg_packet_send_keepalive(peer); 187 } 188 break; 189 } 190 } 191 192 if (unlikely(!peer)) { 193 WARN(1, "Somehow a wrong type of packet wound up in the handshake queue!\n"); 194 return; 195 } 196 197 local_bh_disable(); 198 update_rx_stats(peer, skb->len); 199 local_bh_enable(); 200 201 wg_timers_any_authenticated_packet_received(peer); 202 wg_timers_any_authenticated_packet_traversal(peer); 203 wg_peer_put(peer); 204 } 205 206 void wg_packet_handshake_receive_worker(struct work_struct *work) 207 { 208 struct crypt_queue *queue = container_of(work, struct multicore_worker, work)->ptr; 209 struct wg_device *wg = container_of(queue, struct wg_device, handshake_queue); 210 struct sk_buff *skb; 211 212 while ((skb = ptr_ring_consume_bh(&queue->ring)) != NULL) { 213 wg_receive_handshake_packet(wg, skb); 214 dev_kfree_skb(skb); 215 atomic_dec(&wg->handshake_queue_len); 216 cond_resched(); 217 } 218 } 219 220 static void keep_key_fresh(struct wg_peer *peer) 221 { 222 struct noise_keypair *keypair; 223 bool send; 224 225 if (peer->sent_lastminute_handshake) 226 return; 227 228 rcu_read_lock_bh(); 229 keypair = rcu_dereference_bh(peer->keypairs.current_keypair); 230 send = keypair && READ_ONCE(keypair->sending.is_valid) && 231 keypair->i_am_the_initiator && 232 wg_birthdate_has_expired(keypair->sending.birthdate, 233 REJECT_AFTER_TIME - KEEPALIVE_TIMEOUT - REKEY_TIMEOUT); 234 rcu_read_unlock_bh(); 235 236 if (unlikely(send)) { 237 peer->sent_lastminute_handshake = true; 238 wg_packet_send_queued_handshake_initiation(peer, false); 239 } 240 } 241 242 static bool decrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair) 243 { 244 struct scatterlist sg[MAX_SKB_FRAGS + 8]; 245 struct sk_buff *trailer; 246 unsigned int offset; 247 int num_frags; 248 249 if (unlikely(!keypair)) 250 return false; 251 252 if (unlikely(!READ_ONCE(keypair->receiving.is_valid) || 253 wg_birthdate_has_expired(keypair->receiving.birthdate, REJECT_AFTER_TIME) || 254 READ_ONCE(keypair->receiving_counter.counter) >= REJECT_AFTER_MESSAGES)) { 255 WRITE_ONCE(keypair->receiving.is_valid, false); 256 return false; 257 } 258 259 PACKET_CB(skb)->nonce = 260 le64_to_cpu(((struct message_data *)skb->data)->counter); 261 262 /* We ensure that the network header is part of the packet before we 263 * call skb_cow_data, so that there's no chance that data is removed 264 * from the skb, so that later we can extract the original endpoint. 265 */ 266 offset = -skb_network_offset(skb); 267 skb_push(skb, offset); 268 num_frags = skb_cow_data(skb, 0, &trailer); 269 offset += sizeof(struct message_data); 270 skb_pull(skb, offset); 271 if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg))) 272 return false; 273 274 sg_init_table(sg, num_frags); 275 if (skb_to_sgvec(skb, sg, 0, skb->len) <= 0) 276 return false; 277 278 if (!chacha20poly1305_decrypt_sg_inplace(sg, skb->len, NULL, 0, 279 PACKET_CB(skb)->nonce, 280 keypair->receiving.key)) 281 return false; 282 283 /* Another ugly situation of pushing and pulling the header so as to 284 * keep endpoint information intact. 285 */ 286 skb_push(skb, offset); 287 if (pskb_trim(skb, skb->len - noise_encrypted_len(0))) 288 return false; 289 skb_pull(skb, offset); 290 291 return true; 292 } 293 294 /* This is RFC6479, a replay detection bitmap algorithm that avoids bitshifts */ 295 static bool counter_validate(struct noise_replay_counter *counter, u64 their_counter) 296 { 297 unsigned long index, index_current, top, i; 298 bool ret = false; 299 300 spin_lock_bh(&counter->lock); 301 302 if (unlikely(counter->counter >= REJECT_AFTER_MESSAGES + 1 || 303 their_counter >= REJECT_AFTER_MESSAGES)) 304 goto out; 305 306 ++their_counter; 307 308 if (unlikely((COUNTER_WINDOW_SIZE + their_counter) < 309 counter->counter)) 310 goto out; 311 312 index = their_counter >> ilog2(BITS_PER_LONG); 313 314 if (likely(their_counter > counter->counter)) { 315 index_current = counter->counter >> ilog2(BITS_PER_LONG); 316 top = min_t(unsigned long, index - index_current, 317 COUNTER_BITS_TOTAL / BITS_PER_LONG); 318 for (i = 1; i <= top; ++i) 319 counter->backtrack[(i + index_current) & 320 ((COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1)] = 0; 321 WRITE_ONCE(counter->counter, their_counter); 322 } 323 324 index &= (COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1; 325 ret = !test_and_set_bit(their_counter & (BITS_PER_LONG - 1), 326 &counter->backtrack[index]); 327 328 out: 329 spin_unlock_bh(&counter->lock); 330 return ret; 331 } 332 333 #include "selftest/counter.c" 334 335 static void wg_packet_consume_data_done(struct wg_peer *peer, 336 struct sk_buff *skb, 337 struct endpoint *endpoint) 338 { 339 struct net_device *dev = peer->device->dev; 340 unsigned int len, len_before_trim; 341 struct wg_peer *routed_peer; 342 343 wg_socket_set_peer_endpoint(peer, endpoint); 344 345 if (unlikely(wg_noise_received_with_keypair(&peer->keypairs, 346 PACKET_CB(skb)->keypair))) { 347 wg_timers_handshake_complete(peer); 348 wg_packet_send_staged_packets(peer); 349 } 350 351 keep_key_fresh(peer); 352 353 wg_timers_any_authenticated_packet_received(peer); 354 wg_timers_any_authenticated_packet_traversal(peer); 355 356 /* A packet with length 0 is a keepalive packet */ 357 if (unlikely(!skb->len)) { 358 update_rx_stats(peer, message_data_len(0)); 359 net_dbg_ratelimited("%s: Receiving keepalive packet from peer %llu (%pISpfsc)\n", 360 dev->name, peer->internal_id, 361 &peer->endpoint.addr); 362 goto packet_processed; 363 } 364 365 wg_timers_data_received(peer); 366 367 if (unlikely(skb_network_header(skb) < skb->head)) 368 goto dishonest_packet_size; 369 if (unlikely(!(pskb_network_may_pull(skb, sizeof(struct iphdr)) && 370 (ip_hdr(skb)->version == 4 || 371 (ip_hdr(skb)->version == 6 && 372 pskb_network_may_pull(skb, sizeof(struct ipv6hdr))))))) 373 goto dishonest_packet_type; 374 375 skb->dev = dev; 376 /* We've already verified the Poly1305 auth tag, which means this packet 377 * was not modified in transit. We can therefore tell the networking 378 * stack that all checksums of every layer of encapsulation have already 379 * been checked "by the hardware" and therefore is unnecessary to check 380 * again in software. 381 */ 382 skb->ip_summed = CHECKSUM_UNNECESSARY; 383 skb->csum_level = ~0; /* All levels */ 384 skb->protocol = ip_tunnel_parse_protocol(skb); 385 if (skb->protocol == htons(ETH_P_IP)) { 386 len = ntohs(ip_hdr(skb)->tot_len); 387 if (unlikely(len < sizeof(struct iphdr))) 388 goto dishonest_packet_size; 389 INET_ECN_decapsulate(skb, PACKET_CB(skb)->ds, ip_hdr(skb)->tos); 390 } else if (skb->protocol == htons(ETH_P_IPV6)) { 391 len = ntohs(ipv6_hdr(skb)->payload_len) + 392 sizeof(struct ipv6hdr); 393 INET_ECN_decapsulate(skb, PACKET_CB(skb)->ds, ipv6_get_dsfield(ipv6_hdr(skb))); 394 } else { 395 goto dishonest_packet_type; 396 } 397 398 if (unlikely(len > skb->len)) 399 goto dishonest_packet_size; 400 len_before_trim = skb->len; 401 if (unlikely(pskb_trim(skb, len))) 402 goto packet_processed; 403 404 routed_peer = wg_allowedips_lookup_src(&peer->device->peer_allowedips, 405 skb); 406 wg_peer_put(routed_peer); /* We don't need the extra reference. */ 407 408 if (unlikely(routed_peer != peer)) 409 goto dishonest_packet_peer; 410 411 napi_gro_receive(&peer->napi, skb); 412 update_rx_stats(peer, message_data_len(len_before_trim)); 413 return; 414 415 dishonest_packet_peer: 416 net_dbg_skb_ratelimited("%s: Packet has unallowed src IP (%pISc) from peer %llu (%pISpfsc)\n", 417 dev->name, skb, peer->internal_id, 418 &peer->endpoint.addr); 419 DEV_STATS_INC(dev, rx_errors); 420 DEV_STATS_INC(dev, rx_frame_errors); 421 goto packet_processed; 422 dishonest_packet_type: 423 net_dbg_ratelimited("%s: Packet is neither ipv4 nor ipv6 from peer %llu (%pISpfsc)\n", 424 dev->name, peer->internal_id, &peer->endpoint.addr); 425 DEV_STATS_INC(dev, rx_errors); 426 DEV_STATS_INC(dev, rx_frame_errors); 427 goto packet_processed; 428 dishonest_packet_size: 429 net_dbg_ratelimited("%s: Packet has incorrect size from peer %llu (%pISpfsc)\n", 430 dev->name, peer->internal_id, &peer->endpoint.addr); 431 DEV_STATS_INC(dev, rx_errors); 432 DEV_STATS_INC(dev, rx_length_errors); 433 goto packet_processed; 434 packet_processed: 435 dev_kfree_skb(skb); 436 } 437 438 int wg_packet_rx_poll(struct napi_struct *napi, int budget) 439 { 440 struct wg_peer *peer = container_of(napi, struct wg_peer, napi); 441 struct noise_keypair *keypair; 442 struct endpoint endpoint; 443 enum packet_state state; 444 struct sk_buff *skb; 445 int work_done = 0; 446 bool free; 447 448 if (unlikely(budget <= 0)) 449 return 0; 450 451 while ((skb = wg_prev_queue_peek(&peer->rx_queue)) != NULL && 452 (state = atomic_read_acquire(&PACKET_CB(skb)->state)) != 453 PACKET_STATE_UNCRYPTED) { 454 wg_prev_queue_drop_peeked(&peer->rx_queue); 455 keypair = PACKET_CB(skb)->keypair; 456 free = true; 457 458 if (unlikely(state != PACKET_STATE_CRYPTED)) 459 goto next; 460 461 if (unlikely(!counter_validate(&keypair->receiving_counter, 462 PACKET_CB(skb)->nonce))) { 463 net_dbg_ratelimited("%s: Packet has invalid nonce %llu (max %llu)\n", 464 peer->device->dev->name, 465 PACKET_CB(skb)->nonce, 466 READ_ONCE(keypair->receiving_counter.counter)); 467 goto next; 468 } 469 470 if (unlikely(wg_socket_endpoint_from_skb(&endpoint, skb))) 471 goto next; 472 473 wg_reset_packet(skb, false); 474 wg_packet_consume_data_done(peer, skb, &endpoint); 475 free = false; 476 477 next: 478 wg_noise_keypair_put(keypair, false); 479 wg_peer_put(peer); 480 if (unlikely(free)) 481 dev_kfree_skb(skb); 482 483 if (++work_done >= budget) 484 break; 485 } 486 487 if (work_done < budget) 488 napi_complete_done(napi, work_done); 489 490 return work_done; 491 } 492 493 void wg_packet_decrypt_worker(struct work_struct *work) 494 { 495 struct crypt_queue *queue = container_of(work, struct multicore_worker, 496 work)->ptr; 497 struct sk_buff *skb; 498 499 while ((skb = ptr_ring_consume_bh(&queue->ring)) != NULL) { 500 enum packet_state state = 501 likely(decrypt_packet(skb, PACKET_CB(skb)->keypair)) ? 502 PACKET_STATE_CRYPTED : PACKET_STATE_DEAD; 503 wg_queue_enqueue_per_peer_rx(skb, state); 504 if (need_resched()) 505 cond_resched(); 506 } 507 } 508 509 static void wg_packet_consume_data(struct wg_device *wg, struct sk_buff *skb) 510 { 511 __le32 idx = ((struct message_data *)skb->data)->key_idx; 512 struct wg_peer *peer = NULL; 513 int ret; 514 515 rcu_read_lock_bh(); 516 PACKET_CB(skb)->keypair = 517 (struct noise_keypair *)wg_index_hashtable_lookup( 518 wg->index_hashtable, INDEX_HASHTABLE_KEYPAIR, idx, 519 &peer); 520 if (unlikely(!wg_noise_keypair_get(PACKET_CB(skb)->keypair))) 521 goto err_keypair; 522 523 if (unlikely(READ_ONCE(peer->is_dead))) 524 goto err; 525 526 ret = wg_queue_enqueue_per_device_and_peer(&wg->decrypt_queue, &peer->rx_queue, skb, 527 wg->packet_crypt_wq); 528 if (unlikely(ret == -EPIPE)) 529 wg_queue_enqueue_per_peer_rx(skb, PACKET_STATE_DEAD); 530 if (likely(!ret || ret == -EPIPE)) { 531 rcu_read_unlock_bh(); 532 return; 533 } 534 err: 535 wg_noise_keypair_put(PACKET_CB(skb)->keypair, false); 536 err_keypair: 537 rcu_read_unlock_bh(); 538 wg_peer_put(peer); 539 dev_kfree_skb(skb); 540 } 541 542 void wg_packet_receive(struct wg_device *wg, struct sk_buff *skb) 543 { 544 if (unlikely(prepare_skb_header(skb, wg) < 0)) 545 goto err; 546 switch (SKB_TYPE_LE32(skb)) { 547 case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): 548 case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): 549 case cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE): { 550 int cpu, ret = -EBUSY; 551 552 if (unlikely(!rng_is_initialized())) 553 goto drop; 554 if (atomic_read(&wg->handshake_queue_len) > MAX_QUEUED_INCOMING_HANDSHAKES / 2) { 555 if (spin_trylock_bh(&wg->handshake_queue.ring.producer_lock)) { 556 ret = __ptr_ring_produce(&wg->handshake_queue.ring, skb); 557 spin_unlock_bh(&wg->handshake_queue.ring.producer_lock); 558 } 559 } else 560 ret = ptr_ring_produce_bh(&wg->handshake_queue.ring, skb); 561 if (ret) { 562 drop: 563 net_dbg_skb_ratelimited("%s: Dropping handshake packet from %pISpfsc\n", 564 wg->dev->name, skb); 565 goto err; 566 } 567 atomic_inc(&wg->handshake_queue_len); 568 cpu = wg_cpumask_next_online(&wg->handshake_queue.last_cpu); 569 /* Queues up a call to packet_process_queued_handshake_packets(skb): */ 570 queue_work_on(cpu, wg->handshake_receive_wq, 571 &per_cpu_ptr(wg->handshake_queue.worker, cpu)->work); 572 break; 573 } 574 case cpu_to_le32(MESSAGE_DATA): 575 PACKET_CB(skb)->ds = ip_tunnel_get_dsfield(ip_hdr(skb), skb); 576 wg_packet_consume_data(wg, skb); 577 break; 578 default: 579 WARN(1, "Non-exhaustive parsing of packet header lead to unknown packet type!\n"); 580 goto err; 581 } 582 return; 583 584 err: 585 dev_kfree_skb(skb); 586 } 587