1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C)2002 USAGI/WIDE Project 4 * 5 * Authors 6 * 7 * Mitsuru KANDA @USAGI : IPv6 Support 8 * Kazunori MIYAZAWA @USAGI : 9 * Kunihiro Ishiguro <kunihiro@ipinfusion.com> 10 * 11 * This file is derived from net/ipv4/esp.c 12 */ 13 14 #define pr_fmt(fmt) "IPv6: " fmt 15 16 #include <crypto/aead.h> 17 #include <crypto/authenc.h> 18 #include <linux/err.h> 19 #include <linux/module.h> 20 #include <net/ip.h> 21 #include <net/xfrm.h> 22 #include <net/esp.h> 23 #include <linux/scatterlist.h> 24 #include <linux/kernel.h> 25 #include <linux/pfkeyv2.h> 26 #include <linux/random.h> 27 #include <linux/slab.h> 28 #include <linux/spinlock.h> 29 #include <net/ip6_checksum.h> 30 #include <net/ip6_route.h> 31 #include <net/icmp.h> 32 #include <net/ipv6.h> 33 #include <net/protocol.h> 34 #include <net/udp.h> 35 #include <linux/icmpv6.h> 36 #include <net/tcp.h> 37 #include <net/espintcp.h> 38 #include <net/inet6_hashtables.h> 39 #include <linux/skbuff_ref.h> 40 41 #include <linux/highmem.h> 42 43 struct esp_skb_cb { 44 struct xfrm_skb_cb xfrm; 45 void *tmp; 46 }; 47 48 struct esp_output_extra { 49 __be32 seqhi; 50 u32 esphoff; 51 }; 52 53 #define ESP_SKB_CB(__skb) ((struct esp_skb_cb *)&((__skb)->cb[0])) 54 55 /* 56 * Allocate an AEAD request structure with extra space for SG and IV. 57 * 58 * For alignment considerations the upper 32 bits of the sequence number are 59 * placed at the front, if present. Followed by the IV, the request and finally 60 * the SG list. 61 * 62 * TODO: Use spare space in skb for this where possible. 63 */ 64 static void *esp_alloc_tmp(struct crypto_aead *aead, int nfrags, int seqihlen) 65 { 66 unsigned int len; 67 68 len = seqihlen; 69 70 len += crypto_aead_ivsize(aead); 71 72 if (len) { 73 len += crypto_aead_alignmask(aead) & 74 ~(crypto_tfm_ctx_alignment() - 1); 75 len = ALIGN(len, crypto_tfm_ctx_alignment()); 76 } 77 78 len += sizeof(struct aead_request) + crypto_aead_reqsize(aead); 79 len = ALIGN(len, __alignof__(struct scatterlist)); 80 81 len += sizeof(struct scatterlist) * nfrags; 82 83 return kmalloc(len, GFP_ATOMIC); 84 } 85 86 static inline void *esp_tmp_extra(void *tmp) 87 { 88 return PTR_ALIGN(tmp, __alignof__(struct esp_output_extra)); 89 } 90 91 static inline u8 *esp_tmp_iv(struct crypto_aead *aead, void *tmp, int seqhilen) 92 { 93 return crypto_aead_ivsize(aead) ? 94 PTR_ALIGN((u8 *)tmp + seqhilen, 95 crypto_aead_alignmask(aead) + 1) : tmp + seqhilen; 96 } 97 98 static inline struct aead_request *esp_tmp_req(struct crypto_aead *aead, u8 *iv) 99 { 100 struct aead_request *req; 101 102 req = (void *)PTR_ALIGN(iv + crypto_aead_ivsize(aead), 103 crypto_tfm_ctx_alignment()); 104 aead_request_set_tfm(req, aead); 105 return req; 106 } 107 108 static inline struct scatterlist *esp_req_sg(struct crypto_aead *aead, 109 struct aead_request *req) 110 { 111 return (void *)ALIGN((unsigned long)(req + 1) + 112 crypto_aead_reqsize(aead), 113 __alignof__(struct scatterlist)); 114 } 115 116 static void esp_ssg_unref(struct xfrm_state *x, void *tmp, struct sk_buff *skb, bool already_unref) 117 { 118 struct crypto_aead *aead = x->data; 119 int extralen = 0; 120 u8 *iv; 121 struct aead_request *req; 122 struct scatterlist *sg; 123 124 if (x->props.flags & XFRM_STATE_ESN) 125 extralen += sizeof(struct esp_output_extra); 126 127 iv = esp_tmp_iv(aead, tmp, extralen); 128 req = esp_tmp_req(aead, iv); 129 130 /* Unref skb_frag_pages in the src scatterlist if necessary. 131 * Skip the first sg which comes from skb->data. 132 */ 133 if (already_unref || req->src != req->dst) { 134 struct scatterlist *src = already_unref ? esp_req_sg(aead, req) : req->src; 135 136 for (sg = sg_next(src); sg; sg = sg_next(sg)) 137 skb_page_unref(page_to_netmem(sg_page(sg)), 138 skb->pp_recycle); 139 } 140 } 141 142 #ifdef CONFIG_INET6_ESPINTCP 143 static struct sock *esp6_find_tcp_sk(struct xfrm_state *x) 144 { 145 struct xfrm_encap_tmpl *encap = x->encap; 146 struct net *net = xs_net(x); 147 __be16 sport, dport; 148 struct sock *sk; 149 150 spin_lock_bh(&x->lock); 151 sport = encap->encap_sport; 152 dport = encap->encap_dport; 153 spin_unlock_bh(&x->lock); 154 155 sk = __inet6_lookup_established(net, &x->id.daddr.in6, dport, 156 &x->props.saddr.in6, ntohs(sport), 0, 0); 157 if (!sk) 158 return ERR_PTR(-ENOENT); 159 160 if (!tcp_is_ulp_esp(sk)) { 161 sock_put(sk); 162 return ERR_PTR(-EINVAL); 163 } 164 165 return sk; 166 } 167 168 static int esp_output_tcp_finish(struct xfrm_state *x, struct sk_buff *skb) 169 { 170 struct sock *sk; 171 int err; 172 173 rcu_read_lock(); 174 175 sk = esp6_find_tcp_sk(x); 176 err = PTR_ERR_OR_ZERO(sk); 177 if (err) { 178 kfree_skb(skb); 179 goto out; 180 } 181 182 bh_lock_sock(sk); 183 if (sock_owned_by_user(sk)) 184 err = espintcp_queue_out(sk, skb); 185 else 186 err = espintcp_push_skb(sk, skb); 187 bh_unlock_sock(sk); 188 189 sock_put(sk); 190 191 out: 192 rcu_read_unlock(); 193 return err; 194 } 195 196 static int esp_output_tcp_encap_cb(struct net *net, struct sock *sk, 197 struct sk_buff *skb) 198 { 199 struct dst_entry *dst = skb_dst(skb); 200 struct xfrm_state *x = dst->xfrm; 201 202 return esp_output_tcp_finish(x, skb); 203 } 204 205 static int esp_output_tail_tcp(struct xfrm_state *x, struct sk_buff *skb) 206 { 207 int err; 208 209 local_bh_disable(); 210 err = xfrm_trans_queue_net(xs_net(x), skb, esp_output_tcp_encap_cb); 211 local_bh_enable(); 212 213 /* EINPROGRESS just happens to do the right thing. It 214 * actually means that the skb has been consumed and 215 * isn't coming back. 216 */ 217 return err ?: -EINPROGRESS; 218 } 219 #else 220 static int esp_output_tail_tcp(struct xfrm_state *x, struct sk_buff *skb) 221 { 222 WARN_ON(1); 223 return -EOPNOTSUPP; 224 } 225 #endif 226 227 static void esp_output_encap_csum(struct sk_buff *skb) 228 { 229 /* UDP encap with IPv6 requires a valid checksum */ 230 if (*skb_mac_header(skb) == IPPROTO_UDP) { 231 struct udphdr *uh = udp_hdr(skb); 232 struct ipv6hdr *ip6h = ipv6_hdr(skb); 233 int len = ntohs(uh->len); 234 unsigned int offset = skb_transport_offset(skb); 235 __wsum csum = skb_checksum(skb, offset, skb->len - offset, 0); 236 237 uh->check = csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr, 238 len, IPPROTO_UDP, csum); 239 if (uh->check == 0) 240 uh->check = CSUM_MANGLED_0; 241 } 242 } 243 244 static void esp_output_done(void *data, int err) 245 { 246 struct sk_buff *skb = data; 247 struct xfrm_offload *xo = xfrm_offload(skb); 248 void *tmp; 249 struct xfrm_state *x; 250 251 if (xo && (xo->flags & XFRM_DEV_RESUME)) { 252 struct sec_path *sp = skb_sec_path(skb); 253 254 x = sp->xvec[sp->len - 1]; 255 } else { 256 x = skb_dst(skb)->xfrm; 257 } 258 259 tmp = ESP_SKB_CB(skb)->tmp; 260 esp_ssg_unref(x, tmp, skb, false); 261 kfree(tmp); 262 263 esp_output_encap_csum(skb); 264 265 if (xo && (xo->flags & XFRM_DEV_RESUME)) { 266 if (err) { 267 XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR); 268 kfree_skb(skb); 269 return; 270 } 271 272 skb_push(skb, skb->data - skb_mac_header(skb)); 273 secpath_reset(skb); 274 xfrm_dev_resume(skb); 275 } else { 276 if (!err && 277 x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP) { 278 err = esp_output_tail_tcp(x, skb); 279 if (err != -EINPROGRESS) 280 kfree_skb(skb); 281 } else { 282 xfrm_output_resume(skb_to_full_sk(skb), skb, err); 283 } 284 } 285 } 286 287 /* Move ESP header back into place. */ 288 static void esp_restore_header(struct sk_buff *skb, unsigned int offset) 289 { 290 struct ip_esp_hdr *esph = (void *)(skb->data + offset); 291 void *tmp = ESP_SKB_CB(skb)->tmp; 292 __be32 *seqhi = esp_tmp_extra(tmp); 293 294 esph->seq_no = esph->spi; 295 esph->spi = *seqhi; 296 } 297 298 static void esp_output_restore_header(struct sk_buff *skb) 299 { 300 void *tmp = ESP_SKB_CB(skb)->tmp; 301 struct esp_output_extra *extra = esp_tmp_extra(tmp); 302 303 esp_restore_header(skb, skb_transport_offset(skb) + extra->esphoff - 304 sizeof(__be32)); 305 } 306 307 static struct ip_esp_hdr *esp_output_set_esn(struct sk_buff *skb, 308 struct xfrm_state *x, 309 struct ip_esp_hdr *esph, 310 struct esp_output_extra *extra) 311 { 312 /* For ESN we move the header forward by 4 bytes to 313 * accommodate the high bits. We will move it back after 314 * encryption. 315 */ 316 if ((x->props.flags & XFRM_STATE_ESN)) { 317 __u32 seqhi; 318 struct xfrm_offload *xo = xfrm_offload(skb); 319 320 if (xo) 321 seqhi = xo->seq.hi; 322 else 323 seqhi = XFRM_SKB_CB(skb)->seq.output.hi; 324 325 extra->esphoff = (unsigned char *)esph - 326 skb_transport_header(skb); 327 esph = (struct ip_esp_hdr *)((unsigned char *)esph - 4); 328 extra->seqhi = esph->spi; 329 esph->seq_no = htonl(seqhi); 330 } 331 332 esph->spi = x->id.spi; 333 334 return esph; 335 } 336 337 static void esp_output_done_esn(void *data, int err) 338 { 339 struct sk_buff *skb = data; 340 341 esp_output_restore_header(skb); 342 esp_output_done(data, err); 343 } 344 345 static struct ip_esp_hdr *esp6_output_udp_encap(struct sk_buff *skb, 346 int encap_type, 347 struct esp_info *esp, 348 __be16 sport, 349 __be16 dport) 350 { 351 struct udphdr *uh; 352 unsigned int len; 353 354 len = skb->len + esp->tailen - skb_transport_offset(skb); 355 if (len > U16_MAX) 356 return ERR_PTR(-EMSGSIZE); 357 358 uh = (struct udphdr *)esp->esph; 359 uh->source = sport; 360 uh->dest = dport; 361 uh->len = htons(len); 362 uh->check = 0; 363 364 *skb_mac_header(skb) = IPPROTO_UDP; 365 366 return (struct ip_esp_hdr *)(uh + 1); 367 } 368 369 #ifdef CONFIG_INET6_ESPINTCP 370 static struct ip_esp_hdr *esp6_output_tcp_encap(struct xfrm_state *x, 371 struct sk_buff *skb, 372 struct esp_info *esp) 373 { 374 __be16 *lenp = (void *)esp->esph; 375 struct ip_esp_hdr *esph; 376 unsigned int len; 377 struct sock *sk; 378 379 len = skb->len + esp->tailen - skb_transport_offset(skb); 380 if (len > IP_MAX_MTU) 381 return ERR_PTR(-EMSGSIZE); 382 383 rcu_read_lock(); 384 sk = esp6_find_tcp_sk(x); 385 rcu_read_unlock(); 386 387 if (IS_ERR(sk)) 388 return ERR_CAST(sk); 389 390 sock_put(sk); 391 392 *lenp = htons(len); 393 esph = (struct ip_esp_hdr *)(lenp + 1); 394 395 return esph; 396 } 397 #else 398 static struct ip_esp_hdr *esp6_output_tcp_encap(struct xfrm_state *x, 399 struct sk_buff *skb, 400 struct esp_info *esp) 401 { 402 return ERR_PTR(-EOPNOTSUPP); 403 } 404 #endif 405 406 static int esp6_output_encap(struct xfrm_state *x, struct sk_buff *skb, 407 struct esp_info *esp) 408 { 409 struct xfrm_encap_tmpl *encap = x->encap; 410 struct ip_esp_hdr *esph; 411 __be16 sport, dport; 412 int encap_type; 413 414 spin_lock_bh(&x->lock); 415 sport = encap->encap_sport; 416 dport = encap->encap_dport; 417 encap_type = encap->encap_type; 418 spin_unlock_bh(&x->lock); 419 420 switch (encap_type) { 421 default: 422 case UDP_ENCAP_ESPINUDP: 423 esph = esp6_output_udp_encap(skb, encap_type, esp, sport, dport); 424 break; 425 case TCP_ENCAP_ESPINTCP: 426 esph = esp6_output_tcp_encap(x, skb, esp); 427 break; 428 } 429 430 if (IS_ERR(esph)) 431 return PTR_ERR(esph); 432 433 esp->esph = esph; 434 435 return 0; 436 } 437 438 int esp6_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *esp) 439 { 440 u8 *tail; 441 int nfrags; 442 int esph_offset; 443 struct page *page; 444 struct sk_buff *trailer; 445 int tailen = esp->tailen; 446 447 if (x->encap) { 448 int err = esp6_output_encap(x, skb, esp); 449 450 if (err < 0) 451 return err; 452 } 453 454 if (ALIGN(skb->data_len + tailen, L1_CACHE_BYTES) > 455 PAGE_SIZE) 456 goto cow; 457 458 if (!skb_cloned(skb)) { 459 if (tailen <= skb_tailroom(skb)) { 460 nfrags = 1; 461 trailer = skb; 462 tail = skb_tail_pointer(trailer); 463 464 goto skip_cow; 465 } else if ((skb_shinfo(skb)->nr_frags < MAX_SKB_FRAGS) 466 && !skb_has_frag_list(skb)) { 467 int allocsize; 468 struct sock *sk = skb->sk; 469 struct page_frag *pfrag = &x->xfrag; 470 471 esp->inplace = false; 472 473 allocsize = ALIGN(tailen, L1_CACHE_BYTES); 474 475 spin_lock_bh(&x->lock); 476 477 if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) { 478 spin_unlock_bh(&x->lock); 479 goto cow; 480 } 481 482 page = pfrag->page; 483 get_page(page); 484 485 tail = page_address(page) + pfrag->offset; 486 487 esp_output_fill_trailer(tail, esp->tfclen, esp->plen, esp->proto); 488 489 nfrags = skb_shinfo(skb)->nr_frags; 490 491 __skb_fill_page_desc(skb, nfrags, page, pfrag->offset, 492 tailen); 493 skb_shinfo(skb)->nr_frags = ++nfrags; 494 495 pfrag->offset = pfrag->offset + allocsize; 496 497 spin_unlock_bh(&x->lock); 498 499 nfrags++; 500 501 skb->len += tailen; 502 skb->data_len += tailen; 503 skb->truesize += tailen; 504 if (sk && sk_fullsock(sk)) 505 refcount_add(tailen, &sk->sk_wmem_alloc); 506 507 goto out; 508 } 509 } 510 511 cow: 512 esph_offset = (unsigned char *)esp->esph - skb_transport_header(skb); 513 514 nfrags = skb_cow_data(skb, tailen, &trailer); 515 if (nfrags < 0) 516 goto out; 517 tail = skb_tail_pointer(trailer); 518 esp->esph = (struct ip_esp_hdr *)(skb_transport_header(skb) + esph_offset); 519 520 skip_cow: 521 esp_output_fill_trailer(tail, esp->tfclen, esp->plen, esp->proto); 522 pskb_put(skb, trailer, tailen); 523 524 out: 525 return nfrags; 526 } 527 EXPORT_SYMBOL_GPL(esp6_output_head); 528 529 int esp6_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *esp) 530 { 531 u8 *iv; 532 int alen; 533 void *tmp; 534 int ivlen; 535 int assoclen; 536 int extralen; 537 struct page *page; 538 struct ip_esp_hdr *esph; 539 struct aead_request *req; 540 struct crypto_aead *aead; 541 struct scatterlist *sg, *dsg; 542 struct esp_output_extra *extra; 543 int err = -ENOMEM; 544 545 assoclen = sizeof(struct ip_esp_hdr); 546 extralen = 0; 547 548 if (x->props.flags & XFRM_STATE_ESN) { 549 extralen += sizeof(*extra); 550 assoclen += sizeof(__be32); 551 } 552 553 aead = x->data; 554 alen = crypto_aead_authsize(aead); 555 ivlen = crypto_aead_ivsize(aead); 556 557 tmp = esp_alloc_tmp(aead, esp->nfrags + 2, extralen); 558 if (!tmp) 559 goto error; 560 561 extra = esp_tmp_extra(tmp); 562 iv = esp_tmp_iv(aead, tmp, extralen); 563 req = esp_tmp_req(aead, iv); 564 sg = esp_req_sg(aead, req); 565 566 if (esp->inplace) 567 dsg = sg; 568 else 569 dsg = &sg[esp->nfrags]; 570 571 esph = esp_output_set_esn(skb, x, esp->esph, extra); 572 esp->esph = esph; 573 574 sg_init_table(sg, esp->nfrags); 575 err = skb_to_sgvec(skb, sg, 576 (unsigned char *)esph - skb->data, 577 assoclen + ivlen + esp->clen + alen); 578 if (unlikely(err < 0)) 579 goto error_free; 580 581 if (!esp->inplace) { 582 int allocsize; 583 struct page_frag *pfrag = &x->xfrag; 584 585 allocsize = ALIGN(skb->data_len, L1_CACHE_BYTES); 586 587 spin_lock_bh(&x->lock); 588 if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) { 589 spin_unlock_bh(&x->lock); 590 goto error_free; 591 } 592 593 skb_shinfo(skb)->nr_frags = 1; 594 595 page = pfrag->page; 596 get_page(page); 597 /* replace page frags in skb with new page */ 598 __skb_fill_page_desc(skb, 0, page, pfrag->offset, skb->data_len); 599 pfrag->offset = pfrag->offset + allocsize; 600 spin_unlock_bh(&x->lock); 601 602 sg_init_table(dsg, skb_shinfo(skb)->nr_frags + 1); 603 err = skb_to_sgvec(skb, dsg, 604 (unsigned char *)esph - skb->data, 605 assoclen + ivlen + esp->clen + alen); 606 if (unlikely(err < 0)) { 607 esp_ssg_unref(x, tmp, skb, true); 608 goto error_free; 609 } 610 } 611 612 if ((x->props.flags & XFRM_STATE_ESN)) 613 aead_request_set_callback(req, 0, esp_output_done_esn, skb); 614 else 615 aead_request_set_callback(req, 0, esp_output_done, skb); 616 617 aead_request_set_crypt(req, sg, dsg, ivlen + esp->clen, iv); 618 aead_request_set_ad(req, assoclen); 619 620 memset(iv, 0, ivlen); 621 memcpy(iv + ivlen - min(ivlen, 8), (u8 *)&esp->seqno + 8 - min(ivlen, 8), 622 min(ivlen, 8)); 623 624 ESP_SKB_CB(skb)->tmp = tmp; 625 err = crypto_aead_encrypt(req); 626 627 switch (err) { 628 case -EINPROGRESS: 629 goto error; 630 631 case -ENOSPC: 632 err = NET_XMIT_DROP; 633 break; 634 635 case 0: 636 if ((x->props.flags & XFRM_STATE_ESN)) 637 esp_output_restore_header(skb); 638 esp_output_encap_csum(skb); 639 } 640 641 if (sg != dsg) 642 esp_ssg_unref(x, tmp, skb, false); 643 644 if (!err && x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP) 645 err = esp_output_tail_tcp(x, skb); 646 647 error_free: 648 kfree(tmp); 649 error: 650 return err; 651 } 652 EXPORT_SYMBOL_GPL(esp6_output_tail); 653 654 static int esp6_output(struct xfrm_state *x, struct sk_buff *skb) 655 { 656 int alen; 657 int blksize; 658 struct ip_esp_hdr *esph; 659 struct crypto_aead *aead; 660 struct esp_info esp; 661 662 esp.inplace = true; 663 664 esp.proto = *skb_mac_header(skb); 665 *skb_mac_header(skb) = IPPROTO_ESP; 666 667 /* skb is pure payload to encrypt */ 668 669 aead = x->data; 670 alen = crypto_aead_authsize(aead); 671 672 esp.tfclen = 0; 673 if (x->tfcpad) { 674 struct xfrm_dst *dst = (struct xfrm_dst *)skb_dst(skb); 675 u32 padto; 676 677 padto = min(x->tfcpad, xfrm_state_mtu(x, dst->child_mtu_cached)); 678 if (skb->len < padto) 679 esp.tfclen = padto - skb->len; 680 } 681 blksize = ALIGN(crypto_aead_blocksize(aead), 4); 682 esp.clen = ALIGN(skb->len + 2 + esp.tfclen, blksize); 683 esp.plen = esp.clen - skb->len - esp.tfclen; 684 esp.tailen = esp.tfclen + esp.plen + alen; 685 686 esp.esph = ip_esp_hdr(skb); 687 688 esp.nfrags = esp6_output_head(x, skb, &esp); 689 if (esp.nfrags < 0) 690 return esp.nfrags; 691 692 esph = esp.esph; 693 esph->spi = x->id.spi; 694 695 esph->seq_no = htonl(XFRM_SKB_CB(skb)->seq.output.low); 696 esp.seqno = cpu_to_be64(XFRM_SKB_CB(skb)->seq.output.low + 697 ((u64)XFRM_SKB_CB(skb)->seq.output.hi << 32)); 698 699 skb_push(skb, -skb_network_offset(skb)); 700 701 return esp6_output_tail(x, skb, &esp); 702 } 703 704 static inline int esp_remove_trailer(struct sk_buff *skb) 705 { 706 struct xfrm_state *x = xfrm_input_state(skb); 707 struct crypto_aead *aead = x->data; 708 int alen, hlen, elen; 709 int padlen, trimlen; 710 __wsum csumdiff; 711 u8 nexthdr[2]; 712 int ret; 713 714 alen = crypto_aead_authsize(aead); 715 hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead); 716 elen = skb->len - hlen; 717 718 ret = skb_copy_bits(skb, skb->len - alen - 2, nexthdr, 2); 719 BUG_ON(ret); 720 721 ret = -EINVAL; 722 padlen = nexthdr[0]; 723 if (padlen + 2 + alen >= elen) { 724 net_dbg_ratelimited("ipsec esp packet is garbage padlen=%d, elen=%d\n", 725 padlen + 2, elen - alen); 726 goto out; 727 } 728 729 trimlen = alen + padlen + 2; 730 if (skb->ip_summed == CHECKSUM_COMPLETE) { 731 csumdiff = skb_checksum(skb, skb->len - trimlen, trimlen, 0); 732 skb->csum = csum_block_sub(skb->csum, csumdiff, 733 skb->len - trimlen); 734 } 735 ret = pskb_trim(skb, skb->len - trimlen); 736 if (unlikely(ret)) 737 return ret; 738 739 ret = nexthdr[1]; 740 741 out: 742 return ret; 743 } 744 745 int esp6_input_done2(struct sk_buff *skb, int err) 746 { 747 struct xfrm_state *x = xfrm_input_state(skb); 748 struct xfrm_offload *xo = xfrm_offload(skb); 749 struct crypto_aead *aead = x->data; 750 int hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead); 751 int hdr_len = skb_network_header_len(skb); 752 753 if (!xo || !(xo->flags & CRYPTO_DONE)) 754 kfree(ESP_SKB_CB(skb)->tmp); 755 756 if (unlikely(err)) 757 goto out; 758 759 err = esp_remove_trailer(skb); 760 if (unlikely(err < 0)) 761 goto out; 762 763 if (x->encap) { 764 const struct ipv6hdr *ip6h = ipv6_hdr(skb); 765 int offset = skb_network_offset(skb) + sizeof(*ip6h); 766 struct xfrm_encap_tmpl *encap = x->encap; 767 u8 nexthdr = ip6h->nexthdr; 768 __be16 frag_off, source; 769 struct udphdr *uh; 770 struct tcphdr *th; 771 772 offset = ipv6_skip_exthdr(skb, offset, &nexthdr, &frag_off); 773 if (offset == -1) { 774 err = -EINVAL; 775 goto out; 776 } 777 778 uh = (void *)(skb->data + offset); 779 th = (void *)(skb->data + offset); 780 hdr_len += offset; 781 782 switch (x->encap->encap_type) { 783 case TCP_ENCAP_ESPINTCP: 784 source = th->source; 785 break; 786 case UDP_ENCAP_ESPINUDP: 787 source = uh->source; 788 break; 789 default: 790 WARN_ON_ONCE(1); 791 err = -EINVAL; 792 goto out; 793 } 794 795 /* 796 * 1) if the NAT-T peer's IP or port changed then 797 * advertise the change to the keying daemon. 798 * This is an inbound SA, so just compare 799 * SRC ports. 800 */ 801 if (!ipv6_addr_equal(&ip6h->saddr, &x->props.saddr.in6) || 802 source != encap->encap_sport) { 803 xfrm_address_t ipaddr; 804 805 memcpy(&ipaddr.a6, &ip6h->saddr.s6_addr, sizeof(ipaddr.a6)); 806 km_new_mapping(x, &ipaddr, source); 807 808 /* XXX: perhaps add an extra 809 * policy check here, to see 810 * if we should allow or 811 * reject a packet from a 812 * different source 813 * address/port. 814 */ 815 } 816 817 /* 818 * 2) ignore UDP/TCP checksums in case 819 * of NAT-T in Transport Mode, or 820 * perform other post-processing fixes 821 * as per draft-ietf-ipsec-udp-encaps-06, 822 * section 3.1.2 823 */ 824 if (x->props.mode == XFRM_MODE_TRANSPORT) 825 skb->ip_summed = CHECKSUM_UNNECESSARY; 826 } 827 828 skb_postpull_rcsum(skb, skb_network_header(skb), 829 skb_network_header_len(skb)); 830 skb_pull_rcsum(skb, hlen); 831 if (x->props.mode == XFRM_MODE_TUNNEL || 832 x->props.mode == XFRM_MODE_IPTFS) 833 skb_reset_transport_header(skb); 834 else 835 skb_set_transport_header(skb, -hdr_len); 836 837 /* RFC4303: Drop dummy packets without any error */ 838 if (err == IPPROTO_NONE) 839 err = -EINVAL; 840 841 out: 842 return err; 843 } 844 EXPORT_SYMBOL_GPL(esp6_input_done2); 845 846 static void esp_input_done(void *data, int err) 847 { 848 struct sk_buff *skb = data; 849 850 xfrm_input_resume(skb, esp6_input_done2(skb, err)); 851 } 852 853 static void esp_input_restore_header(struct sk_buff *skb) 854 { 855 esp_restore_header(skb, 0); 856 __skb_pull(skb, 4); 857 } 858 859 static void esp_input_set_header(struct sk_buff *skb, __be32 *seqhi) 860 { 861 struct xfrm_state *x = xfrm_input_state(skb); 862 863 /* For ESN we move the header forward by 4 bytes to 864 * accommodate the high bits. We will move it back after 865 * decryption. 866 */ 867 if ((x->props.flags & XFRM_STATE_ESN)) { 868 struct ip_esp_hdr *esph = skb_push(skb, 4); 869 870 *seqhi = esph->spi; 871 esph->spi = esph->seq_no; 872 esph->seq_no = XFRM_SKB_CB(skb)->seq.input.hi; 873 } 874 } 875 876 static void esp_input_done_esn(void *data, int err) 877 { 878 struct sk_buff *skb = data; 879 880 esp_input_restore_header(skb); 881 esp_input_done(data, err); 882 } 883 884 static int esp6_input(struct xfrm_state *x, struct sk_buff *skb) 885 { 886 struct crypto_aead *aead = x->data; 887 struct aead_request *req; 888 struct sk_buff *trailer; 889 int ivlen = crypto_aead_ivsize(aead); 890 int elen = skb->len - sizeof(struct ip_esp_hdr) - ivlen; 891 int nfrags; 892 int assoclen; 893 int seqhilen; 894 int ret = 0; 895 void *tmp; 896 __be32 *seqhi; 897 u8 *iv; 898 struct scatterlist *sg; 899 900 if (!pskb_may_pull(skb, sizeof(struct ip_esp_hdr) + ivlen)) { 901 ret = -EINVAL; 902 goto out; 903 } 904 905 if (elen <= 0) { 906 ret = -EINVAL; 907 goto out; 908 } 909 910 assoclen = sizeof(struct ip_esp_hdr); 911 seqhilen = 0; 912 913 if (x->props.flags & XFRM_STATE_ESN) { 914 seqhilen += sizeof(__be32); 915 assoclen += seqhilen; 916 } 917 918 if (!skb_cloned(skb)) { 919 if (!skb_is_nonlinear(skb)) { 920 nfrags = 1; 921 922 goto skip_cow; 923 } else if (!skb_has_frag_list(skb) && 924 !skb_has_shared_frag(skb)) { 925 nfrags = skb_shinfo(skb)->nr_frags; 926 nfrags++; 927 928 goto skip_cow; 929 } 930 } 931 932 nfrags = skb_cow_data(skb, 0, &trailer); 933 if (nfrags < 0) { 934 ret = -EINVAL; 935 goto out; 936 } 937 938 skip_cow: 939 ret = -ENOMEM; 940 tmp = esp_alloc_tmp(aead, nfrags, seqhilen); 941 if (!tmp) 942 goto out; 943 944 ESP_SKB_CB(skb)->tmp = tmp; 945 seqhi = esp_tmp_extra(tmp); 946 iv = esp_tmp_iv(aead, tmp, seqhilen); 947 req = esp_tmp_req(aead, iv); 948 sg = esp_req_sg(aead, req); 949 950 esp_input_set_header(skb, seqhi); 951 952 sg_init_table(sg, nfrags); 953 ret = skb_to_sgvec(skb, sg, 0, skb->len); 954 if (unlikely(ret < 0)) { 955 kfree(tmp); 956 goto out; 957 } 958 959 skb->ip_summed = CHECKSUM_NONE; 960 961 if ((x->props.flags & XFRM_STATE_ESN)) 962 aead_request_set_callback(req, 0, esp_input_done_esn, skb); 963 else 964 aead_request_set_callback(req, 0, esp_input_done, skb); 965 966 aead_request_set_crypt(req, sg, sg, elen + ivlen, iv); 967 aead_request_set_ad(req, assoclen); 968 969 ret = crypto_aead_decrypt(req); 970 if (ret == -EINPROGRESS) 971 goto out; 972 973 if ((x->props.flags & XFRM_STATE_ESN)) 974 esp_input_restore_header(skb); 975 976 ret = esp6_input_done2(skb, ret); 977 978 out: 979 return ret; 980 } 981 982 static int esp6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, 983 u8 type, u8 code, int offset, __be32 info) 984 { 985 struct net *net = dev_net(skb->dev); 986 const struct ipv6hdr *iph = (const struct ipv6hdr *)skb->data; 987 struct ip_esp_hdr *esph = (struct ip_esp_hdr *)(skb->data + offset); 988 struct xfrm_state *x; 989 990 if (type != ICMPV6_PKT_TOOBIG && 991 type != NDISC_REDIRECT) 992 return 0; 993 994 x = xfrm_state_lookup(net, skb->mark, (const xfrm_address_t *)&iph->daddr, 995 esph->spi, IPPROTO_ESP, AF_INET6); 996 if (!x) 997 return 0; 998 999 if (type == NDISC_REDIRECT) 1000 ip6_redirect(skb, net, skb->dev->ifindex, 0, 1001 sock_net_uid(net, NULL)); 1002 else 1003 ip6_update_pmtu(skb, net, info, 0, 0, sock_net_uid(net, NULL)); 1004 xfrm_state_put(x); 1005 1006 return 0; 1007 } 1008 1009 static void esp6_destroy(struct xfrm_state *x) 1010 { 1011 struct crypto_aead *aead = x->data; 1012 1013 if (!aead) 1014 return; 1015 1016 crypto_free_aead(aead); 1017 } 1018 1019 static int esp_init_aead(struct xfrm_state *x, struct netlink_ext_ack *extack) 1020 { 1021 char aead_name[CRYPTO_MAX_ALG_NAME]; 1022 struct crypto_aead *aead; 1023 int err; 1024 1025 if (snprintf(aead_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", 1026 x->geniv, x->aead->alg_name) >= CRYPTO_MAX_ALG_NAME) { 1027 NL_SET_ERR_MSG(extack, "Algorithm name is too long"); 1028 return -ENAMETOOLONG; 1029 } 1030 1031 aead = crypto_alloc_aead(aead_name, 0, 0); 1032 err = PTR_ERR(aead); 1033 if (IS_ERR(aead)) 1034 goto error; 1035 1036 x->data = aead; 1037 1038 err = crypto_aead_setkey(aead, x->aead->alg_key, 1039 (x->aead->alg_key_len + 7) / 8); 1040 if (err) 1041 goto error; 1042 1043 err = crypto_aead_setauthsize(aead, x->aead->alg_icv_len / 8); 1044 if (err) 1045 goto error; 1046 1047 return 0; 1048 1049 error: 1050 NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations"); 1051 return err; 1052 } 1053 1054 static int esp_init_authenc(struct xfrm_state *x, 1055 struct netlink_ext_ack *extack) 1056 { 1057 struct crypto_aead *aead; 1058 struct crypto_authenc_key_param *param; 1059 struct rtattr *rta; 1060 char *key; 1061 char *p; 1062 char authenc_name[CRYPTO_MAX_ALG_NAME]; 1063 unsigned int keylen; 1064 int err; 1065 1066 err = -ENAMETOOLONG; 1067 1068 if ((x->props.flags & XFRM_STATE_ESN)) { 1069 if (snprintf(authenc_name, CRYPTO_MAX_ALG_NAME, 1070 "%s%sauthencesn(%s,%s)%s", 1071 x->geniv ?: "", x->geniv ? "(" : "", 1072 x->aalg ? x->aalg->alg_name : "digest_null", 1073 x->ealg->alg_name, 1074 x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME) { 1075 NL_SET_ERR_MSG(extack, "Algorithm name is too long"); 1076 goto error; 1077 } 1078 } else { 1079 if (snprintf(authenc_name, CRYPTO_MAX_ALG_NAME, 1080 "%s%sauthenc(%s,%s)%s", 1081 x->geniv ?: "", x->geniv ? "(" : "", 1082 x->aalg ? x->aalg->alg_name : "digest_null", 1083 x->ealg->alg_name, 1084 x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME) { 1085 NL_SET_ERR_MSG(extack, "Algorithm name is too long"); 1086 goto error; 1087 } 1088 } 1089 1090 aead = crypto_alloc_aead(authenc_name, 0, 0); 1091 err = PTR_ERR(aead); 1092 if (IS_ERR(aead)) { 1093 NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations"); 1094 goto error; 1095 } 1096 1097 x->data = aead; 1098 1099 keylen = (x->aalg ? (x->aalg->alg_key_len + 7) / 8 : 0) + 1100 (x->ealg->alg_key_len + 7) / 8 + RTA_SPACE(sizeof(*param)); 1101 err = -ENOMEM; 1102 key = kmalloc(keylen, GFP_KERNEL); 1103 if (!key) 1104 goto error; 1105 1106 p = key; 1107 rta = (void *)p; 1108 rta->rta_type = CRYPTO_AUTHENC_KEYA_PARAM; 1109 rta->rta_len = RTA_LENGTH(sizeof(*param)); 1110 param = RTA_DATA(rta); 1111 p += RTA_SPACE(sizeof(*param)); 1112 1113 if (x->aalg) { 1114 struct xfrm_algo_desc *aalg_desc; 1115 1116 memcpy(p, x->aalg->alg_key, (x->aalg->alg_key_len + 7) / 8); 1117 p += (x->aalg->alg_key_len + 7) / 8; 1118 1119 aalg_desc = xfrm_aalg_get_byname(x->aalg->alg_name, 0); 1120 BUG_ON(!aalg_desc); 1121 1122 err = -EINVAL; 1123 if (aalg_desc->uinfo.auth.icv_fullbits / 8 != 1124 crypto_aead_authsize(aead)) { 1125 NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations"); 1126 goto free_key; 1127 } 1128 1129 err = crypto_aead_setauthsize( 1130 aead, x->aalg->alg_trunc_len / 8); 1131 if (err) { 1132 NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations"); 1133 goto free_key; 1134 } 1135 } 1136 1137 param->enckeylen = cpu_to_be32((x->ealg->alg_key_len + 7) / 8); 1138 memcpy(p, x->ealg->alg_key, (x->ealg->alg_key_len + 7) / 8); 1139 1140 err = crypto_aead_setkey(aead, key, keylen); 1141 1142 free_key: 1143 kfree(key); 1144 1145 error: 1146 return err; 1147 } 1148 1149 static int esp6_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack) 1150 { 1151 struct crypto_aead *aead; 1152 u32 align; 1153 int err; 1154 1155 x->data = NULL; 1156 1157 if (x->aead) { 1158 err = esp_init_aead(x, extack); 1159 } else if (x->ealg) { 1160 err = esp_init_authenc(x, extack); 1161 } else { 1162 NL_SET_ERR_MSG(extack, "ESP: AEAD or CRYPT must be provided"); 1163 err = -EINVAL; 1164 } 1165 1166 if (err) 1167 goto error; 1168 1169 aead = x->data; 1170 1171 x->props.header_len = sizeof(struct ip_esp_hdr) + 1172 crypto_aead_ivsize(aead); 1173 switch (x->props.mode) { 1174 case XFRM_MODE_BEET: 1175 if (x->sel.family != AF_INET6) 1176 x->props.header_len += IPV4_BEET_PHMAXLEN + 1177 (sizeof(struct ipv6hdr) - sizeof(struct iphdr)); 1178 break; 1179 default: 1180 case XFRM_MODE_TRANSPORT: 1181 break; 1182 case XFRM_MODE_TUNNEL: 1183 x->props.header_len += sizeof(struct ipv6hdr); 1184 break; 1185 } 1186 1187 if (x->encap) { 1188 struct xfrm_encap_tmpl *encap = x->encap; 1189 1190 switch (encap->encap_type) { 1191 default: 1192 NL_SET_ERR_MSG(extack, "Unsupported encapsulation type for ESP"); 1193 err = -EINVAL; 1194 goto error; 1195 case UDP_ENCAP_ESPINUDP: 1196 x->props.header_len += sizeof(struct udphdr); 1197 break; 1198 #ifdef CONFIG_INET6_ESPINTCP 1199 case TCP_ENCAP_ESPINTCP: 1200 /* only the length field, TCP encap is done by 1201 * the socket 1202 */ 1203 x->props.header_len += 2; 1204 break; 1205 #endif 1206 } 1207 } 1208 1209 align = ALIGN(crypto_aead_blocksize(aead), 4); 1210 x->props.trailer_len = align + 1 + crypto_aead_authsize(aead); 1211 1212 error: 1213 return err; 1214 } 1215 1216 static int esp6_rcv_cb(struct sk_buff *skb, int err) 1217 { 1218 return 0; 1219 } 1220 1221 static const struct xfrm_type esp6_type = { 1222 .owner = THIS_MODULE, 1223 .proto = IPPROTO_ESP, 1224 .flags = XFRM_TYPE_REPLAY_PROT, 1225 .init_state = esp6_init_state, 1226 .destructor = esp6_destroy, 1227 .input = esp6_input, 1228 .output = esp6_output, 1229 }; 1230 1231 static struct xfrm6_protocol esp6_protocol = { 1232 .handler = xfrm6_rcv, 1233 .input_handler = xfrm_input, 1234 .cb_handler = esp6_rcv_cb, 1235 .err_handler = esp6_err, 1236 .priority = 0, 1237 }; 1238 1239 static int __init esp6_init(void) 1240 { 1241 if (xfrm_register_type(&esp6_type, AF_INET6) < 0) { 1242 pr_info("%s: can't add xfrm type\n", __func__); 1243 return -EAGAIN; 1244 } 1245 if (xfrm6_protocol_register(&esp6_protocol, IPPROTO_ESP) < 0) { 1246 pr_info("%s: can't add protocol\n", __func__); 1247 xfrm_unregister_type(&esp6_type, AF_INET6); 1248 return -EAGAIN; 1249 } 1250 1251 return 0; 1252 } 1253 1254 static void __exit esp6_fini(void) 1255 { 1256 if (xfrm6_protocol_deregister(&esp6_protocol, IPPROTO_ESP) < 0) 1257 pr_info("%s: can't remove protocol\n", __func__); 1258 xfrm_unregister_type(&esp6_type, AF_INET6); 1259 } 1260 1261 module_init(esp6_init); 1262 module_exit(esp6_fini); 1263 1264 MODULE_DESCRIPTION("IPv6 ESP transformation helpers"); 1265 MODULE_LICENSE("GPL"); 1266 MODULE_ALIAS_XFRM_TYPE(AF_INET6, XFRM_PROTO_ESP); 1267