1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/module.h> 3 #include <linux/errno.h> 4 #include <linux/socket.h> 5 #include <linux/skbuff.h> 6 #include <linux/ip.h> 7 #include <linux/icmp.h> 8 #include <linux/udp.h> 9 #include <linux/types.h> 10 #include <linux/kernel.h> 11 #include <net/genetlink.h> 12 #include <net/gro.h> 13 #include <net/gue.h> 14 #include <net/fou.h> 15 #include <net/ip.h> 16 #include <net/protocol.h> 17 #include <net/udp.h> 18 #include <net/udp_tunnel.h> 19 #include <uapi/linux/fou.h> 20 #include <uapi/linux/genetlink.h> 21 22 #include "fou_nl.h" 23 24 struct fou { 25 struct socket *sock; 26 u8 protocol; 27 u8 flags; 28 __be16 port; 29 u8 family; 30 u16 type; 31 struct list_head list; 32 struct rcu_head rcu; 33 }; 34 35 #define FOU_F_REMCSUM_NOPARTIAL BIT(0) 36 37 struct fou_cfg { 38 u16 type; 39 u8 protocol; 40 u8 flags; 41 struct udp_port_cfg udp_config; 42 }; 43 44 static unsigned int fou_net_id; 45 46 struct fou_net { 47 struct list_head fou_list; 48 struct mutex fou_lock; 49 }; 50 51 static inline struct fou *fou_from_sock(struct sock *sk) 52 { 53 return rcu_dereference_sk_user_data(sk); 54 } 55 56 static int fou_recv_pull(struct sk_buff *skb, struct fou *fou, size_t len) 57 { 58 /* Remove 'len' bytes from the packet (UDP header and 59 * FOU header if present). 60 */ 61 if (fou->family == AF_INET) 62 ip_hdr(skb)->tot_len = htons(ntohs(ip_hdr(skb)->tot_len) - len); 63 else 64 ipv6_hdr(skb)->payload_len = 65 htons(ntohs(ipv6_hdr(skb)->payload_len) - len); 66 67 __skb_pull(skb, len); 68 skb_postpull_rcsum(skb, udp_hdr(skb), len); 69 skb_reset_transport_header(skb); 70 return iptunnel_pull_offloads(skb); 71 } 72 73 static int fou_udp_recv(struct sock *sk, struct sk_buff *skb) 74 { 75 struct fou *fou = fou_from_sock(sk); 76 77 if (!fou) 78 return 1; 79 80 if (fou_recv_pull(skb, fou, sizeof(struct udphdr))) 81 goto drop; 82 83 return -fou->protocol; 84 85 drop: 86 kfree_skb(skb); 87 return 0; 88 } 89 90 static struct guehdr *gue_remcsum(struct sk_buff *skb, struct guehdr *guehdr, 91 void *data, size_t hdrlen, u8 ipproto, 92 bool nopartial) 93 { 94 __be16 *pd = data; 95 size_t start = ntohs(pd[0]); 96 size_t offset = ntohs(pd[1]); 97 size_t plen = sizeof(struct udphdr) + hdrlen + 98 max_t(size_t, offset + sizeof(u16), start); 99 100 if (skb->remcsum_offload) 101 return guehdr; 102 103 if (!pskb_may_pull(skb, plen)) 104 return NULL; 105 guehdr = (struct guehdr *)&udp_hdr(skb)[1]; 106 107 skb_remcsum_process(skb, (void *)guehdr + hdrlen, 108 start, offset, nopartial); 109 110 return guehdr; 111 } 112 113 static int gue_control_message(struct sk_buff *skb, struct guehdr *guehdr) 114 { 115 /* No support yet */ 116 kfree_skb(skb); 117 return 0; 118 } 119 120 static int gue_udp_recv(struct sock *sk, struct sk_buff *skb) 121 { 122 struct fou *fou = fou_from_sock(sk); 123 size_t len, optlen, hdrlen; 124 struct guehdr *guehdr; 125 void *data; 126 u16 doffset = 0; 127 u8 proto_ctype; 128 129 if (!fou) 130 return 1; 131 132 len = sizeof(struct udphdr) + sizeof(struct guehdr); 133 if (!pskb_may_pull(skb, len)) 134 goto drop; 135 136 guehdr = (struct guehdr *)&udp_hdr(skb)[1]; 137 138 switch (guehdr->version) { 139 case 0: /* Full GUE header present */ 140 break; 141 142 case 1: { 143 /* Direct encapsulation of IPv4 or IPv6 */ 144 145 int prot; 146 147 switch (((struct iphdr *)guehdr)->version) { 148 case 4: 149 prot = IPPROTO_IPIP; 150 break; 151 case 6: 152 prot = IPPROTO_IPV6; 153 break; 154 default: 155 goto drop; 156 } 157 158 if (fou_recv_pull(skb, fou, sizeof(struct udphdr))) 159 goto drop; 160 161 return -prot; 162 } 163 164 default: /* Undefined version */ 165 goto drop; 166 } 167 168 optlen = guehdr->hlen << 2; 169 len += optlen; 170 171 if (!pskb_may_pull(skb, len)) 172 goto drop; 173 174 /* guehdr may change after pull */ 175 guehdr = (struct guehdr *)&udp_hdr(skb)[1]; 176 177 if (validate_gue_flags(guehdr, optlen)) 178 goto drop; 179 180 hdrlen = sizeof(struct guehdr) + optlen; 181 182 if (fou->family == AF_INET) 183 ip_hdr(skb)->tot_len = htons(ntohs(ip_hdr(skb)->tot_len) - len); 184 else 185 ipv6_hdr(skb)->payload_len = 186 htons(ntohs(ipv6_hdr(skb)->payload_len) - len); 187 188 /* Pull csum through the guehdr now . This can be used if 189 * there is a remote checksum offload. 190 */ 191 skb_postpull_rcsum(skb, udp_hdr(skb), len); 192 193 data = &guehdr[1]; 194 195 if (guehdr->flags & GUE_FLAG_PRIV) { 196 __be32 flags = *(__be32 *)(data + doffset); 197 198 doffset += GUE_LEN_PRIV; 199 200 if (flags & GUE_PFLAG_REMCSUM) { 201 guehdr = gue_remcsum(skb, guehdr, data + doffset, 202 hdrlen, guehdr->proto_ctype, 203 !!(fou->flags & 204 FOU_F_REMCSUM_NOPARTIAL)); 205 if (!guehdr) 206 goto drop; 207 208 data = &guehdr[1]; 209 210 doffset += GUE_PLEN_REMCSUM; 211 } 212 } 213 214 if (unlikely(guehdr->control)) 215 return gue_control_message(skb, guehdr); 216 217 proto_ctype = guehdr->proto_ctype; 218 __skb_pull(skb, sizeof(struct udphdr) + hdrlen); 219 skb_reset_transport_header(skb); 220 221 if (iptunnel_pull_offloads(skb)) 222 goto drop; 223 224 return -proto_ctype; 225 226 drop: 227 kfree_skb(skb); 228 return 0; 229 } 230 231 static const struct net_offload *fou_gro_ops(const struct sock *sk, 232 int proto) 233 { 234 const struct net_offload __rcu **offloads; 235 236 /* FOU doesn't allow IPv4 on IPv6 sockets. */ 237 offloads = sk->sk_family == AF_INET6 ? inet6_offloads : inet_offloads; 238 return rcu_dereference(offloads[proto]); 239 } 240 241 static struct sk_buff *fou_gro_receive(struct sock *sk, 242 struct list_head *head, 243 struct sk_buff *skb) 244 { 245 struct fou *fou = fou_from_sock(sk); 246 const struct net_offload *ops; 247 struct sk_buff *pp = NULL; 248 249 if (!fou) 250 goto out; 251 252 /* We can clear the encap_mark for FOU as we are essentially doing 253 * one of two possible things. We are either adding an L4 tunnel 254 * header to the outer L3 tunnel header, or we are simply 255 * treating the GRE tunnel header as though it is a UDP protocol 256 * specific header such as VXLAN or GENEVE. 257 */ 258 NAPI_GRO_CB(skb)->encap_mark = 0; 259 260 /* Flag this frame as already having an outer encap header */ 261 NAPI_GRO_CB(skb)->is_fou = 1; 262 263 ops = fou_gro_ops(sk, fou->protocol); 264 if (!ops || !ops->callbacks.gro_receive) 265 goto out; 266 267 pp = call_gro_receive(ops->callbacks.gro_receive, head, skb); 268 269 out: 270 return pp; 271 } 272 273 static int fou_gro_complete(struct sock *sk, struct sk_buff *skb, 274 int nhoff) 275 { 276 struct fou *fou = fou_from_sock(sk); 277 const struct net_offload *ops; 278 int err; 279 280 if (!fou) { 281 err = -ENOENT; 282 goto out; 283 } 284 285 ops = fou_gro_ops(sk, fou->protocol); 286 if (WARN_ON(!ops || !ops->callbacks.gro_complete)) { 287 err = -ENOSYS; 288 goto out; 289 } 290 291 err = ops->callbacks.gro_complete(skb, nhoff); 292 293 skb_set_inner_mac_header(skb, nhoff); 294 295 out: 296 return err; 297 } 298 299 static struct guehdr *gue_gro_remcsum(struct sk_buff *skb, unsigned int off, 300 struct guehdr *guehdr, void *data, 301 size_t hdrlen, struct gro_remcsum *grc, 302 bool nopartial) 303 { 304 __be16 *pd = data; 305 size_t start = ntohs(pd[0]); 306 size_t offset = ntohs(pd[1]); 307 308 if (skb->remcsum_offload) 309 return guehdr; 310 311 if (!NAPI_GRO_CB(skb)->csum_valid) 312 return NULL; 313 314 guehdr = skb_gro_remcsum_process(skb, (void *)guehdr, off, hdrlen, 315 start, offset, grc, nopartial); 316 317 skb->remcsum_offload = 1; 318 319 return guehdr; 320 } 321 322 static struct sk_buff *gue_gro_receive(struct sock *sk, 323 struct list_head *head, 324 struct sk_buff *skb) 325 { 326 const struct net_offload *ops; 327 struct sk_buff *pp = NULL; 328 struct sk_buff *p; 329 struct guehdr *guehdr; 330 size_t len, optlen, hdrlen, off; 331 void *data; 332 u16 doffset = 0; 333 int flush = 1; 334 struct fou *fou = fou_from_sock(sk); 335 struct gro_remcsum grc; 336 u8 proto; 337 338 skb_gro_remcsum_init(&grc); 339 340 if (!fou) 341 goto out; 342 343 off = skb_gro_offset(skb); 344 len = off + sizeof(*guehdr); 345 346 guehdr = skb_gro_header(skb, len, off); 347 if (unlikely(!guehdr)) 348 goto out; 349 350 switch (guehdr->version) { 351 case 0: 352 break; 353 case 1: 354 switch (((struct iphdr *)guehdr)->version) { 355 case 4: 356 proto = IPPROTO_IPIP; 357 break; 358 case 6: 359 proto = IPPROTO_IPV6; 360 break; 361 default: 362 goto out; 363 } 364 goto next_proto; 365 default: 366 goto out; 367 } 368 369 optlen = guehdr->hlen << 2; 370 len += optlen; 371 372 if (!skb_gro_may_pull(skb, len)) { 373 guehdr = skb_gro_header_slow(skb, len, off); 374 if (unlikely(!guehdr)) 375 goto out; 376 } 377 378 if (unlikely(guehdr->control) || guehdr->version != 0 || 379 validate_gue_flags(guehdr, optlen)) 380 goto out; 381 382 hdrlen = sizeof(*guehdr) + optlen; 383 384 /* Adjust NAPI_GRO_CB(skb)->csum to account for guehdr, 385 * this is needed if there is a remote checkcsum offload. 386 */ 387 skb_gro_postpull_rcsum(skb, guehdr, hdrlen); 388 389 data = &guehdr[1]; 390 391 if (guehdr->flags & GUE_FLAG_PRIV) { 392 __be32 flags = *(__be32 *)(data + doffset); 393 394 doffset += GUE_LEN_PRIV; 395 396 if (flags & GUE_PFLAG_REMCSUM) { 397 guehdr = gue_gro_remcsum(skb, off, guehdr, 398 data + doffset, hdrlen, &grc, 399 !!(fou->flags & 400 FOU_F_REMCSUM_NOPARTIAL)); 401 402 if (!guehdr) 403 goto out; 404 405 data = &guehdr[1]; 406 407 doffset += GUE_PLEN_REMCSUM; 408 } 409 } 410 411 skb_gro_pull(skb, hdrlen); 412 413 list_for_each_entry(p, head, list) { 414 const struct guehdr *guehdr2; 415 416 if (!NAPI_GRO_CB(p)->same_flow) 417 continue; 418 419 guehdr2 = (struct guehdr *)(p->data + off); 420 421 /* Compare base GUE header to be equal (covers 422 * hlen, version, proto_ctype, and flags. 423 */ 424 if (guehdr->word != guehdr2->word) { 425 NAPI_GRO_CB(p)->same_flow = 0; 426 continue; 427 } 428 429 /* Compare optional fields are the same. */ 430 if (guehdr->hlen && memcmp(&guehdr[1], &guehdr2[1], 431 guehdr->hlen << 2)) { 432 NAPI_GRO_CB(p)->same_flow = 0; 433 continue; 434 } 435 } 436 437 proto = guehdr->proto_ctype; 438 439 next_proto: 440 441 /* We can clear the encap_mark for GUE as we are essentially doing 442 * one of two possible things. We are either adding an L4 tunnel 443 * header to the outer L3 tunnel header, or we are simply 444 * treating the GRE tunnel header as though it is a UDP protocol 445 * specific header such as VXLAN or GENEVE. 446 */ 447 NAPI_GRO_CB(skb)->encap_mark = 0; 448 449 /* Flag this frame as already having an outer encap header */ 450 NAPI_GRO_CB(skb)->is_fou = 1; 451 452 ops = fou_gro_ops(sk, proto); 453 if (!ops || !ops->callbacks.gro_receive) 454 goto out; 455 456 pp = call_gro_receive(ops->callbacks.gro_receive, head, skb); 457 flush = 0; 458 459 out: 460 skb_gro_flush_final_remcsum(skb, pp, flush, &grc); 461 462 return pp; 463 } 464 465 static int gue_gro_complete(struct sock *sk, struct sk_buff *skb, int nhoff) 466 { 467 struct guehdr *guehdr = (struct guehdr *)(skb->data + nhoff); 468 const struct net_offload *ops; 469 unsigned int guehlen = 0; 470 u8 proto; 471 int err = -ENOENT; 472 473 switch (guehdr->version) { 474 case 0: 475 proto = guehdr->proto_ctype; 476 guehlen = sizeof(*guehdr) + (guehdr->hlen << 2); 477 break; 478 case 1: 479 switch (((struct iphdr *)guehdr)->version) { 480 case 4: 481 proto = IPPROTO_IPIP; 482 break; 483 case 6: 484 proto = IPPROTO_IPV6; 485 break; 486 default: 487 return err; 488 } 489 break; 490 default: 491 return err; 492 } 493 494 ops = fou_gro_ops(sk, proto); 495 if (WARN_ON(!ops || !ops->callbacks.gro_complete)) 496 goto out; 497 498 err = ops->callbacks.gro_complete(skb, nhoff + guehlen); 499 500 skb_set_inner_mac_header(skb, nhoff + guehlen); 501 502 out: 503 return err; 504 } 505 506 static bool fou_cfg_cmp(struct fou *fou, struct fou_cfg *cfg) 507 { 508 struct sock *sk = fou->sock->sk; 509 struct udp_port_cfg *udp_cfg = &cfg->udp_config; 510 511 if (fou->family != udp_cfg->family || 512 fou->port != udp_cfg->local_udp_port || 513 sk->sk_dport != udp_cfg->peer_udp_port || 514 sk->sk_bound_dev_if != udp_cfg->bind_ifindex) 515 return false; 516 517 if (fou->family == AF_INET) { 518 if (sk->sk_rcv_saddr != udp_cfg->local_ip.s_addr || 519 sk->sk_daddr != udp_cfg->peer_ip.s_addr) 520 return false; 521 else 522 return true; 523 #if IS_ENABLED(CONFIG_IPV6) 524 } else { 525 if (ipv6_addr_cmp(&sk->sk_v6_rcv_saddr, &udp_cfg->local_ip6) || 526 ipv6_addr_cmp(&sk->sk_v6_daddr, &udp_cfg->peer_ip6)) 527 return false; 528 else 529 return true; 530 #endif 531 } 532 533 return false; 534 } 535 536 static int fou_add_to_port_list(struct net *net, struct fou *fou, 537 struct fou_cfg *cfg) 538 { 539 struct fou_net *fn = net_generic(net, fou_net_id); 540 struct fou *fout; 541 542 mutex_lock(&fn->fou_lock); 543 list_for_each_entry(fout, &fn->fou_list, list) { 544 if (fou_cfg_cmp(fout, cfg)) { 545 mutex_unlock(&fn->fou_lock); 546 return -EALREADY; 547 } 548 } 549 550 list_add(&fou->list, &fn->fou_list); 551 mutex_unlock(&fn->fou_lock); 552 553 return 0; 554 } 555 556 static void fou_release(struct fou *fou) 557 { 558 struct socket *sock = fou->sock; 559 560 list_del(&fou->list); 561 udp_tunnel_sock_release(sock); 562 563 kfree_rcu(fou, rcu); 564 } 565 566 static int fou_create(struct net *net, struct fou_cfg *cfg, 567 struct socket **sockp) 568 { 569 struct socket *sock = NULL; 570 struct fou *fou = NULL; 571 struct sock *sk; 572 struct udp_tunnel_sock_cfg tunnel_cfg; 573 int err; 574 575 /* Open UDP socket */ 576 err = udp_sock_create(net, &cfg->udp_config, &sock); 577 if (err < 0) 578 goto error; 579 580 /* Allocate FOU port structure */ 581 fou = kzalloc(sizeof(*fou), GFP_KERNEL); 582 if (!fou) { 583 err = -ENOMEM; 584 goto error; 585 } 586 587 sk = sock->sk; 588 589 fou->port = cfg->udp_config.local_udp_port; 590 fou->family = cfg->udp_config.family; 591 fou->flags = cfg->flags; 592 fou->type = cfg->type; 593 fou->sock = sock; 594 595 memset(&tunnel_cfg, 0, sizeof(tunnel_cfg)); 596 tunnel_cfg.encap_type = 1; 597 tunnel_cfg.sk_user_data = fou; 598 tunnel_cfg.encap_destroy = NULL; 599 600 /* Initial for fou type */ 601 switch (cfg->type) { 602 case FOU_ENCAP_DIRECT: 603 tunnel_cfg.encap_rcv = fou_udp_recv; 604 tunnel_cfg.gro_receive = fou_gro_receive; 605 tunnel_cfg.gro_complete = fou_gro_complete; 606 fou->protocol = cfg->protocol; 607 break; 608 case FOU_ENCAP_GUE: 609 tunnel_cfg.encap_rcv = gue_udp_recv; 610 tunnel_cfg.gro_receive = gue_gro_receive; 611 tunnel_cfg.gro_complete = gue_gro_complete; 612 break; 613 default: 614 err = -EINVAL; 615 goto error; 616 } 617 618 setup_udp_tunnel_sock(net, sock, &tunnel_cfg); 619 620 sk->sk_allocation = GFP_ATOMIC; 621 622 err = fou_add_to_port_list(net, fou, cfg); 623 if (err) 624 goto error; 625 626 if (sockp) 627 *sockp = sock; 628 629 return 0; 630 631 error: 632 kfree(fou); 633 if (sock) 634 udp_tunnel_sock_release(sock); 635 636 return err; 637 } 638 639 static int fou_destroy(struct net *net, struct fou_cfg *cfg) 640 { 641 struct fou_net *fn = net_generic(net, fou_net_id); 642 int err = -EINVAL; 643 struct fou *fou; 644 645 mutex_lock(&fn->fou_lock); 646 list_for_each_entry(fou, &fn->fou_list, list) { 647 if (fou_cfg_cmp(fou, cfg)) { 648 fou_release(fou); 649 err = 0; 650 break; 651 } 652 } 653 mutex_unlock(&fn->fou_lock); 654 655 return err; 656 } 657 658 static struct genl_family fou_nl_family; 659 660 static int parse_nl_config(struct genl_info *info, 661 struct fou_cfg *cfg) 662 { 663 bool has_local = false, has_peer = false; 664 struct nlattr *attr; 665 int ifindex; 666 __be16 port; 667 668 memset(cfg, 0, sizeof(*cfg)); 669 670 cfg->udp_config.family = AF_INET; 671 672 if (info->attrs[FOU_ATTR_AF]) { 673 u8 family = nla_get_u8(info->attrs[FOU_ATTR_AF]); 674 675 switch (family) { 676 case AF_INET: 677 break; 678 case AF_INET6: 679 cfg->udp_config.ipv6_v6only = 1; 680 break; 681 default: 682 return -EAFNOSUPPORT; 683 } 684 685 cfg->udp_config.family = family; 686 } 687 688 if (info->attrs[FOU_ATTR_PORT]) { 689 port = nla_get_be16(info->attrs[FOU_ATTR_PORT]); 690 cfg->udp_config.local_udp_port = port; 691 } 692 693 if (info->attrs[FOU_ATTR_IPPROTO]) 694 cfg->protocol = nla_get_u8(info->attrs[FOU_ATTR_IPPROTO]); 695 696 if (info->attrs[FOU_ATTR_TYPE]) 697 cfg->type = nla_get_u8(info->attrs[FOU_ATTR_TYPE]); 698 699 if (info->attrs[FOU_ATTR_REMCSUM_NOPARTIAL]) 700 cfg->flags |= FOU_F_REMCSUM_NOPARTIAL; 701 702 if (cfg->udp_config.family == AF_INET) { 703 if (info->attrs[FOU_ATTR_LOCAL_V4]) { 704 attr = info->attrs[FOU_ATTR_LOCAL_V4]; 705 cfg->udp_config.local_ip.s_addr = nla_get_in_addr(attr); 706 has_local = true; 707 } 708 709 if (info->attrs[FOU_ATTR_PEER_V4]) { 710 attr = info->attrs[FOU_ATTR_PEER_V4]; 711 cfg->udp_config.peer_ip.s_addr = nla_get_in_addr(attr); 712 has_peer = true; 713 } 714 #if IS_ENABLED(CONFIG_IPV6) 715 } else { 716 if (info->attrs[FOU_ATTR_LOCAL_V6]) { 717 attr = info->attrs[FOU_ATTR_LOCAL_V6]; 718 cfg->udp_config.local_ip6 = nla_get_in6_addr(attr); 719 has_local = true; 720 } 721 722 if (info->attrs[FOU_ATTR_PEER_V6]) { 723 attr = info->attrs[FOU_ATTR_PEER_V6]; 724 cfg->udp_config.peer_ip6 = nla_get_in6_addr(attr); 725 has_peer = true; 726 } 727 #endif 728 } 729 730 if (has_peer) { 731 if (info->attrs[FOU_ATTR_PEER_PORT]) { 732 port = nla_get_be16(info->attrs[FOU_ATTR_PEER_PORT]); 733 cfg->udp_config.peer_udp_port = port; 734 } else { 735 return -EINVAL; 736 } 737 } 738 739 if (info->attrs[FOU_ATTR_IFINDEX]) { 740 if (!has_local) 741 return -EINVAL; 742 743 ifindex = nla_get_s32(info->attrs[FOU_ATTR_IFINDEX]); 744 745 cfg->udp_config.bind_ifindex = ifindex; 746 } 747 748 return 0; 749 } 750 751 int fou_nl_add_doit(struct sk_buff *skb, struct genl_info *info) 752 { 753 struct net *net = genl_info_net(info); 754 struct fou_cfg cfg; 755 int err; 756 757 err = parse_nl_config(info, &cfg); 758 if (err) 759 return err; 760 761 return fou_create(net, &cfg, NULL); 762 } 763 764 int fou_nl_del_doit(struct sk_buff *skb, struct genl_info *info) 765 { 766 struct net *net = genl_info_net(info); 767 struct fou_cfg cfg; 768 int err; 769 770 err = parse_nl_config(info, &cfg); 771 if (err) 772 return err; 773 774 return fou_destroy(net, &cfg); 775 } 776 777 static int fou_fill_info(struct fou *fou, struct sk_buff *msg) 778 { 779 struct sock *sk = fou->sock->sk; 780 781 if (nla_put_u8(msg, FOU_ATTR_AF, fou->sock->sk->sk_family) || 782 nla_put_be16(msg, FOU_ATTR_PORT, fou->port) || 783 nla_put_be16(msg, FOU_ATTR_PEER_PORT, sk->sk_dport) || 784 nla_put_u8(msg, FOU_ATTR_IPPROTO, fou->protocol) || 785 nla_put_u8(msg, FOU_ATTR_TYPE, fou->type) || 786 nla_put_s32(msg, FOU_ATTR_IFINDEX, sk->sk_bound_dev_if)) 787 return -1; 788 789 if (fou->flags & FOU_F_REMCSUM_NOPARTIAL) 790 if (nla_put_flag(msg, FOU_ATTR_REMCSUM_NOPARTIAL)) 791 return -1; 792 793 if (fou->sock->sk->sk_family == AF_INET) { 794 if (nla_put_in_addr(msg, FOU_ATTR_LOCAL_V4, sk->sk_rcv_saddr)) 795 return -1; 796 797 if (nla_put_in_addr(msg, FOU_ATTR_PEER_V4, sk->sk_daddr)) 798 return -1; 799 #if IS_ENABLED(CONFIG_IPV6) 800 } else { 801 if (nla_put_in6_addr(msg, FOU_ATTR_LOCAL_V6, 802 &sk->sk_v6_rcv_saddr)) 803 return -1; 804 805 if (nla_put_in6_addr(msg, FOU_ATTR_PEER_V6, &sk->sk_v6_daddr)) 806 return -1; 807 #endif 808 } 809 810 return 0; 811 } 812 813 static int fou_dump_info(struct fou *fou, u32 portid, u32 seq, 814 u32 flags, struct sk_buff *skb, u8 cmd) 815 { 816 void *hdr; 817 818 hdr = genlmsg_put(skb, portid, seq, &fou_nl_family, flags, cmd); 819 if (!hdr) 820 return -ENOMEM; 821 822 if (fou_fill_info(fou, skb) < 0) 823 goto nla_put_failure; 824 825 genlmsg_end(skb, hdr); 826 return 0; 827 828 nla_put_failure: 829 genlmsg_cancel(skb, hdr); 830 return -EMSGSIZE; 831 } 832 833 int fou_nl_get_doit(struct sk_buff *skb, struct genl_info *info) 834 { 835 struct net *net = genl_info_net(info); 836 struct fou_net *fn = net_generic(net, fou_net_id); 837 struct sk_buff *msg; 838 struct fou_cfg cfg; 839 struct fou *fout; 840 __be16 port; 841 u8 family; 842 int ret; 843 844 ret = parse_nl_config(info, &cfg); 845 if (ret) 846 return ret; 847 port = cfg.udp_config.local_udp_port; 848 if (port == 0) 849 return -EINVAL; 850 851 family = cfg.udp_config.family; 852 if (family != AF_INET && family != AF_INET6) 853 return -EINVAL; 854 855 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 856 if (!msg) 857 return -ENOMEM; 858 859 ret = -ESRCH; 860 mutex_lock(&fn->fou_lock); 861 list_for_each_entry(fout, &fn->fou_list, list) { 862 if (fou_cfg_cmp(fout, &cfg)) { 863 ret = fou_dump_info(fout, info->snd_portid, 864 info->snd_seq, 0, msg, 865 info->genlhdr->cmd); 866 break; 867 } 868 } 869 mutex_unlock(&fn->fou_lock); 870 if (ret < 0) 871 goto out_free; 872 873 return genlmsg_reply(msg, info); 874 875 out_free: 876 nlmsg_free(msg); 877 return ret; 878 } 879 880 int fou_nl_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 881 { 882 struct net *net = sock_net(skb->sk); 883 struct fou_net *fn = net_generic(net, fou_net_id); 884 struct fou *fout; 885 int idx = 0, ret; 886 887 mutex_lock(&fn->fou_lock); 888 list_for_each_entry(fout, &fn->fou_list, list) { 889 if (idx++ < cb->args[0]) 890 continue; 891 ret = fou_dump_info(fout, NETLINK_CB(cb->skb).portid, 892 cb->nlh->nlmsg_seq, NLM_F_MULTI, 893 skb, FOU_CMD_GET); 894 if (ret) 895 break; 896 } 897 mutex_unlock(&fn->fou_lock); 898 899 cb->args[0] = idx; 900 return skb->len; 901 } 902 903 static struct genl_family fou_nl_family __ro_after_init = { 904 .hdrsize = 0, 905 .name = FOU_GENL_NAME, 906 .version = FOU_GENL_VERSION, 907 .maxattr = FOU_ATTR_MAX, 908 .policy = fou_nl_policy, 909 .netnsok = true, 910 .module = THIS_MODULE, 911 .small_ops = fou_nl_ops, 912 .n_small_ops = ARRAY_SIZE(fou_nl_ops), 913 .resv_start_op = FOU_CMD_GET + 1, 914 }; 915 916 size_t fou_encap_hlen(struct ip_tunnel_encap *e) 917 { 918 return sizeof(struct udphdr); 919 } 920 EXPORT_SYMBOL(fou_encap_hlen); 921 922 size_t gue_encap_hlen(struct ip_tunnel_encap *e) 923 { 924 size_t len; 925 bool need_priv = false; 926 927 len = sizeof(struct udphdr) + sizeof(struct guehdr); 928 929 if (e->flags & TUNNEL_ENCAP_FLAG_REMCSUM) { 930 len += GUE_PLEN_REMCSUM; 931 need_priv = true; 932 } 933 934 len += need_priv ? GUE_LEN_PRIV : 0; 935 936 return len; 937 } 938 EXPORT_SYMBOL(gue_encap_hlen); 939 940 int __fou_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e, 941 u8 *protocol, __be16 *sport, int type) 942 { 943 int err; 944 945 err = iptunnel_handle_offloads(skb, type); 946 if (err) 947 return err; 948 949 *sport = e->sport ? : udp_flow_src_port(dev_net(skb->dev), 950 skb, 0, 0, false); 951 952 return 0; 953 } 954 EXPORT_SYMBOL(__fou_build_header); 955 956 int __gue_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e, 957 u8 *protocol, __be16 *sport, int type) 958 { 959 struct guehdr *guehdr; 960 size_t hdrlen, optlen = 0; 961 void *data; 962 bool need_priv = false; 963 int err; 964 965 if ((e->flags & TUNNEL_ENCAP_FLAG_REMCSUM) && 966 skb->ip_summed == CHECKSUM_PARTIAL) { 967 optlen += GUE_PLEN_REMCSUM; 968 type |= SKB_GSO_TUNNEL_REMCSUM; 969 need_priv = true; 970 } 971 972 optlen += need_priv ? GUE_LEN_PRIV : 0; 973 974 err = iptunnel_handle_offloads(skb, type); 975 if (err) 976 return err; 977 978 /* Get source port (based on flow hash) before skb_push */ 979 *sport = e->sport ? : udp_flow_src_port(dev_net(skb->dev), 980 skb, 0, 0, false); 981 982 hdrlen = sizeof(struct guehdr) + optlen; 983 984 skb_push(skb, hdrlen); 985 986 guehdr = (struct guehdr *)skb->data; 987 988 guehdr->control = 0; 989 guehdr->version = 0; 990 guehdr->hlen = optlen >> 2; 991 guehdr->flags = 0; 992 guehdr->proto_ctype = *protocol; 993 994 data = &guehdr[1]; 995 996 if (need_priv) { 997 __be32 *flags = data; 998 999 guehdr->flags |= GUE_FLAG_PRIV; 1000 *flags = 0; 1001 data += GUE_LEN_PRIV; 1002 1003 if (type & SKB_GSO_TUNNEL_REMCSUM) { 1004 u16 csum_start = skb_checksum_start_offset(skb); 1005 __be16 *pd = data; 1006 1007 if (csum_start < hdrlen) 1008 return -EINVAL; 1009 1010 csum_start -= hdrlen; 1011 pd[0] = htons(csum_start); 1012 pd[1] = htons(csum_start + skb->csum_offset); 1013 1014 if (!skb_is_gso(skb)) { 1015 skb->ip_summed = CHECKSUM_NONE; 1016 skb->encapsulation = 0; 1017 } 1018 1019 *flags |= GUE_PFLAG_REMCSUM; 1020 data += GUE_PLEN_REMCSUM; 1021 } 1022 1023 } 1024 1025 return 0; 1026 } 1027 EXPORT_SYMBOL(__gue_build_header); 1028 1029 #ifdef CONFIG_NET_FOU_IP_TUNNELS 1030 1031 static void fou_build_udp(struct sk_buff *skb, struct ip_tunnel_encap *e, 1032 struct flowi4 *fl4, u8 *protocol, __be16 sport) 1033 { 1034 struct udphdr *uh; 1035 1036 skb_push(skb, sizeof(struct udphdr)); 1037 skb_reset_transport_header(skb); 1038 1039 uh = udp_hdr(skb); 1040 1041 uh->dest = e->dport; 1042 uh->source = sport; 1043 uh->len = htons(skb->len); 1044 udp_set_csum(!(e->flags & TUNNEL_ENCAP_FLAG_CSUM), skb, 1045 fl4->saddr, fl4->daddr, skb->len); 1046 1047 *protocol = IPPROTO_UDP; 1048 } 1049 1050 static int fou_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e, 1051 u8 *protocol, struct flowi4 *fl4) 1052 { 1053 int type = e->flags & TUNNEL_ENCAP_FLAG_CSUM ? SKB_GSO_UDP_TUNNEL_CSUM : 1054 SKB_GSO_UDP_TUNNEL; 1055 __be16 sport; 1056 int err; 1057 1058 err = __fou_build_header(skb, e, protocol, &sport, type); 1059 if (err) 1060 return err; 1061 1062 fou_build_udp(skb, e, fl4, protocol, sport); 1063 1064 return 0; 1065 } 1066 1067 static int gue_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e, 1068 u8 *protocol, struct flowi4 *fl4) 1069 { 1070 int type = e->flags & TUNNEL_ENCAP_FLAG_CSUM ? SKB_GSO_UDP_TUNNEL_CSUM : 1071 SKB_GSO_UDP_TUNNEL; 1072 __be16 sport; 1073 int err; 1074 1075 err = __gue_build_header(skb, e, protocol, &sport, type); 1076 if (err) 1077 return err; 1078 1079 fou_build_udp(skb, e, fl4, protocol, sport); 1080 1081 return 0; 1082 } 1083 1084 static int gue_err_proto_handler(int proto, struct sk_buff *skb, u32 info) 1085 { 1086 const struct net_protocol *ipprot = rcu_dereference(inet_protos[proto]); 1087 1088 if (ipprot && ipprot->err_handler) { 1089 if (!ipprot->err_handler(skb, info)) 1090 return 0; 1091 } 1092 1093 return -ENOENT; 1094 } 1095 1096 static int gue_err(struct sk_buff *skb, u32 info) 1097 { 1098 int transport_offset = skb_transport_offset(skb); 1099 struct guehdr *guehdr; 1100 size_t len, optlen; 1101 int ret; 1102 1103 len = sizeof(struct udphdr) + sizeof(struct guehdr); 1104 if (!pskb_may_pull(skb, transport_offset + len)) 1105 return -EINVAL; 1106 1107 guehdr = (struct guehdr *)&udp_hdr(skb)[1]; 1108 1109 switch (guehdr->version) { 1110 case 0: /* Full GUE header present */ 1111 break; 1112 case 1: { 1113 /* Direct encapsulation of IPv4 or IPv6 */ 1114 skb_set_transport_header(skb, -(int)sizeof(struct icmphdr)); 1115 1116 switch (((struct iphdr *)guehdr)->version) { 1117 case 4: 1118 ret = gue_err_proto_handler(IPPROTO_IPIP, skb, info); 1119 goto out; 1120 #if IS_ENABLED(CONFIG_IPV6) 1121 case 6: 1122 ret = gue_err_proto_handler(IPPROTO_IPV6, skb, info); 1123 goto out; 1124 #endif 1125 default: 1126 ret = -EOPNOTSUPP; 1127 goto out; 1128 } 1129 } 1130 default: /* Undefined version */ 1131 return -EOPNOTSUPP; 1132 } 1133 1134 if (guehdr->control) 1135 return -ENOENT; 1136 1137 optlen = guehdr->hlen << 2; 1138 1139 if (!pskb_may_pull(skb, transport_offset + len + optlen)) 1140 return -EINVAL; 1141 1142 guehdr = (struct guehdr *)&udp_hdr(skb)[1]; 1143 if (validate_gue_flags(guehdr, optlen)) 1144 return -EINVAL; 1145 1146 /* Handling exceptions for direct UDP encapsulation in GUE would lead to 1147 * recursion. Besides, this kind of encapsulation can't even be 1148 * configured currently. Discard this. 1149 */ 1150 if (guehdr->proto_ctype == IPPROTO_UDP || 1151 guehdr->proto_ctype == IPPROTO_UDPLITE) 1152 return -EOPNOTSUPP; 1153 1154 skb_set_transport_header(skb, -(int)sizeof(struct icmphdr)); 1155 ret = gue_err_proto_handler(guehdr->proto_ctype, skb, info); 1156 1157 out: 1158 skb_set_transport_header(skb, transport_offset); 1159 return ret; 1160 } 1161 1162 1163 static const struct ip_tunnel_encap_ops fou_iptun_ops = { 1164 .encap_hlen = fou_encap_hlen, 1165 .build_header = fou_build_header, 1166 .err_handler = gue_err, 1167 }; 1168 1169 static const struct ip_tunnel_encap_ops gue_iptun_ops = { 1170 .encap_hlen = gue_encap_hlen, 1171 .build_header = gue_build_header, 1172 .err_handler = gue_err, 1173 }; 1174 1175 static int ip_tunnel_encap_add_fou_ops(void) 1176 { 1177 int ret; 1178 1179 ret = ip_tunnel_encap_add_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU); 1180 if (ret < 0) { 1181 pr_err("can't add fou ops\n"); 1182 return ret; 1183 } 1184 1185 ret = ip_tunnel_encap_add_ops(&gue_iptun_ops, TUNNEL_ENCAP_GUE); 1186 if (ret < 0) { 1187 pr_err("can't add gue ops\n"); 1188 ip_tunnel_encap_del_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU); 1189 return ret; 1190 } 1191 1192 return 0; 1193 } 1194 1195 static void ip_tunnel_encap_del_fou_ops(void) 1196 { 1197 ip_tunnel_encap_del_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU); 1198 ip_tunnel_encap_del_ops(&gue_iptun_ops, TUNNEL_ENCAP_GUE); 1199 } 1200 1201 #else 1202 1203 static int ip_tunnel_encap_add_fou_ops(void) 1204 { 1205 return 0; 1206 } 1207 1208 static void ip_tunnel_encap_del_fou_ops(void) 1209 { 1210 } 1211 1212 #endif 1213 1214 static __net_init int fou_init_net(struct net *net) 1215 { 1216 struct fou_net *fn = net_generic(net, fou_net_id); 1217 1218 INIT_LIST_HEAD(&fn->fou_list); 1219 mutex_init(&fn->fou_lock); 1220 return 0; 1221 } 1222 1223 static __net_exit void fou_exit_net(struct net *net) 1224 { 1225 struct fou_net *fn = net_generic(net, fou_net_id); 1226 struct fou *fou, *next; 1227 1228 /* Close all the FOU sockets */ 1229 mutex_lock(&fn->fou_lock); 1230 list_for_each_entry_safe(fou, next, &fn->fou_list, list) 1231 fou_release(fou); 1232 mutex_unlock(&fn->fou_lock); 1233 } 1234 1235 static struct pernet_operations fou_net_ops = { 1236 .init = fou_init_net, 1237 .exit = fou_exit_net, 1238 .id = &fou_net_id, 1239 .size = sizeof(struct fou_net), 1240 }; 1241 1242 static int __init fou_init(void) 1243 { 1244 int ret; 1245 1246 ret = register_pernet_device(&fou_net_ops); 1247 if (ret) 1248 goto exit; 1249 1250 ret = genl_register_family(&fou_nl_family); 1251 if (ret < 0) 1252 goto unregister; 1253 1254 ret = register_fou_bpf(); 1255 if (ret < 0) 1256 goto kfunc_failed; 1257 1258 ret = ip_tunnel_encap_add_fou_ops(); 1259 if (ret == 0) 1260 return 0; 1261 1262 kfunc_failed: 1263 genl_unregister_family(&fou_nl_family); 1264 unregister: 1265 unregister_pernet_device(&fou_net_ops); 1266 exit: 1267 return ret; 1268 } 1269 1270 static void __exit fou_fini(void) 1271 { 1272 ip_tunnel_encap_del_fou_ops(); 1273 genl_unregister_family(&fou_nl_family); 1274 unregister_pernet_device(&fou_net_ops); 1275 } 1276 1277 module_init(fou_init); 1278 module_exit(fou_fini); 1279 MODULE_AUTHOR("Tom Herbert <therbert@google.com>"); 1280 MODULE_LICENSE("GPL"); 1281 MODULE_DESCRIPTION("Foo over UDP"); 1282