1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/module.h> 3 #include <linux/errno.h> 4 #include <linux/socket.h> 5 #include <linux/skbuff.h> 6 #include <linux/ip.h> 7 #include <linux/icmp.h> 8 #include <linux/udp.h> 9 #include <linux/types.h> 10 #include <linux/kernel.h> 11 #include <net/genetlink.h> 12 #include <net/gro.h> 13 #include <net/gue.h> 14 #include <net/fou.h> 15 #include <net/ip.h> 16 #include <net/protocol.h> 17 #include <net/udp.h> 18 #include <net/udp_tunnel.h> 19 #include <uapi/linux/fou.h> 20 #include <uapi/linux/genetlink.h> 21 22 #include "fou_nl.h" 23 24 struct fou { 25 struct sock *sk; 26 u8 protocol; 27 u8 flags; 28 __be16 port; 29 u8 family; 30 u16 type; 31 struct list_head list; 32 struct rcu_head rcu; 33 }; 34 35 #define FOU_F_REMCSUM_NOPARTIAL BIT(0) 36 37 struct fou_cfg { 38 u16 type; 39 u8 protocol; 40 u8 flags; 41 struct udp_port_cfg udp_config; 42 }; 43 44 static unsigned int fou_net_id; 45 46 struct fou_net { 47 struct list_head fou_list; 48 struct mutex fou_lock; 49 }; 50 51 static inline struct fou *fou_from_sock(struct sock *sk) 52 { 53 return rcu_dereference_sk_user_data(sk); 54 } 55 56 static int fou_recv_pull(struct sk_buff *skb, struct fou *fou, size_t len) 57 { 58 /* Remove 'len' bytes from the packet (UDP header and 59 * FOU header if present). 60 */ 61 if (fou->family == AF_INET) 62 ip_hdr(skb)->tot_len = htons(ntohs(ip_hdr(skb)->tot_len) - len); 63 else 64 ipv6_hdr(skb)->payload_len = 65 htons(ntohs(ipv6_hdr(skb)->payload_len) - len); 66 67 __skb_pull(skb, len); 68 skb_postpull_rcsum(skb, udp_hdr(skb), len); 69 skb_reset_transport_header(skb); 70 return iptunnel_pull_offloads(skb); 71 } 72 73 static int fou_udp_recv(struct sock *sk, struct sk_buff *skb) 74 { 75 struct fou *fou = fou_from_sock(sk); 76 77 if (!fou) 78 return 1; 79 80 if (fou_recv_pull(skb, fou, sizeof(struct udphdr))) 81 goto drop; 82 83 return -fou->protocol; 84 85 drop: 86 kfree_skb(skb); 87 return 0; 88 } 89 90 static struct guehdr *gue_remcsum(struct sk_buff *skb, struct guehdr *guehdr, 91 void *data, size_t hdrlen, u8 ipproto, 92 bool nopartial) 93 { 94 __be16 *pd = data; 95 size_t start = ntohs(pd[0]); 96 size_t offset = ntohs(pd[1]); 97 size_t plen = sizeof(struct udphdr) + hdrlen + 98 max_t(size_t, offset + sizeof(u16), start); 99 100 if (skb->remcsum_offload) 101 return guehdr; 102 103 if (!pskb_may_pull(skb, plen)) 104 return NULL; 105 guehdr = (struct guehdr *)&udp_hdr(skb)[1]; 106 107 skb_remcsum_process(skb, (void *)guehdr + hdrlen, 108 start, offset, nopartial); 109 110 return guehdr; 111 } 112 113 static int gue_control_message(struct sk_buff *skb, struct guehdr *guehdr) 114 { 115 /* No support yet */ 116 kfree_skb(skb); 117 return 0; 118 } 119 120 static int gue_udp_recv(struct sock *sk, struct sk_buff *skb) 121 { 122 struct fou *fou = fou_from_sock(sk); 123 size_t len, optlen, hdrlen; 124 struct guehdr *guehdr; 125 void *data; 126 u16 doffset = 0; 127 u8 proto_ctype; 128 129 if (!fou) 130 return 1; 131 132 len = sizeof(struct udphdr) + sizeof(struct guehdr); 133 if (!pskb_may_pull(skb, len)) 134 goto drop; 135 136 guehdr = (struct guehdr *)&udp_hdr(skb)[1]; 137 138 switch (guehdr->version) { 139 case 0: /* Full GUE header present */ 140 break; 141 142 case 1: { 143 /* Direct encapsulation of IPv4 or IPv6 */ 144 145 int prot; 146 147 switch (((struct iphdr *)guehdr)->version) { 148 case 4: 149 prot = IPPROTO_IPIP; 150 break; 151 case 6: 152 prot = IPPROTO_IPV6; 153 break; 154 default: 155 goto drop; 156 } 157 158 if (fou_recv_pull(skb, fou, sizeof(struct udphdr))) 159 goto drop; 160 161 return -prot; 162 } 163 164 default: /* Undefined version */ 165 goto drop; 166 } 167 168 optlen = guehdr->hlen << 2; 169 len += optlen; 170 171 if (!pskb_may_pull(skb, len)) 172 goto drop; 173 174 /* guehdr may change after pull */ 175 guehdr = (struct guehdr *)&udp_hdr(skb)[1]; 176 177 if (validate_gue_flags(guehdr, optlen)) 178 goto drop; 179 180 hdrlen = sizeof(struct guehdr) + optlen; 181 182 if (fou->family == AF_INET) 183 ip_hdr(skb)->tot_len = htons(ntohs(ip_hdr(skb)->tot_len) - len); 184 else 185 ipv6_hdr(skb)->payload_len = 186 htons(ntohs(ipv6_hdr(skb)->payload_len) - len); 187 188 /* Pull csum through the guehdr now . This can be used if 189 * there is a remote checksum offload. 190 */ 191 skb_postpull_rcsum(skb, udp_hdr(skb), len); 192 193 data = &guehdr[1]; 194 195 if (guehdr->flags & GUE_FLAG_PRIV) { 196 __be32 flags = *(__be32 *)(data + doffset); 197 198 doffset += GUE_LEN_PRIV; 199 200 if (flags & GUE_PFLAG_REMCSUM) { 201 guehdr = gue_remcsum(skb, guehdr, data + doffset, 202 hdrlen, guehdr->proto_ctype, 203 !!(fou->flags & 204 FOU_F_REMCSUM_NOPARTIAL)); 205 if (!guehdr) 206 goto drop; 207 208 data = &guehdr[1]; 209 210 doffset += GUE_PLEN_REMCSUM; 211 } 212 } 213 214 if (unlikely(guehdr->control)) 215 return gue_control_message(skb, guehdr); 216 217 proto_ctype = guehdr->proto_ctype; 218 if (unlikely(!proto_ctype)) 219 goto drop; 220 221 __skb_pull(skb, sizeof(struct udphdr) + hdrlen); 222 skb_reset_transport_header(skb); 223 224 if (iptunnel_pull_offloads(skb)) 225 goto drop; 226 227 return -proto_ctype; 228 229 drop: 230 kfree_skb(skb); 231 return 0; 232 } 233 234 static const struct net_offload *fou_gro_ops(const struct sock *sk, 235 int proto) 236 { 237 const struct net_offload __rcu **offloads; 238 239 /* FOU doesn't allow IPv4 on IPv6 sockets. */ 240 offloads = sk->sk_family == AF_INET6 ? inet6_offloads : inet_offloads; 241 return rcu_dereference(offloads[proto]); 242 } 243 244 static struct sk_buff *fou_gro_receive(struct sock *sk, 245 struct list_head *head, 246 struct sk_buff *skb) 247 { 248 struct fou *fou = fou_from_sock(sk); 249 const struct net_offload *ops; 250 struct sk_buff *pp = NULL; 251 252 if (!fou) 253 goto out; 254 255 /* We can clear the encap_mark for FOU as we are essentially doing 256 * one of two possible things. We are either adding an L4 tunnel 257 * header to the outer L3 tunnel header, or we are simply 258 * treating the GRE tunnel header as though it is a UDP protocol 259 * specific header such as VXLAN or GENEVE. 260 */ 261 NAPI_GRO_CB(skb)->encap_mark = 0; 262 263 /* Flag this frame as already having an outer encap header */ 264 NAPI_GRO_CB(skb)->is_fou = 1; 265 266 ops = fou_gro_ops(sk, fou->protocol); 267 if (!ops || !ops->callbacks.gro_receive) 268 goto out; 269 270 pp = call_gro_receive(ops->callbacks.gro_receive, head, skb); 271 272 out: 273 return pp; 274 } 275 276 static int fou_gro_complete(struct sock *sk, struct sk_buff *skb, 277 int nhoff) 278 { 279 struct fou *fou = fou_from_sock(sk); 280 const struct net_offload *ops; 281 int err; 282 283 if (!fou) { 284 err = -ENOENT; 285 goto out; 286 } 287 288 ops = fou_gro_ops(sk, fou->protocol); 289 if (WARN_ON(!ops || !ops->callbacks.gro_complete)) { 290 err = -ENOSYS; 291 goto out; 292 } 293 294 err = ops->callbacks.gro_complete(skb, nhoff); 295 296 skb_set_inner_mac_header(skb, nhoff); 297 298 out: 299 return err; 300 } 301 302 static struct guehdr *gue_gro_remcsum(struct sk_buff *skb, unsigned int off, 303 struct guehdr *guehdr, void *data, 304 size_t hdrlen, struct gro_remcsum *grc, 305 bool nopartial) 306 { 307 __be16 *pd = data; 308 size_t start = ntohs(pd[0]); 309 size_t offset = ntohs(pd[1]); 310 311 if (skb->remcsum_offload) 312 return guehdr; 313 314 if (!NAPI_GRO_CB(skb)->csum_valid) 315 return NULL; 316 317 guehdr = skb_gro_remcsum_process(skb, (void *)guehdr, off, hdrlen, 318 start, offset, grc, nopartial); 319 320 skb->remcsum_offload = 1; 321 322 return guehdr; 323 } 324 325 static struct sk_buff *gue_gro_receive(struct sock *sk, 326 struct list_head *head, 327 struct sk_buff *skb) 328 { 329 const struct net_offload *ops; 330 struct sk_buff *pp = NULL; 331 struct sk_buff *p; 332 struct guehdr *guehdr; 333 size_t len, optlen, hdrlen, off; 334 void *data; 335 u16 doffset = 0; 336 int flush = 1; 337 struct fou *fou = fou_from_sock(sk); 338 struct gro_remcsum grc; 339 u8 proto; 340 341 skb_gro_remcsum_init(&grc); 342 343 if (!fou) 344 goto out; 345 346 off = skb_gro_offset(skb); 347 len = off + sizeof(*guehdr); 348 349 guehdr = skb_gro_header(skb, len, off); 350 if (unlikely(!guehdr)) 351 goto out; 352 353 switch (guehdr->version) { 354 case 0: 355 break; 356 case 1: 357 switch (((struct iphdr *)guehdr)->version) { 358 case 4: 359 proto = IPPROTO_IPIP; 360 break; 361 case 6: 362 proto = IPPROTO_IPV6; 363 break; 364 default: 365 goto out; 366 } 367 goto next_proto; 368 default: 369 goto out; 370 } 371 372 optlen = guehdr->hlen << 2; 373 len += optlen; 374 375 if (!skb_gro_may_pull(skb, len)) { 376 guehdr = skb_gro_header_slow(skb, len, off); 377 if (unlikely(!guehdr)) 378 goto out; 379 } 380 381 if (unlikely(guehdr->control) || guehdr->version != 0 || 382 validate_gue_flags(guehdr, optlen)) 383 goto out; 384 385 hdrlen = sizeof(*guehdr) + optlen; 386 387 /* Adjust NAPI_GRO_CB(skb)->csum to account for guehdr, 388 * this is needed if there is a remote checkcsum offload. 389 */ 390 skb_gro_postpull_rcsum(skb, guehdr, hdrlen); 391 392 data = &guehdr[1]; 393 394 if (guehdr->flags & GUE_FLAG_PRIV) { 395 __be32 flags = *(__be32 *)(data + doffset); 396 397 doffset += GUE_LEN_PRIV; 398 399 if (flags & GUE_PFLAG_REMCSUM) { 400 guehdr = gue_gro_remcsum(skb, off, guehdr, 401 data + doffset, hdrlen, &grc, 402 !!(fou->flags & 403 FOU_F_REMCSUM_NOPARTIAL)); 404 405 if (!guehdr) 406 goto out; 407 408 data = &guehdr[1]; 409 410 doffset += GUE_PLEN_REMCSUM; 411 } 412 } 413 414 skb_gro_pull(skb, hdrlen); 415 416 list_for_each_entry(p, head, list) { 417 const struct guehdr *guehdr2; 418 419 if (!NAPI_GRO_CB(p)->same_flow) 420 continue; 421 422 guehdr2 = (struct guehdr *)(p->data + off); 423 424 /* Compare base GUE header to be equal (covers 425 * hlen, version, proto_ctype, and flags. 426 */ 427 if (guehdr->word != guehdr2->word) { 428 NAPI_GRO_CB(p)->same_flow = 0; 429 continue; 430 } 431 432 /* Compare optional fields are the same. */ 433 if (guehdr->hlen && memcmp(&guehdr[1], &guehdr2[1], 434 guehdr->hlen << 2)) { 435 NAPI_GRO_CB(p)->same_flow = 0; 436 continue; 437 } 438 } 439 440 proto = guehdr->proto_ctype; 441 442 next_proto: 443 444 /* We can clear the encap_mark for GUE as we are essentially doing 445 * one of two possible things. We are either adding an L4 tunnel 446 * header to the outer L3 tunnel header, or we are simply 447 * treating the GRE tunnel header as though it is a UDP protocol 448 * specific header such as VXLAN or GENEVE. 449 */ 450 NAPI_GRO_CB(skb)->encap_mark = 0; 451 452 /* Flag this frame as already having an outer encap header */ 453 NAPI_GRO_CB(skb)->is_fou = 1; 454 455 ops = fou_gro_ops(sk, proto); 456 if (!ops || !ops->callbacks.gro_receive) 457 goto out; 458 459 pp = call_gro_receive(ops->callbacks.gro_receive, head, skb); 460 flush = 0; 461 462 out: 463 skb_gro_flush_final_remcsum(skb, pp, flush, &grc); 464 465 return pp; 466 } 467 468 static int gue_gro_complete(struct sock *sk, struct sk_buff *skb, int nhoff) 469 { 470 struct guehdr *guehdr = (struct guehdr *)(skb->data + nhoff); 471 const struct net_offload *ops; 472 unsigned int guehlen = 0; 473 u8 proto; 474 int err = -ENOENT; 475 476 switch (guehdr->version) { 477 case 0: 478 proto = guehdr->proto_ctype; 479 guehlen = sizeof(*guehdr) + (guehdr->hlen << 2); 480 break; 481 case 1: 482 switch (((struct iphdr *)guehdr)->version) { 483 case 4: 484 proto = IPPROTO_IPIP; 485 break; 486 case 6: 487 proto = IPPROTO_IPV6; 488 break; 489 default: 490 return err; 491 } 492 break; 493 default: 494 return err; 495 } 496 497 ops = fou_gro_ops(sk, proto); 498 if (WARN_ON(!ops || !ops->callbacks.gro_complete)) 499 goto out; 500 501 err = ops->callbacks.gro_complete(skb, nhoff + guehlen); 502 503 skb_set_inner_mac_header(skb, nhoff + guehlen); 504 505 out: 506 return err; 507 } 508 509 static bool fou_cfg_cmp(struct fou *fou, struct fou_cfg *cfg) 510 { 511 struct udp_port_cfg *udp_cfg = &cfg->udp_config; 512 struct sock *sk = fou->sk; 513 514 if (fou->family != udp_cfg->family || 515 fou->port != udp_cfg->local_udp_port || 516 sk->sk_dport != udp_cfg->peer_udp_port || 517 sk->sk_bound_dev_if != udp_cfg->bind_ifindex) 518 return false; 519 520 if (fou->family == AF_INET) { 521 if (sk->sk_rcv_saddr != udp_cfg->local_ip.s_addr || 522 sk->sk_daddr != udp_cfg->peer_ip.s_addr) 523 return false; 524 else 525 return true; 526 #if IS_ENABLED(CONFIG_IPV6) 527 } else { 528 if (ipv6_addr_cmp(&sk->sk_v6_rcv_saddr, &udp_cfg->local_ip6) || 529 ipv6_addr_cmp(&sk->sk_v6_daddr, &udp_cfg->peer_ip6)) 530 return false; 531 else 532 return true; 533 #endif 534 } 535 536 return false; 537 } 538 539 static int fou_add_to_port_list(struct net *net, struct fou *fou, 540 struct fou_cfg *cfg) 541 { 542 struct fou_net *fn = net_generic(net, fou_net_id); 543 struct fou *fout; 544 545 mutex_lock(&fn->fou_lock); 546 list_for_each_entry(fout, &fn->fou_list, list) { 547 if (fou_cfg_cmp(fout, cfg)) { 548 mutex_unlock(&fn->fou_lock); 549 return -EALREADY; 550 } 551 } 552 553 list_add(&fou->list, &fn->fou_list); 554 mutex_unlock(&fn->fou_lock); 555 556 return 0; 557 } 558 559 static void fou_release(struct fou *fou) 560 { 561 list_del(&fou->list); 562 udp_tunnel_sock_release(fou->sk); 563 kfree_rcu(fou, rcu); 564 } 565 566 static int fou_create(struct net *net, struct fou_cfg *cfg, 567 struct socket **sockp) 568 { 569 struct socket *sock = NULL; 570 struct fou *fou = NULL; 571 struct sock *sk; 572 struct udp_tunnel_sock_cfg tunnel_cfg; 573 int err; 574 575 /* Open UDP socket */ 576 err = udp_sock_create(net, &cfg->udp_config, &sock); 577 if (err < 0) 578 goto error; 579 580 /* Allocate FOU port structure */ 581 fou = kzalloc_obj(*fou); 582 if (!fou) { 583 err = -ENOMEM; 584 goto error; 585 } 586 587 sk = sock->sk; 588 589 fou->port = cfg->udp_config.local_udp_port; 590 fou->family = cfg->udp_config.family; 591 fou->flags = cfg->flags; 592 fou->type = cfg->type; 593 fou->sk = sk; 594 595 memset(&tunnel_cfg, 0, sizeof(tunnel_cfg)); 596 tunnel_cfg.encap_type = 1; 597 tunnel_cfg.sk_user_data = fou; 598 tunnel_cfg.encap_destroy = NULL; 599 600 /* Initial for fou type */ 601 switch (cfg->type) { 602 case FOU_ENCAP_DIRECT: 603 tunnel_cfg.encap_rcv = fou_udp_recv; 604 tunnel_cfg.gro_receive = fou_gro_receive; 605 tunnel_cfg.gro_complete = fou_gro_complete; 606 fou->protocol = cfg->protocol; 607 break; 608 case FOU_ENCAP_GUE: 609 tunnel_cfg.encap_rcv = gue_udp_recv; 610 tunnel_cfg.gro_receive = gue_gro_receive; 611 tunnel_cfg.gro_complete = gue_gro_complete; 612 break; 613 default: 614 err = -EINVAL; 615 goto error; 616 } 617 618 setup_udp_tunnel_sock(net, sk, &tunnel_cfg); 619 620 sk->sk_allocation = GFP_ATOMIC; 621 622 err = fou_add_to_port_list(net, fou, cfg); 623 if (err) 624 goto error; 625 626 if (sockp) 627 *sockp = sock; 628 629 return 0; 630 631 error: 632 kfree(fou); 633 if (sock) 634 udp_tunnel_sock_release(sock->sk); 635 636 return err; 637 } 638 639 static int fou_destroy(struct net *net, struct fou_cfg *cfg) 640 { 641 struct fou_net *fn = net_generic(net, fou_net_id); 642 int err = -EINVAL; 643 struct fou *fou; 644 645 mutex_lock(&fn->fou_lock); 646 list_for_each_entry(fou, &fn->fou_list, list) { 647 if (fou_cfg_cmp(fou, cfg)) { 648 fou_release(fou); 649 err = 0; 650 break; 651 } 652 } 653 mutex_unlock(&fn->fou_lock); 654 655 return err; 656 } 657 658 static struct genl_family fou_nl_family; 659 660 static int parse_nl_config(struct genl_info *info, 661 struct fou_cfg *cfg) 662 { 663 bool has_local = false, has_peer = false; 664 struct nlattr *attr; 665 int ifindex; 666 __be16 port; 667 668 memset(cfg, 0, sizeof(*cfg)); 669 670 cfg->udp_config.family = AF_INET; 671 672 if (info->attrs[FOU_ATTR_AF]) { 673 u8 family = nla_get_u8(info->attrs[FOU_ATTR_AF]); 674 675 switch (family) { 676 case AF_INET: 677 break; 678 case AF_INET6: 679 cfg->udp_config.ipv6_v6only = 1; 680 break; 681 default: 682 return -EAFNOSUPPORT; 683 } 684 685 cfg->udp_config.family = family; 686 } 687 688 if (info->attrs[FOU_ATTR_PORT]) { 689 port = nla_get_be16(info->attrs[FOU_ATTR_PORT]); 690 cfg->udp_config.local_udp_port = port; 691 } 692 693 if (info->attrs[FOU_ATTR_IPPROTO]) 694 cfg->protocol = nla_get_u8(info->attrs[FOU_ATTR_IPPROTO]); 695 696 if (info->attrs[FOU_ATTR_TYPE]) 697 cfg->type = nla_get_u8(info->attrs[FOU_ATTR_TYPE]); 698 699 if (info->attrs[FOU_ATTR_REMCSUM_NOPARTIAL]) 700 cfg->flags |= FOU_F_REMCSUM_NOPARTIAL; 701 702 if (cfg->udp_config.family == AF_INET) { 703 if (info->attrs[FOU_ATTR_LOCAL_V4]) { 704 attr = info->attrs[FOU_ATTR_LOCAL_V4]; 705 cfg->udp_config.local_ip.s_addr = nla_get_in_addr(attr); 706 has_local = true; 707 } 708 709 if (info->attrs[FOU_ATTR_PEER_V4]) { 710 attr = info->attrs[FOU_ATTR_PEER_V4]; 711 cfg->udp_config.peer_ip.s_addr = nla_get_in_addr(attr); 712 has_peer = true; 713 } 714 #if IS_ENABLED(CONFIG_IPV6) 715 } else { 716 if (info->attrs[FOU_ATTR_LOCAL_V6]) { 717 attr = info->attrs[FOU_ATTR_LOCAL_V6]; 718 cfg->udp_config.local_ip6 = nla_get_in6_addr(attr); 719 has_local = true; 720 } 721 722 if (info->attrs[FOU_ATTR_PEER_V6]) { 723 attr = info->attrs[FOU_ATTR_PEER_V6]; 724 cfg->udp_config.peer_ip6 = nla_get_in6_addr(attr); 725 has_peer = true; 726 } 727 #endif 728 } 729 730 if (has_peer) { 731 if (info->attrs[FOU_ATTR_PEER_PORT]) { 732 port = nla_get_be16(info->attrs[FOU_ATTR_PEER_PORT]); 733 cfg->udp_config.peer_udp_port = port; 734 } else { 735 return -EINVAL; 736 } 737 } 738 739 if (info->attrs[FOU_ATTR_IFINDEX]) { 740 if (!has_local) 741 return -EINVAL; 742 743 ifindex = nla_get_s32(info->attrs[FOU_ATTR_IFINDEX]); 744 745 cfg->udp_config.bind_ifindex = ifindex; 746 } 747 748 return 0; 749 } 750 751 int fou_nl_add_doit(struct sk_buff *skb, struct genl_info *info) 752 { 753 struct net *net = genl_info_net(info); 754 struct fou_cfg cfg; 755 int err; 756 757 err = parse_nl_config(info, &cfg); 758 if (err) 759 return err; 760 761 return fou_create(net, &cfg, NULL); 762 } 763 764 int fou_nl_del_doit(struct sk_buff *skb, struct genl_info *info) 765 { 766 struct net *net = genl_info_net(info); 767 struct fou_cfg cfg; 768 int err; 769 770 err = parse_nl_config(info, &cfg); 771 if (err) 772 return err; 773 774 return fou_destroy(net, &cfg); 775 } 776 777 static int fou_fill_info(struct fou *fou, struct sk_buff *msg) 778 { 779 struct sock *sk = fou->sk; 780 781 if (nla_put_u8(msg, FOU_ATTR_AF, sk->sk_family) || 782 nla_put_be16(msg, FOU_ATTR_PORT, fou->port) || 783 nla_put_be16(msg, FOU_ATTR_PEER_PORT, sk->sk_dport) || 784 nla_put_u8(msg, FOU_ATTR_IPPROTO, fou->protocol) || 785 nla_put_u8(msg, FOU_ATTR_TYPE, fou->type) || 786 nla_put_s32(msg, FOU_ATTR_IFINDEX, sk->sk_bound_dev_if)) 787 return -1; 788 789 if (fou->flags & FOU_F_REMCSUM_NOPARTIAL) 790 if (nla_put_flag(msg, FOU_ATTR_REMCSUM_NOPARTIAL)) 791 return -1; 792 793 if (sk->sk_family == AF_INET) { 794 if (nla_put_in_addr(msg, FOU_ATTR_LOCAL_V4, sk->sk_rcv_saddr)) 795 return -1; 796 797 if (nla_put_in_addr(msg, FOU_ATTR_PEER_V4, sk->sk_daddr)) 798 return -1; 799 #if IS_ENABLED(CONFIG_IPV6) 800 } else { 801 if (nla_put_in6_addr(msg, FOU_ATTR_LOCAL_V6, 802 &sk->sk_v6_rcv_saddr)) 803 return -1; 804 805 if (nla_put_in6_addr(msg, FOU_ATTR_PEER_V6, &sk->sk_v6_daddr)) 806 return -1; 807 #endif 808 } 809 810 return 0; 811 } 812 813 static int fou_dump_info(struct fou *fou, u32 portid, u32 seq, 814 u32 flags, struct sk_buff *skb, u8 cmd) 815 { 816 void *hdr; 817 818 hdr = genlmsg_put(skb, portid, seq, &fou_nl_family, flags, cmd); 819 if (!hdr) 820 return -ENOMEM; 821 822 if (fou_fill_info(fou, skb) < 0) 823 goto nla_put_failure; 824 825 genlmsg_end(skb, hdr); 826 return 0; 827 828 nla_put_failure: 829 genlmsg_cancel(skb, hdr); 830 return -EMSGSIZE; 831 } 832 833 int fou_nl_get_doit(struct sk_buff *skb, struct genl_info *info) 834 { 835 struct net *net = genl_info_net(info); 836 struct fou_net *fn = net_generic(net, fou_net_id); 837 struct sk_buff *msg; 838 struct fou_cfg cfg; 839 struct fou *fout; 840 __be16 port; 841 u8 family; 842 int ret; 843 844 ret = parse_nl_config(info, &cfg); 845 if (ret) 846 return ret; 847 port = cfg.udp_config.local_udp_port; 848 if (port == 0) 849 return -EINVAL; 850 851 family = cfg.udp_config.family; 852 if (family != AF_INET && family != AF_INET6) 853 return -EINVAL; 854 855 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 856 if (!msg) 857 return -ENOMEM; 858 859 ret = -ESRCH; 860 mutex_lock(&fn->fou_lock); 861 list_for_each_entry(fout, &fn->fou_list, list) { 862 if (fou_cfg_cmp(fout, &cfg)) { 863 ret = fou_dump_info(fout, info->snd_portid, 864 info->snd_seq, 0, msg, 865 info->genlhdr->cmd); 866 break; 867 } 868 } 869 mutex_unlock(&fn->fou_lock); 870 if (ret < 0) 871 goto out_free; 872 873 return genlmsg_reply(msg, info); 874 875 out_free: 876 nlmsg_free(msg); 877 return ret; 878 } 879 880 int fou_nl_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 881 { 882 struct net *net = sock_net(skb->sk); 883 struct fou_net *fn = net_generic(net, fou_net_id); 884 struct fou *fout; 885 int idx = 0, ret; 886 887 mutex_lock(&fn->fou_lock); 888 list_for_each_entry(fout, &fn->fou_list, list) { 889 if (idx++ < cb->args[0]) 890 continue; 891 ret = fou_dump_info(fout, NETLINK_CB(cb->skb).portid, 892 cb->nlh->nlmsg_seq, NLM_F_MULTI, 893 skb, FOU_CMD_GET); 894 if (ret) 895 break; 896 } 897 mutex_unlock(&fn->fou_lock); 898 899 cb->args[0] = idx; 900 return skb->len; 901 } 902 903 static struct genl_family fou_nl_family __ro_after_init = { 904 .hdrsize = 0, 905 .name = FOU_GENL_NAME, 906 .version = FOU_GENL_VERSION, 907 .maxattr = FOU_ATTR_MAX, 908 .policy = fou_nl_policy, 909 .netnsok = true, 910 .module = THIS_MODULE, 911 .small_ops = fou_nl_ops, 912 .n_small_ops = ARRAY_SIZE(fou_nl_ops), 913 .resv_start_op = FOU_CMD_GET + 1, 914 }; 915 916 size_t fou_encap_hlen(struct ip_tunnel_encap *e) 917 { 918 return sizeof(struct udphdr); 919 } 920 EXPORT_SYMBOL(fou_encap_hlen); 921 922 size_t gue_encap_hlen(struct ip_tunnel_encap *e) 923 { 924 size_t len; 925 bool need_priv = false; 926 927 len = sizeof(struct udphdr) + sizeof(struct guehdr); 928 929 if (e->flags & TUNNEL_ENCAP_FLAG_REMCSUM) { 930 len += GUE_PLEN_REMCSUM; 931 need_priv = true; 932 } 933 934 len += need_priv ? GUE_LEN_PRIV : 0; 935 936 return len; 937 } 938 EXPORT_SYMBOL(gue_encap_hlen); 939 940 int __fou_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e, 941 u8 *protocol, __be16 *sport, int type) 942 { 943 int err; 944 945 err = iptunnel_handle_offloads(skb, type); 946 if (err) 947 return err; 948 949 *sport = e->sport ? : udp_flow_src_port(dev_net(skb->dev), 950 skb, 0, 0, false); 951 952 return 0; 953 } 954 EXPORT_SYMBOL(__fou_build_header); 955 956 int __gue_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e, 957 u8 *protocol, __be16 *sport, int type) 958 { 959 struct guehdr *guehdr; 960 size_t hdrlen, optlen = 0; 961 void *data; 962 bool need_priv = false; 963 int err; 964 965 if ((e->flags & TUNNEL_ENCAP_FLAG_REMCSUM) && 966 skb->ip_summed == CHECKSUM_PARTIAL) { 967 optlen += GUE_PLEN_REMCSUM; 968 type |= SKB_GSO_TUNNEL_REMCSUM; 969 need_priv = true; 970 } 971 972 optlen += need_priv ? GUE_LEN_PRIV : 0; 973 974 err = iptunnel_handle_offloads(skb, type); 975 if (err) 976 return err; 977 978 /* Get source port (based on flow hash) before skb_push */ 979 *sport = e->sport ? : udp_flow_src_port(dev_net(skb->dev), 980 skb, 0, 0, false); 981 982 hdrlen = sizeof(struct guehdr) + optlen; 983 984 skb_push(skb, hdrlen); 985 986 guehdr = (struct guehdr *)skb->data; 987 988 guehdr->control = 0; 989 guehdr->version = 0; 990 guehdr->hlen = optlen >> 2; 991 guehdr->flags = 0; 992 guehdr->proto_ctype = *protocol; 993 994 data = &guehdr[1]; 995 996 if (need_priv) { 997 __be32 *flags = data; 998 999 guehdr->flags |= GUE_FLAG_PRIV; 1000 *flags = 0; 1001 data += GUE_LEN_PRIV; 1002 1003 if (type & SKB_GSO_TUNNEL_REMCSUM) { 1004 u16 csum_start = skb_checksum_start_offset(skb); 1005 __be16 *pd = data; 1006 1007 if (csum_start < hdrlen) 1008 return -EINVAL; 1009 1010 csum_start -= hdrlen; 1011 pd[0] = htons(csum_start); 1012 pd[1] = htons(csum_start + skb->csum_offset); 1013 1014 if (!skb_is_gso(skb)) { 1015 skb->ip_summed = CHECKSUM_NONE; 1016 skb->encapsulation = 0; 1017 } 1018 1019 *flags |= GUE_PFLAG_REMCSUM; 1020 data += GUE_PLEN_REMCSUM; 1021 } 1022 1023 } 1024 1025 return 0; 1026 } 1027 EXPORT_SYMBOL(__gue_build_header); 1028 1029 #ifdef CONFIG_NET_FOU_IP_TUNNELS 1030 1031 static void fou_build_udp(struct sk_buff *skb, struct ip_tunnel_encap *e, 1032 struct flowi4 *fl4, u8 *protocol, __be16 sport) 1033 { 1034 struct udphdr *uh; 1035 1036 skb_push(skb, sizeof(struct udphdr)); 1037 skb_reset_transport_header(skb); 1038 1039 uh = udp_hdr(skb); 1040 1041 uh->dest = e->dport; 1042 uh->source = sport; 1043 uh->len = htons(skb->len); 1044 udp_set_csum(!(e->flags & TUNNEL_ENCAP_FLAG_CSUM), skb, 1045 fl4->saddr, fl4->daddr, skb->len); 1046 1047 *protocol = IPPROTO_UDP; 1048 } 1049 1050 static int fou_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e, 1051 u8 *protocol, struct flowi4 *fl4) 1052 { 1053 int type = e->flags & TUNNEL_ENCAP_FLAG_CSUM ? SKB_GSO_UDP_TUNNEL_CSUM : 1054 SKB_GSO_UDP_TUNNEL; 1055 __be16 sport; 1056 int err; 1057 1058 err = __fou_build_header(skb, e, protocol, &sport, type); 1059 if (err) 1060 return err; 1061 1062 fou_build_udp(skb, e, fl4, protocol, sport); 1063 1064 return 0; 1065 } 1066 1067 static int gue_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e, 1068 u8 *protocol, struct flowi4 *fl4) 1069 { 1070 int type = e->flags & TUNNEL_ENCAP_FLAG_CSUM ? SKB_GSO_UDP_TUNNEL_CSUM : 1071 SKB_GSO_UDP_TUNNEL; 1072 __be16 sport; 1073 int err; 1074 1075 err = __gue_build_header(skb, e, protocol, &sport, type); 1076 if (err) 1077 return err; 1078 1079 fou_build_udp(skb, e, fl4, protocol, sport); 1080 1081 return 0; 1082 } 1083 1084 static int gue_err_proto_handler(int proto, struct sk_buff *skb, u32 info) 1085 { 1086 const struct net_protocol *ipprot = rcu_dereference(inet_protos[proto]); 1087 1088 if (ipprot && ipprot->err_handler) { 1089 if (!ipprot->err_handler(skb, info)) 1090 return 0; 1091 } 1092 1093 return -ENOENT; 1094 } 1095 1096 static int gue_err(struct sk_buff *skb, u32 info) 1097 { 1098 int transport_offset = skb_transport_offset(skb); 1099 struct guehdr *guehdr; 1100 size_t len, optlen; 1101 int ret; 1102 1103 len = sizeof(struct udphdr) + sizeof(struct guehdr); 1104 if (!pskb_may_pull(skb, transport_offset + len)) 1105 return -EINVAL; 1106 1107 guehdr = (struct guehdr *)&udp_hdr(skb)[1]; 1108 1109 switch (guehdr->version) { 1110 case 0: /* Full GUE header present */ 1111 break; 1112 case 1: { 1113 /* Direct encapsulation of IPv4 or IPv6 */ 1114 skb_set_transport_header(skb, -(int)sizeof(struct icmphdr)); 1115 1116 switch (((struct iphdr *)guehdr)->version) { 1117 case 4: 1118 ret = gue_err_proto_handler(IPPROTO_IPIP, skb, info); 1119 goto out; 1120 #if IS_ENABLED(CONFIG_IPV6) 1121 case 6: 1122 ret = gue_err_proto_handler(IPPROTO_IPV6, skb, info); 1123 goto out; 1124 #endif 1125 default: 1126 ret = -EOPNOTSUPP; 1127 goto out; 1128 } 1129 } 1130 default: /* Undefined version */ 1131 return -EOPNOTSUPP; 1132 } 1133 1134 if (guehdr->control) 1135 return -ENOENT; 1136 1137 optlen = guehdr->hlen << 2; 1138 1139 if (!pskb_may_pull(skb, transport_offset + len + optlen)) 1140 return -EINVAL; 1141 1142 guehdr = (struct guehdr *)&udp_hdr(skb)[1]; 1143 if (validate_gue_flags(guehdr, optlen)) 1144 return -EINVAL; 1145 1146 /* Handling exceptions for direct UDP encapsulation in GUE would lead to 1147 * recursion. Besides, this kind of encapsulation can't even be 1148 * configured currently. Discard this. 1149 */ 1150 if (guehdr->proto_ctype == IPPROTO_UDP) 1151 return -EOPNOTSUPP; 1152 1153 skb_set_transport_header(skb, -(int)sizeof(struct icmphdr)); 1154 ret = gue_err_proto_handler(guehdr->proto_ctype, skb, info); 1155 1156 out: 1157 skb_set_transport_header(skb, transport_offset); 1158 return ret; 1159 } 1160 1161 1162 static const struct ip_tunnel_encap_ops fou_iptun_ops = { 1163 .encap_hlen = fou_encap_hlen, 1164 .build_header = fou_build_header, 1165 .err_handler = gue_err, 1166 }; 1167 1168 static const struct ip_tunnel_encap_ops gue_iptun_ops = { 1169 .encap_hlen = gue_encap_hlen, 1170 .build_header = gue_build_header, 1171 .err_handler = gue_err, 1172 }; 1173 1174 static int ip_tunnel_encap_add_fou_ops(void) 1175 { 1176 int ret; 1177 1178 ret = ip_tunnel_encap_add_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU); 1179 if (ret < 0) { 1180 pr_err("can't add fou ops\n"); 1181 return ret; 1182 } 1183 1184 ret = ip_tunnel_encap_add_ops(&gue_iptun_ops, TUNNEL_ENCAP_GUE); 1185 if (ret < 0) { 1186 pr_err("can't add gue ops\n"); 1187 ip_tunnel_encap_del_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU); 1188 return ret; 1189 } 1190 1191 return 0; 1192 } 1193 1194 static void ip_tunnel_encap_del_fou_ops(void) 1195 { 1196 ip_tunnel_encap_del_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU); 1197 ip_tunnel_encap_del_ops(&gue_iptun_ops, TUNNEL_ENCAP_GUE); 1198 } 1199 1200 #else 1201 1202 static int ip_tunnel_encap_add_fou_ops(void) 1203 { 1204 return 0; 1205 } 1206 1207 static void ip_tunnel_encap_del_fou_ops(void) 1208 { 1209 } 1210 1211 #endif 1212 1213 static __net_init int fou_init_net(struct net *net) 1214 { 1215 struct fou_net *fn = net_generic(net, fou_net_id); 1216 1217 INIT_LIST_HEAD(&fn->fou_list); 1218 mutex_init(&fn->fou_lock); 1219 return 0; 1220 } 1221 1222 static __net_exit void fou_exit_net(struct net *net) 1223 { 1224 struct fou_net *fn = net_generic(net, fou_net_id); 1225 struct fou *fou, *next; 1226 1227 /* Close all the FOU sockets */ 1228 mutex_lock(&fn->fou_lock); 1229 list_for_each_entry_safe(fou, next, &fn->fou_list, list) 1230 fou_release(fou); 1231 mutex_unlock(&fn->fou_lock); 1232 } 1233 1234 static struct pernet_operations fou_net_ops = { 1235 .init = fou_init_net, 1236 .exit = fou_exit_net, 1237 .id = &fou_net_id, 1238 .size = sizeof(struct fou_net), 1239 }; 1240 1241 static int __init fou_init(void) 1242 { 1243 int ret; 1244 1245 ret = register_pernet_device(&fou_net_ops); 1246 if (ret) 1247 goto exit; 1248 1249 ret = genl_register_family(&fou_nl_family); 1250 if (ret < 0) 1251 goto unregister; 1252 1253 ret = register_fou_bpf(); 1254 if (ret < 0) 1255 goto kfunc_failed; 1256 1257 ret = ip_tunnel_encap_add_fou_ops(); 1258 if (ret == 0) 1259 return 0; 1260 1261 kfunc_failed: 1262 genl_unregister_family(&fou_nl_family); 1263 unregister: 1264 unregister_pernet_device(&fou_net_ops); 1265 exit: 1266 return ret; 1267 } 1268 1269 static void __exit fou_fini(void) 1270 { 1271 ip_tunnel_encap_del_fou_ops(); 1272 genl_unregister_family(&fou_nl_family); 1273 unregister_pernet_device(&fou_net_ops); 1274 } 1275 1276 module_init(fou_init); 1277 module_exit(fou_fini); 1278 MODULE_AUTHOR("Tom Herbert <therbert@google.com>"); 1279 MODULE_LICENSE("GPL"); 1280 MODULE_DESCRIPTION("Foo over UDP"); 1281