1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __NET_IP_TUNNELS_H 3 #define __NET_IP_TUNNELS_H 1 4 5 #include <linux/if_tunnel.h> 6 #include <linux/netdevice.h> 7 #include <linux/skbuff.h> 8 #include <linux/socket.h> 9 #include <linux/types.h> 10 #include <linux/u64_stats_sync.h> 11 #include <linux/bitops.h> 12 13 #include <net/dsfield.h> 14 #include <net/flow.h> 15 #include <net/gro_cells.h> 16 #include <net/inet_dscp.h> 17 #include <net/inet_ecn.h> 18 #include <net/netns/generic.h> 19 #include <net/rtnetlink.h> 20 #include <net/lwtunnel.h> 21 #include <net/dst_cache.h> 22 #include <net/netdev_lock.h> 23 24 #if IS_ENABLED(CONFIG_IPV6) 25 #include <net/ipv6.h> 26 #include <net/ip6_fib.h> 27 #include <net/ip6_route.h> 28 #endif 29 30 /* Recursion limit for tunnel xmit to detect routing loops. 31 * Unlike XMIT_RECURSION_LIMIT (8) used in the no-qdisc path, tunnel 32 * recursion involves route lookups and full IP output, consuming much 33 * more stack per level, so a lower limit is needed. 34 */ 35 #define IP_TUNNEL_RECURSION_LIMIT 5 36 37 /* Keep error state on tunnel for 30 sec */ 38 #define IPTUNNEL_ERR_TIMEO (30*HZ) 39 40 /* Used to memset ip_tunnel padding. */ 41 #define IP_TUNNEL_KEY_SIZE offsetofend(struct ip_tunnel_key, tp_dst) 42 43 /* Used to memset ipv4 address padding. */ 44 #define IP_TUNNEL_KEY_IPV4_PAD offsetofend(struct ip_tunnel_key, u.ipv4.dst) 45 #define IP_TUNNEL_KEY_IPV4_PAD_LEN \ 46 (sizeof_field(struct ip_tunnel_key, u) - \ 47 sizeof_field(struct ip_tunnel_key, u.ipv4)) 48 49 #define __ipt_flag_op(op, ...) \ 50 op(__VA_ARGS__, __IP_TUNNEL_FLAG_NUM) 51 52 #define IP_TUNNEL_DECLARE_FLAGS(...) \ 53 __ipt_flag_op(DECLARE_BITMAP, __VA_ARGS__) 54 55 #define ip_tunnel_flags_zero(...) __ipt_flag_op(bitmap_zero, __VA_ARGS__) 56 #define ip_tunnel_flags_copy(...) __ipt_flag_op(bitmap_copy, __VA_ARGS__) 57 #define ip_tunnel_flags_and(...) __ipt_flag_op(bitmap_and, __VA_ARGS__) 58 #define ip_tunnel_flags_or(...) __ipt_flag_op(bitmap_or, __VA_ARGS__) 59 60 #define ip_tunnel_flags_empty(...) \ 61 __ipt_flag_op(bitmap_empty, __VA_ARGS__) 62 #define ip_tunnel_flags_intersect(...) \ 63 __ipt_flag_op(bitmap_intersects, __VA_ARGS__) 64 #define ip_tunnel_flags_subset(...) \ 65 __ipt_flag_op(bitmap_subset, __VA_ARGS__) 66 67 struct ip_tunnel_key { 68 __be64 tun_id; 69 union { 70 struct { 71 __be32 src; 72 __be32 dst; 73 } ipv4; 74 struct { 75 struct in6_addr src; 76 struct in6_addr dst; 77 } ipv6; 78 } u; 79 IP_TUNNEL_DECLARE_FLAGS(tun_flags); 80 __be32 label; /* Flow Label for IPv6 */ 81 u32 nhid; 82 u8 tos; /* TOS for IPv4, TC for IPv6 */ 83 u8 ttl; /* TTL for IPv4, HL for IPv6 */ 84 __be16 tp_src; 85 __be16 tp_dst; 86 __u8 flow_flags; 87 }; 88 89 struct ip_tunnel_encap { 90 u16 type; 91 u16 flags; 92 __be16 sport; 93 __be16 dport; 94 }; 95 96 /* Flags for ip_tunnel_info mode. */ 97 #define IP_TUNNEL_INFO_TX 0x01 /* represents tx tunnel parameters */ 98 #define IP_TUNNEL_INFO_IPV6 0x02 /* key contains IPv6 addresses */ 99 #define IP_TUNNEL_INFO_BRIDGE 0x04 /* represents a bridged tunnel id */ 100 101 /* Maximum tunnel options length. */ 102 #define IP_TUNNEL_OPTS_MAX \ 103 GENMASK((sizeof_field(struct ip_tunnel_info, \ 104 options_len) * BITS_PER_BYTE) - 1, 0) 105 106 #define ip_tunnel_info_opts(info) \ 107 _Generic(info, \ 108 const struct ip_tunnel_info * : ((const void *)(info)->options),\ 109 struct ip_tunnel_info * : ((void *)(info)->options)\ 110 ) 111 112 struct ip_tunnel_info { 113 struct ip_tunnel_key key; 114 struct ip_tunnel_encap encap; 115 #ifdef CONFIG_DST_CACHE 116 struct dst_cache dst_cache; 117 #endif 118 u8 options_len; 119 u8 mode; 120 u8 options[] __aligned_largest __counted_by(options_len); 121 }; 122 123 /* 6rd prefix/relay information */ 124 #ifdef CONFIG_IPV6_SIT_6RD 125 struct ip_tunnel_6rd_parm { 126 struct in6_addr prefix; 127 __be32 relay_prefix; 128 u16 prefixlen; 129 u16 relay_prefixlen; 130 }; 131 #endif 132 133 struct ip_tunnel_prl_entry { 134 struct ip_tunnel_prl_entry __rcu *next; 135 __be32 addr; 136 u16 flags; 137 struct rcu_head rcu_head; 138 }; 139 140 struct metadata_dst; 141 142 /* Kernel-side variant of ip_tunnel_parm */ 143 struct ip_tunnel_parm_kern { 144 char name[IFNAMSIZ]; 145 IP_TUNNEL_DECLARE_FLAGS(i_flags); 146 IP_TUNNEL_DECLARE_FLAGS(o_flags); 147 __be32 i_key; 148 __be32 o_key; 149 int link; 150 struct iphdr iph; 151 }; 152 153 struct ip_tunnel { 154 struct ip_tunnel __rcu *next; 155 struct hlist_node hash_node; 156 157 struct net_device *dev; 158 netdevice_tracker dev_tracker; 159 160 struct net *net; /* netns for packet i/o */ 161 162 unsigned long err_time; /* Time when the last ICMP error 163 * arrived */ 164 int err_count; /* Number of arrived ICMP errors */ 165 166 /* These four fields used only by GRE */ 167 u32 i_seqno; /* The last seen seqno */ 168 atomic_t o_seqno; /* The last output seqno */ 169 int tun_hlen; /* Precalculated header length */ 170 171 /* These four fields used only by ERSPAN */ 172 u32 index; /* ERSPAN type II index */ 173 u8 erspan_ver; /* ERSPAN version */ 174 u8 dir; /* ERSPAN direction */ 175 u16 hwid; /* ERSPAN hardware ID */ 176 177 struct dst_cache dst_cache; 178 179 struct ip_tunnel_parm_kern parms; 180 181 int mlink; 182 int encap_hlen; /* Encap header length (FOU,GUE) */ 183 int hlen; /* tun_hlen + encap_hlen */ 184 struct ip_tunnel_encap encap; 185 186 /* for SIT */ 187 #ifdef CONFIG_IPV6_SIT_6RD 188 struct ip_tunnel_6rd_parm ip6rd; 189 #endif 190 struct ip_tunnel_prl_entry __rcu *prl; /* potential router list */ 191 unsigned int prl_count; /* # of entries in PRL */ 192 unsigned int ip_tnl_net_id; 193 struct gro_cells gro_cells; 194 __u32 fwmark; 195 bool collect_md; 196 bool ignore_df; 197 }; 198 199 struct tnl_ptk_info { 200 IP_TUNNEL_DECLARE_FLAGS(flags); 201 __be16 proto; 202 __be32 key; 203 __be32 seq; 204 int hdr_len; 205 }; 206 207 #define PACKET_RCVD 0 208 #define PACKET_REJECT 1 209 #define PACKET_NEXT 2 210 211 #define IP_TNL_HASH_BITS 7 212 #define IP_TNL_HASH_SIZE (1 << IP_TNL_HASH_BITS) 213 214 struct ip_tunnel_net { 215 struct net_device *fb_tunnel_dev; 216 struct rtnl_link_ops *rtnl_link_ops; 217 struct hlist_head tunnels[IP_TNL_HASH_SIZE]; 218 struct ip_tunnel __rcu *collect_md_tun; 219 int type; 220 }; 221 222 static inline void ip_tunnel_set_options_present(unsigned long *flags) 223 { 224 IP_TUNNEL_DECLARE_FLAGS(present) = { }; 225 226 __set_bit(IP_TUNNEL_GENEVE_OPT_BIT, present); 227 __set_bit(IP_TUNNEL_VXLAN_OPT_BIT, present); 228 __set_bit(IP_TUNNEL_ERSPAN_OPT_BIT, present); 229 __set_bit(IP_TUNNEL_GTP_OPT_BIT, present); 230 __set_bit(IP_TUNNEL_PFCP_OPT_BIT, present); 231 232 ip_tunnel_flags_or(flags, flags, present); 233 } 234 235 static inline void ip_tunnel_clear_options_present(unsigned long *flags) 236 { 237 IP_TUNNEL_DECLARE_FLAGS(present) = { }; 238 239 __set_bit(IP_TUNNEL_GENEVE_OPT_BIT, present); 240 __set_bit(IP_TUNNEL_VXLAN_OPT_BIT, present); 241 __set_bit(IP_TUNNEL_ERSPAN_OPT_BIT, present); 242 __set_bit(IP_TUNNEL_GTP_OPT_BIT, present); 243 __set_bit(IP_TUNNEL_PFCP_OPT_BIT, present); 244 245 __ipt_flag_op(bitmap_andnot, flags, flags, present); 246 } 247 248 static inline bool ip_tunnel_is_options_present(const unsigned long *flags) 249 { 250 IP_TUNNEL_DECLARE_FLAGS(present) = { }; 251 252 __set_bit(IP_TUNNEL_GENEVE_OPT_BIT, present); 253 __set_bit(IP_TUNNEL_VXLAN_OPT_BIT, present); 254 __set_bit(IP_TUNNEL_ERSPAN_OPT_BIT, present); 255 __set_bit(IP_TUNNEL_GTP_OPT_BIT, present); 256 __set_bit(IP_TUNNEL_PFCP_OPT_BIT, present); 257 258 return ip_tunnel_flags_intersect(flags, present); 259 } 260 261 static inline bool ip_tunnel_flags_is_be16_compat(const unsigned long *flags) 262 { 263 IP_TUNNEL_DECLARE_FLAGS(supp) = { }; 264 265 bitmap_set(supp, 0, BITS_PER_TYPE(__be16)); 266 __set_bit(IP_TUNNEL_VTI_BIT, supp); 267 268 return ip_tunnel_flags_subset(flags, supp); 269 } 270 271 static inline void ip_tunnel_flags_from_be16(unsigned long *dst, __be16 flags) 272 { 273 ip_tunnel_flags_zero(dst); 274 275 bitmap_write(dst, be16_to_cpu(flags), 0, BITS_PER_TYPE(__be16)); 276 __assign_bit(IP_TUNNEL_VTI_BIT, dst, flags & VTI_ISVTI); 277 } 278 279 static inline __be16 ip_tunnel_flags_to_be16(const unsigned long *flags) 280 { 281 __be16 ret; 282 283 ret = cpu_to_be16(bitmap_read(flags, 0, BITS_PER_TYPE(__be16))); 284 if (test_bit(IP_TUNNEL_VTI_BIT, flags)) 285 ret |= VTI_ISVTI; 286 287 return ret; 288 } 289 290 static inline void ip_tunnel_key_init(struct ip_tunnel_key *key, 291 __be32 saddr, __be32 daddr, 292 u8 tos, u8 ttl, __be32 label, 293 __be16 tp_src, __be16 tp_dst, 294 __be64 tun_id, 295 const unsigned long *tun_flags) 296 { 297 key->tun_id = tun_id; 298 key->u.ipv4.src = saddr; 299 key->u.ipv4.dst = daddr; 300 memset((unsigned char *)key + IP_TUNNEL_KEY_IPV4_PAD, 301 0, IP_TUNNEL_KEY_IPV4_PAD_LEN); 302 key->tos = tos; 303 key->ttl = ttl; 304 key->label = label; 305 ip_tunnel_flags_copy(key->tun_flags, tun_flags); 306 307 /* For the tunnel types on the top of IPsec, the tp_src and tp_dst of 308 * the upper tunnel are used. 309 * E.g: GRE over IPSEC, the tp_src and tp_port are zero. 310 */ 311 key->tp_src = tp_src; 312 key->tp_dst = tp_dst; 313 314 /* Clear struct padding. */ 315 if (sizeof(*key) != IP_TUNNEL_KEY_SIZE) 316 memset((unsigned char *)key + IP_TUNNEL_KEY_SIZE, 317 0, sizeof(*key) - IP_TUNNEL_KEY_SIZE); 318 } 319 320 static inline bool 321 ip_tunnel_dst_cache_usable(const struct sk_buff *skb, 322 const struct ip_tunnel_info *info) 323 { 324 if (skb->mark) 325 return false; 326 327 return !info || !test_bit(IP_TUNNEL_NOCACHE_BIT, info->key.tun_flags); 328 } 329 330 static inline unsigned short ip_tunnel_info_af(const struct ip_tunnel_info 331 *tun_info) 332 { 333 return tun_info->mode & IP_TUNNEL_INFO_IPV6 ? AF_INET6 : AF_INET; 334 } 335 336 static inline __be64 key32_to_tunnel_id(__be32 key) 337 { 338 #ifdef __BIG_ENDIAN 339 return (__force __be64)key; 340 #else 341 return (__force __be64)((__force u64)key << 32); 342 #endif 343 } 344 345 /* Returns the least-significant 32 bits of a __be64. */ 346 static inline __be32 tunnel_id_to_key32(__be64 tun_id) 347 { 348 #ifdef __BIG_ENDIAN 349 return (__force __be32)tun_id; 350 #else 351 return (__force __be32)((__force u64)tun_id >> 32); 352 #endif 353 } 354 355 #ifdef CONFIG_INET 356 357 static inline void ip_tunnel_init_flow(struct flowi4 *fl4, 358 int proto, 359 __be32 daddr, __be32 saddr, 360 __be32 key, __u8 tos, 361 struct net *net, int oif, 362 __u32 mark, __u32 tun_inner_hash, 363 __u8 flow_flags) 364 { 365 memset(fl4, 0, sizeof(*fl4)); 366 367 if (oif) { 368 fl4->flowi4_l3mdev = l3mdev_master_upper_ifindex_by_index(net, oif); 369 /* Legacy VRF/l3mdev use case */ 370 fl4->flowi4_oif = fl4->flowi4_l3mdev ? 0 : oif; 371 } 372 373 fl4->daddr = daddr; 374 fl4->saddr = saddr; 375 fl4->flowi4_dscp = inet_dsfield_to_dscp(tos); 376 fl4->flowi4_proto = proto; 377 fl4->fl4_gre_key = key; 378 fl4->flowi4_mark = mark; 379 fl4->flowi4_multipath_hash = tun_inner_hash; 380 fl4->flowi4_flags = flow_flags; 381 } 382 383 int __ip_tunnel_init(struct net_device *dev); 384 #define ip_tunnel_init(DEV) \ 385 ({ \ 386 struct net_device *__dev = (DEV); \ 387 int __res = __ip_tunnel_init(__dev); \ 388 \ 389 if (!__res) \ 390 netdev_lockdep_set_classes(__dev);\ 391 __res; \ 392 }) 393 394 void ip_tunnel_uninit(struct net_device *dev); 395 void ip_tunnel_dellink(struct net_device *dev, struct list_head *head); 396 struct net *ip_tunnel_get_link_net(const struct net_device *dev); 397 int ip_tunnel_get_iflink(const struct net_device *dev); 398 int ip_tunnel_init_net(struct net *net, unsigned int ip_tnl_net_id, 399 struct rtnl_link_ops *ops, char *devname); 400 void ip_tunnel_delete_net(struct net *net, unsigned int id, 401 struct rtnl_link_ops *ops, 402 struct list_head *dev_to_kill); 403 404 void ip_tunnel_xmit(struct sk_buff *skb, struct net_device *dev, 405 const struct iphdr *tnl_params, const u8 protocol); 406 void ip_md_tunnel_xmit(struct sk_buff *skb, struct net_device *dev, 407 const u8 proto, int tunnel_hlen); 408 int ip_tunnel_ctl(struct net_device *dev, struct ip_tunnel_parm_kern *p, 409 int cmd); 410 bool ip_tunnel_parm_from_user(struct ip_tunnel_parm_kern *kp, 411 const void __user *data); 412 bool ip_tunnel_parm_to_user(void __user *data, struct ip_tunnel_parm_kern *kp); 413 int ip_tunnel_siocdevprivate(struct net_device *dev, struct ifreq *ifr, 414 void __user *data, int cmd); 415 int ip_tunnel_change_mtu(struct net_device *dev, int new_mtu); 416 417 struct ip_tunnel *ip_tunnel_lookup(struct ip_tunnel_net *itn, 418 int link, const unsigned long *flags, 419 __be32 remote, __be32 local, 420 __be32 key); 421 422 void ip_tunnel_md_udp_encap(struct sk_buff *skb, struct ip_tunnel_info *info); 423 int ip_tunnel_rcv(struct ip_tunnel *tunnel, struct sk_buff *skb, 424 const struct tnl_ptk_info *tpi, struct metadata_dst *tun_dst, 425 bool log_ecn_error); 426 int ip_tunnel_changelink(struct net_device *dev, struct nlattr *tb[], 427 struct ip_tunnel_parm_kern *p, __u32 fwmark); 428 int ip_tunnel_newlink(struct net *net, struct net_device *dev, 429 struct nlattr *tb[], struct ip_tunnel_parm_kern *p, 430 __u32 fwmark); 431 void ip_tunnel_setup(struct net_device *dev, unsigned int net_id); 432 433 bool ip_tunnel_netlink_encap_parms(struct nlattr *data[], 434 struct ip_tunnel_encap *encap); 435 436 void ip_tunnel_netlink_parms(struct nlattr *data[], 437 struct ip_tunnel_parm_kern *parms); 438 439 extern const struct header_ops ip_tunnel_header_ops; 440 __be16 ip_tunnel_parse_protocol(const struct sk_buff *skb); 441 442 struct ip_tunnel_encap_ops { 443 size_t (*encap_hlen)(struct ip_tunnel_encap *e); 444 int (*build_header)(struct sk_buff *skb, struct ip_tunnel_encap *e, 445 u8 *protocol, struct flowi4 *fl4); 446 int (*err_handler)(struct sk_buff *skb, u32 info); 447 }; 448 449 #define MAX_IPTUN_ENCAP_OPS 8 450 451 extern const struct ip_tunnel_encap_ops __rcu * 452 iptun_encaps[MAX_IPTUN_ENCAP_OPS]; 453 454 int ip_tunnel_encap_add_ops(const struct ip_tunnel_encap_ops *op, 455 unsigned int num); 456 int ip_tunnel_encap_del_ops(const struct ip_tunnel_encap_ops *op, 457 unsigned int num); 458 459 int ip_tunnel_encap_setup(struct ip_tunnel *t, 460 struct ip_tunnel_encap *ipencap); 461 462 static inline enum skb_drop_reason 463 pskb_inet_may_pull_reason(struct sk_buff *skb) 464 { 465 int nhlen; 466 467 switch (skb->protocol) { 468 #if IS_ENABLED(CONFIG_IPV6) 469 case htons(ETH_P_IPV6): 470 nhlen = sizeof(struct ipv6hdr); 471 break; 472 #endif 473 case htons(ETH_P_IP): 474 nhlen = sizeof(struct iphdr); 475 break; 476 default: 477 nhlen = 0; 478 } 479 480 return pskb_network_may_pull_reason(skb, nhlen); 481 } 482 483 static inline bool pskb_inet_may_pull(struct sk_buff *skb) 484 { 485 return pskb_inet_may_pull_reason(skb) == SKB_NOT_DROPPED_YET; 486 } 487 488 /* Variant of pskb_inet_may_pull(). 489 */ 490 static inline enum skb_drop_reason 491 skb_vlan_inet_prepare(struct sk_buff *skb, bool inner_proto_inherit) 492 { 493 int nhlen = 0, maclen = inner_proto_inherit ? 0 : ETH_HLEN; 494 __be16 type = skb->protocol; 495 enum skb_drop_reason reason; 496 497 /* Essentially this is skb_protocol(skb, true) 498 * And we get MAC len. 499 */ 500 if (eth_type_vlan(type)) 501 type = __vlan_get_protocol(skb, type, &maclen); 502 503 switch (type) { 504 #if IS_ENABLED(CONFIG_IPV6) 505 case htons(ETH_P_IPV6): 506 nhlen = sizeof(struct ipv6hdr); 507 break; 508 #endif 509 case htons(ETH_P_IP): 510 nhlen = sizeof(struct iphdr); 511 break; 512 } 513 /* For ETH_P_IPV6/ETH_P_IP we make sure to pull 514 * a base network header in skb->head. 515 */ 516 reason = pskb_may_pull_reason(skb, maclen + nhlen); 517 if (reason) 518 return reason; 519 520 skb_set_network_header(skb, maclen); 521 522 return SKB_NOT_DROPPED_YET; 523 } 524 525 static inline int ip_encap_hlen(struct ip_tunnel_encap *e) 526 { 527 const struct ip_tunnel_encap_ops *ops; 528 int hlen = -EINVAL; 529 530 if (e->type == TUNNEL_ENCAP_NONE) 531 return 0; 532 533 if (e->type >= MAX_IPTUN_ENCAP_OPS) 534 return -EINVAL; 535 536 rcu_read_lock(); 537 ops = rcu_dereference(iptun_encaps[e->type]); 538 if (likely(ops && ops->encap_hlen)) 539 hlen = ops->encap_hlen(e); 540 rcu_read_unlock(); 541 542 return hlen; 543 } 544 545 static inline int ip_tunnel_encap(struct sk_buff *skb, 546 struct ip_tunnel_encap *e, 547 u8 *protocol, struct flowi4 *fl4) 548 { 549 const struct ip_tunnel_encap_ops *ops; 550 int ret = -EINVAL; 551 552 if (e->type == TUNNEL_ENCAP_NONE) 553 return 0; 554 555 if (e->type >= MAX_IPTUN_ENCAP_OPS) 556 return -EINVAL; 557 558 rcu_read_lock(); 559 ops = rcu_dereference(iptun_encaps[e->type]); 560 if (likely(ops && ops->build_header)) 561 ret = ops->build_header(skb, e, protocol, fl4); 562 rcu_read_unlock(); 563 564 return ret; 565 } 566 567 /* Extract dsfield from inner protocol */ 568 static inline u8 ip_tunnel_get_dsfield(const struct iphdr *iph, 569 const struct sk_buff *skb) 570 { 571 __be16 payload_protocol = skb_protocol(skb, true); 572 573 if (payload_protocol == htons(ETH_P_IP)) 574 return iph->tos; 575 else if (payload_protocol == htons(ETH_P_IPV6)) 576 return ipv6_get_dsfield((const struct ipv6hdr *)iph); 577 else 578 return 0; 579 } 580 581 static inline __be32 ip_tunnel_get_flowlabel(const struct iphdr *iph, 582 const struct sk_buff *skb) 583 { 584 __be16 payload_protocol = skb_protocol(skb, true); 585 586 if (payload_protocol == htons(ETH_P_IPV6)) 587 return ip6_flowlabel((const struct ipv6hdr *)iph); 588 else 589 return 0; 590 } 591 592 static inline u8 ip_tunnel_get_ttl(const struct iphdr *iph, 593 const struct sk_buff *skb) 594 { 595 __be16 payload_protocol = skb_protocol(skb, true); 596 597 if (payload_protocol == htons(ETH_P_IP)) 598 return iph->ttl; 599 else if (payload_protocol == htons(ETH_P_IPV6)) 600 return ((const struct ipv6hdr *)iph)->hop_limit; 601 else 602 return 0; 603 } 604 605 /* Propagate ECN bits out */ 606 static inline u8 ip_tunnel_ecn_encap(u8 tos, const struct iphdr *iph, 607 const struct sk_buff *skb) 608 { 609 u8 inner = ip_tunnel_get_dsfield(iph, skb); 610 611 return INET_ECN_encapsulate(tos, inner); 612 } 613 614 int __iptunnel_pull_header(struct sk_buff *skb, int hdr_len, 615 __be16 inner_proto, bool raw_proto, bool xnet); 616 617 static inline int iptunnel_pull_header(struct sk_buff *skb, int hdr_len, 618 __be16 inner_proto, bool xnet) 619 { 620 return __iptunnel_pull_header(skb, hdr_len, inner_proto, false, xnet); 621 } 622 623 void iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb, 624 __be32 src, __be32 dst, u8 proto, 625 u8 tos, u8 ttl, __be16 df, bool xnet, u16 ipcb_flags); 626 struct metadata_dst *iptunnel_metadata_reply(struct metadata_dst *md, 627 gfp_t flags); 628 int skb_tunnel_check_pmtu(struct sk_buff *skb, struct dst_entry *encap_dst, 629 int headroom, bool reply); 630 631 static inline unsigned int ip_tunnel_limit_headroom(unsigned int headroom) 632 { 633 /* we must cap headroom to some upperlimit, else pskb_expand_head 634 * will overflow header offsets in skb_headers_offset_update(). 635 */ 636 const unsigned int max_allowed = 512; 637 638 if (headroom > max_allowed) 639 headroom = max_allowed; 640 641 return headroom; 642 } 643 644 static inline void ip_tunnel_adj_headroom(struct net_device *dev, 645 unsigned int headroom) 646 { 647 headroom = ip_tunnel_limit_headroom(headroom); 648 649 if (headroom > READ_ONCE(dev->needed_headroom)) 650 WRITE_ONCE(dev->needed_headroom, headroom); 651 } 652 653 int iptunnel_handle_offloads(struct sk_buff *skb, int gso_type_mask); 654 655 static inline int iptunnel_pull_offloads(struct sk_buff *skb) 656 { 657 if (skb_is_gso(skb)) { 658 int err; 659 660 err = skb_unclone(skb, GFP_ATOMIC); 661 if (unlikely(err)) 662 return err; 663 skb_shinfo(skb)->gso_type &= ~(NETIF_F_GSO_ENCAP_ALL >> 664 NETIF_F_GSO_SHIFT); 665 } 666 667 skb->encapsulation = 0; 668 return 0; 669 } 670 671 static inline void iptunnel_xmit_stats(struct net_device *dev, int pkt_len) 672 { 673 if (pkt_len > 0) { 674 if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_DSTATS) { 675 struct pcpu_dstats *dstats = get_cpu_ptr(dev->dstats); 676 677 u64_stats_update_begin(&dstats->syncp); 678 u64_stats_add(&dstats->tx_bytes, pkt_len); 679 u64_stats_inc(&dstats->tx_packets); 680 u64_stats_update_end(&dstats->syncp); 681 put_cpu_ptr(dstats); 682 return; 683 } 684 if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_TSTATS) { 685 struct pcpu_sw_netstats *tstats = get_cpu_ptr(dev->tstats); 686 687 u64_stats_update_begin(&tstats->syncp); 688 u64_stats_add(&tstats->tx_bytes, pkt_len); 689 u64_stats_inc(&tstats->tx_packets); 690 u64_stats_update_end(&tstats->syncp); 691 put_cpu_ptr(tstats); 692 return; 693 } 694 pr_err_once("iptunnel_xmit_stats pcpu_stat_type=%d\n", 695 dev->pcpu_stat_type); 696 WARN_ON_ONCE(1); 697 return; 698 } 699 700 if (pkt_len < 0) { 701 DEV_STATS_INC(dev, tx_errors); 702 DEV_STATS_INC(dev, tx_aborted_errors); 703 } else { 704 DEV_STATS_INC(dev, tx_dropped); 705 } 706 } 707 708 static inline void ip_tunnel_info_opts_get(void *to, 709 const struct ip_tunnel_info *info) 710 { 711 memcpy(to, ip_tunnel_info_opts(info), info->options_len); 712 } 713 714 static inline void ip_tunnel_info_opts_set(struct ip_tunnel_info *info, 715 const void *from, int len, 716 const unsigned long *flags) 717 { 718 info->options_len = len; 719 if (len > 0) { 720 memcpy(ip_tunnel_info_opts(info), from, len); 721 ip_tunnel_flags_or(info->key.tun_flags, info->key.tun_flags, 722 flags); 723 } 724 } 725 726 static inline struct ip_tunnel_info *lwt_tun_info(struct lwtunnel_state *lwtstate) 727 { 728 return (struct ip_tunnel_info *)lwtstate->data; 729 } 730 731 DECLARE_STATIC_KEY_FALSE(ip_tunnel_metadata_cnt); 732 733 /* Returns > 0 if metadata should be collected */ 734 static inline int ip_tunnel_collect_metadata(void) 735 { 736 return static_branch_unlikely(&ip_tunnel_metadata_cnt); 737 } 738 739 void __init ip_tunnel_core_init(void); 740 741 void ip_tunnel_need_metadata(void); 742 void ip_tunnel_unneed_metadata(void); 743 744 #else /* CONFIG_INET */ 745 746 static inline struct ip_tunnel_info *lwt_tun_info(struct lwtunnel_state *lwtstate) 747 { 748 return NULL; 749 } 750 751 static inline void ip_tunnel_need_metadata(void) 752 { 753 } 754 755 static inline void ip_tunnel_unneed_metadata(void) 756 { 757 } 758 759 static inline void ip_tunnel_info_opts_get(void *to, 760 const struct ip_tunnel_info *info) 761 { 762 } 763 764 static inline void ip_tunnel_info_opts_set(struct ip_tunnel_info *info, 765 const void *from, int len, 766 const unsigned long *flags) 767 { 768 info->options_len = 0; 769 } 770 771 #endif /* CONFIG_INET */ 772 773 #endif /* __NET_IP_TUNNELS_H */ 774