Lines Matching +full:t +full:- +full:head
1 // SPDX-License-Identifier: GPL-2.0-only
65 return !test_bit(IP_TUNNEL_KEY_BIT, p->i_flags);
67 return test_bit(IP_TUNNEL_KEY_BIT, p->i_flags) && p->i_key == key;
86 struct ip_tunnel *t, *cand = NULL;
87 struct hlist_head *head;
92 head = &itn->tunnels[hash];
94 hlist_for_each_entry_rcu(t, head, hash_node) {
95 if (local != t->parms.iph.saddr ||
96 remote != t->parms.iph.daddr ||
97 !(t->dev->flags & IFF_UP))
100 if (!ip_tunnel_key_match(&t->parms, flags, key))
103 if (READ_ONCE(t->parms.link) == link)
104 return t;
105 cand = t;
108 hlist_for_each_entry_rcu(t, head, hash_node) {
109 if (remote != t->parms.iph.daddr ||
110 t->parms.iph.saddr != 0 ||
111 !(t->dev->flags & IFF_UP))
114 if (!ip_tunnel_key_match(&t->parms, flags, key))
117 if (READ_ONCE(t->parms.link) == link)
118 return t;
120 cand = t;
124 head = &itn->tunnels[hash];
126 hlist_for_each_entry_rcu(t, head, hash_node) {
127 if ((local != t->parms.iph.saddr || t->parms.iph.daddr != 0) &&
128 (local != t->parms.iph.daddr || !ipv4_is_multicast(local)))
131 if (!(t->dev->flags & IFF_UP))
134 if (!ip_tunnel_key_match(&t->parms, flags, key))
137 if (READ_ONCE(t->parms.link) == link)
138 return t;
140 cand = t;
143 hlist_for_each_entry_rcu(t, head, hash_node) {
145 t->parms.i_key != key) ||
146 t->parms.iph.saddr != 0 ||
147 t->parms.iph.daddr != 0 ||
148 !(t->dev->flags & IFF_UP))
151 if (READ_ONCE(t->parms.link) == link)
152 return t;
154 cand = t;
160 t = rcu_dereference(itn->collect_md_tun);
161 if (t && t->dev->flags & IFF_UP)
162 return t;
164 ndev = READ_ONCE(itn->fb_tunnel_dev);
165 if (ndev && ndev->flags & IFF_UP)
177 __be32 i_key = parms->i_key;
179 if (parms->iph.daddr && !ipv4_is_multicast(parms->iph.daddr))
180 remote = parms->iph.daddr;
184 if (!test_bit(IP_TUNNEL_KEY_BIT, parms->i_flags) &&
185 test_bit(IP_TUNNEL_VTI_BIT, parms->i_flags))
189 return &itn->tunnels[h];
192 static void ip_tunnel_add(struct ip_tunnel_net *itn, struct ip_tunnel *t)
194 struct hlist_head *head = ip_bucket(itn, &t->parms);
196 if (t->collect_md)
197 rcu_assign_pointer(itn->collect_md_tun, t);
198 hlist_add_head_rcu(&t->hash_node, head);
201 static void ip_tunnel_del(struct ip_tunnel_net *itn, struct ip_tunnel *t)
203 if (t->collect_md)
204 rcu_assign_pointer(itn->collect_md_tun, NULL);
205 hlist_del_init_rcu(&t->hash_node);
212 __be32 remote = parms->iph.daddr;
213 __be32 local = parms->iph.saddr;
215 __be32 key = parms->i_key;
216 int link = parms->link;
217 struct ip_tunnel *t = NULL;
218 struct hlist_head *head = ip_bucket(itn, parms);
220 ip_tunnel_flags_copy(flags, parms->i_flags);
222 hlist_for_each_entry_rcu(t, head, hash_node, lockdep_rtnl_is_held()) {
223 if (local == t->parms.iph.saddr &&
224 remote == t->parms.iph.daddr &&
225 link == READ_ONCE(t->parms.link) &&
226 type == t->dev->type &&
227 ip_tunnel_key_match(&t->parms, flags, key))
230 return t;
242 err = -E2BIG;
243 if (parms->name[0]) {
244 if (!dev_valid_name(parms->name))
246 strscpy(name, parms->name);
248 if (strlen(ops->kind) > (IFNAMSIZ - 3))
250 strscpy(name, ops->kind);
255 dev = alloc_netdev(ops->priv_size, name, NET_NAME_UNKNOWN, ops->setup);
257 err = -ENOMEM;
262 dev->rtnl_link_ops = ops;
265 tunnel->parms = *parms;
266 tunnel->net = net;
287 int t_hlen = tunnel->hlen + sizeof(struct iphdr);
289 iph = &tunnel->parms.iph;
292 if (iph->daddr) {
296 ip_tunnel_init_flow(&fl4, iph->protocol, iph->daddr,
297 iph->saddr, tunnel->parms.o_key,
298 iph->tos & INET_DSCP_MASK, tunnel->net,
299 tunnel->parms.link, tunnel->fwmark, 0, 0);
300 rt = ip_route_output_key(tunnel->net, &fl4);
303 tdev = rt->dst.dev;
306 if (dev->type != ARPHRD_ETHER)
307 dev->flags |= IFF_POINTOPOINT;
309 dst_cache_reset(&tunnel->dst_cache);
312 if (!tdev && tunnel->parms.link)
313 tdev = __dev_get_by_index(tunnel->net, tunnel->parms.link);
316 hlen = tdev->hard_header_len + tdev->needed_headroom;
317 mtu = min(tdev->mtu, IP_MAX_MTU);
320 dev->needed_headroom = t_hlen + hlen;
321 mtu -= t_hlen + (dev->type == ARPHRD_ETHER ? dev->hard_header_len : 0);
339 dev = __ip_tunnel_create(net, itn->rtnl_link_ops, parms);
349 t_hlen = nt->hlen + sizeof(struct iphdr);
350 dev->min_mtu = ETH_MIN_MTU;
351 dev->max_mtu = IP_MAX_MTU - t_hlen;
352 if (dev->type == ARPHRD_ETHER)
353 dev->max_mtu -= dev->hard_header_len;
368 if (iph->protocol != IPPROTO_UDP)
371 udph = (struct udphdr *)((__u8 *)iph + (iph->ihl << 2));
372 info->encap.sport = udph->source;
373 info->encap.dport = udph->dest;
385 if (ipv4_is_multicast(iph->daddr)) {
386 DEV_STATS_INC(tunnel->dev, multicast);
387 skb->pkt_type = PACKET_BROADCAST;
391 if (test_bit(IP_TUNNEL_CSUM_BIT, tunnel->parms.i_flags) !=
392 test_bit(IP_TUNNEL_CSUM_BIT, tpi->flags)) {
393 DEV_STATS_INC(tunnel->dev, rx_crc_errors);
394 DEV_STATS_INC(tunnel->dev, rx_errors);
398 if (test_bit(IP_TUNNEL_SEQ_BIT, tunnel->parms.i_flags)) {
399 if (!test_bit(IP_TUNNEL_SEQ_BIT, tpi->flags) ||
400 (tunnel->i_seqno && (s32)(ntohl(tpi->seq) - tunnel->i_seqno) < 0)) {
401 DEV_STATS_INC(tunnel->dev, rx_fifo_errors);
402 DEV_STATS_INC(tunnel->dev, rx_errors);
405 tunnel->i_seqno = ntohl(tpi->seq) + 1;
408 /* Save offset of outer header relative to skb->head,
410 * and might change skb->head.
412 nh = skb_network_header(skb) - skb->head;
414 skb_set_network_header(skb, (tunnel->dev->type == ARPHRD_ETHER) ? ETH_HLEN : 0);
417 DEV_STATS_INC(tunnel->dev, rx_length_errors);
418 DEV_STATS_INC(tunnel->dev, rx_errors);
421 iph = (struct iphdr *)(skb->head + nh);
426 net_info_ratelimited("non-ECT from %pI4 with TOS=%#x\n",
427 &iph->saddr, iph->tos);
429 DEV_STATS_INC(tunnel->dev, rx_frame_errors);
430 DEV_STATS_INC(tunnel->dev, rx_errors);
435 dev_sw_netstats_rx_add(tunnel->dev, skb->len);
436 skb_scrub_packet(skb, !net_eq(tunnel->net, dev_net(tunnel->dev)));
438 if (tunnel->dev->type == ARPHRD_ETHER) {
439 skb->protocol = eth_type_trans(skb, tunnel->dev);
442 skb->dev = tunnel->dev;
448 gro_cells_receive(&tunnel->gro_cells, skb);
463 return -ERANGE;
467 NULL, ops) ? 0 : -1;
477 return -ERANGE;
481 ops, NULL) == ops) ? 0 : -1;
489 int ip_tunnel_encap_setup(struct ip_tunnel *t,
494 memset(&t->encap, 0, sizeof(t->encap));
500 t->encap.type = ipencap->type;
501 t->encap.sport = ipencap->sport;
502 t->encap.dport = ipencap->dport;
503 t->encap.flags = ipencap->flags;
505 t->encap_hlen = hlen;
506 t->hlen = t->encap_hlen + t->tun_hlen;
521 tunnel_hlen = md ? tunnel_hlen : tunnel->hlen;
522 pkt_size = skb->len - tunnel_hlen;
523 pkt_size -= dev->type == ARPHRD_ETHER ? dev->hard_header_len : 0;
526 mtu = dst_mtu(&rt->dst) - (sizeof(struct iphdr) + tunnel_hlen);
527 mtu -= dev->type == ARPHRD_ETHER ? dev->hard_header_len : 0;
529 mtu = skb_valid_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu;
535 if (skb->protocol == htons(ETH_P_IP)) {
537 (inner_iph->frag_off & htons(IP_DF)) &&
540 return -E2BIG;
544 else if (skb->protocol == htons(ETH_P_IPV6)) {
550 daddr = md ? dst : tunnel->parms.iph.daddr;
555 rt6->rt6i_dst.plen == 128) {
556 rt6->rt6i_flags |= RTF_MODIFIED;
564 return -E2BIG;
581 if (headroom > READ_ONCE(dev->needed_headroom))
582 WRITE_ONCE(dev->needed_headroom, headroom);
600 if (unlikely(!tun_info || !(tun_info->mode & IP_TUNNEL_INFO_TX) ||
603 key = &tun_info->key;
604 memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
606 tos = key->tos;
608 if (skb->protocol == htons(ETH_P_IP))
609 tos = inner_iph->tos;
610 else if (skb->protocol == htons(ETH_P_IPV6))
613 ip_tunnel_init_flow(&fl4, proto, key->u.ipv4.dst, key->u.ipv4.src,
614 tunnel_id_to_key32(key->tun_id),
615 tos & INET_DSCP_MASK, tunnel->net, 0, skb->mark,
616 skb_get_hash(skb), key->flow_flags);
619 tunnel_hlen = ip_encap_hlen(&tun_info->encap);
621 if (ip_tunnel_encap(skb, &tun_info->encap, &proto, &fl4) < 0)
626 rt = dst_cache_get_ip4(&tun_info->dst_cache, &fl4.saddr);
628 rt = ip_route_output_key(tunnel->net, &fl4);
634 dst_cache_set_ip4(&tun_info->dst_cache, &rt->dst,
637 if (rt->dst.dev == dev) {
643 if (test_bit(IP_TUNNEL_DONT_FRAGMENT_BIT, key->tun_flags))
646 key->u.ipv4.dst, true)) {
652 ttl = key->ttl;
654 if (skb->protocol == htons(ETH_P_IP))
655 ttl = inner_iph->ttl;
656 else if (skb->protocol == htons(ETH_P_IPV6))
657 ttl = ((const struct ipv6hdr *)inner_iph)->hop_limit;
659 ttl = ip4_dst_hoplimit(&rt->dst);
662 headroom += LL_RESERVED_SPACE(rt->dst.dev) + rt->dst.header_len;
671 df, !net_eq(tunnel->net, dev_net(dev)));
701 connected = (tunnel->parms.iph.daddr != 0);
704 memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
706 dst = tnl_params->daddr;
716 if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX) &&
718 tun_info->key.u.ipv4.dst) {
719 dst = tun_info->key.u.ipv4.dst;
724 dst = rt_nexthop(rt, inner_iph->daddr);
734 &ipv6_hdr(skb)->daddr);
738 addr6 = (const struct in6_addr *)&neigh->primary_key;
742 addr6 = &ipv6_hdr(skb)->daddr;
750 dst = addr6->s6_addr32[3];
764 tos = tnl_params->tos;
768 tos = inner_iph->tos;
776 ip_tunnel_init_flow(&fl4, protocol, dst, tnl_params->saddr,
777 tunnel->parms.o_key, tos & INET_DSCP_MASK,
778 tunnel->net, READ_ONCE(tunnel->parms.link),
779 tunnel->fwmark, skb_get_hash(skb), 0);
781 if (ip_tunnel_encap(skb, &tunnel->encap, &protocol, &fl4) < 0)
787 rt = dst_cache_get_ip4(&tun_info->dst_cache,
790 rt = connected ? dst_cache_get_ip4(&tunnel->dst_cache,
795 rt = ip_route_output_key(tunnel->net, &fl4);
802 dst_cache_set_ip4(&tun_info->dst_cache, &rt->dst,
805 dst_cache_set_ip4(&tunnel->dst_cache, &rt->dst,
809 if (rt->dst.dev == dev) {
815 df = tnl_params->frag_off;
816 if (payload_protocol == htons(ETH_P_IP) && !tunnel->ignore_df)
817 df |= (inner_iph->frag_off & htons(IP_DF));
824 if (tunnel->err_count > 0) {
826 tunnel->err_time + IPTUNNEL_ERR_TIMEO)) {
827 tunnel->err_count--;
831 tunnel->err_count = 0;
835 ttl = tnl_params->ttl;
838 ttl = inner_iph->ttl;
841 ttl = ((const struct ipv6hdr *)inner_iph)->hop_limit;
844 ttl = ip4_dst_hoplimit(&rt->dst);
847 max_headroom = LL_RESERVED_SPACE(rt->dst.dev) + sizeof(struct iphdr)
848 + rt->dst.header_len + ip_encap_hlen(&tunnel->encap);
860 df, !net_eq(tunnel->net, dev_net(dev)));
874 struct ip_tunnel *t,
880 ip_tunnel_del(itn, t);
881 t->parms.iph.saddr = p->iph.saddr;
882 t->parms.iph.daddr = p->iph.daddr;
883 t->parms.i_key = p->i_key;
884 t->parms.o_key = p->o_key;
885 if (dev->type != ARPHRD_ETHER) {
886 __dev_addr_set(dev, &p->iph.saddr, 4);
887 memcpy(dev->broadcast, &p->iph.daddr, 4);
889 ip_tunnel_add(itn, t);
891 t->parms.iph.ttl = p->iph.ttl;
892 t->parms.iph.tos = p->iph.tos;
893 t->parms.iph.frag_off = p->iph.frag_off;
895 if (t->parms.link != p->link || t->fwmark != fwmark) {
898 WRITE_ONCE(t->parms.link, p->link);
899 t->fwmark = fwmark;
902 WRITE_ONCE(dev->mtu, mtu);
904 dst_cache_reset(&t->dst_cache);
912 struct ip_tunnel *t = netdev_priv(dev);
913 struct net *net = t->net;
914 struct ip_tunnel_net *itn = net_generic(net, t->ip_tnl_net_id);
918 if (dev == itn->fb_tunnel_dev) {
919 t = ip_tunnel_find(itn, p, itn->fb_tunnel_dev->type);
920 if (!t)
921 t = netdev_priv(dev);
923 memcpy(p, &t->parms, sizeof(*p));
928 err = -EPERM;
929 if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
931 if (p->iph.ttl)
932 p->iph.frag_off |= htons(IP_DF);
933 if (!test_bit(IP_TUNNEL_VTI_BIT, p->i_flags)) {
934 if (!test_bit(IP_TUNNEL_KEY_BIT, p->i_flags))
935 p->i_key = 0;
936 if (!test_bit(IP_TUNNEL_KEY_BIT, p->o_flags))
937 p->o_key = 0;
940 t = ip_tunnel_find(itn, p, itn->type);
943 if (!t) {
944 t = ip_tunnel_create(net, itn, p);
945 err = PTR_ERR_OR_ZERO(t);
949 err = -EEXIST;
952 if (dev != itn->fb_tunnel_dev && cmd == SIOCCHGTUNNEL) {
953 if (t) {
954 if (t->dev != dev) {
955 err = -EEXIST;
961 if (ipv4_is_multicast(p->iph.daddr))
963 else if (p->iph.daddr)
966 if ((dev->flags^nflags)&(IFF_POINTOPOINT|IFF_BROADCAST)) {
967 err = -EINVAL;
971 t = netdev_priv(dev);
975 if (t) {
977 ip_tunnel_update(itn, t, dev, p, true, 0);
979 err = -ENOENT;
984 err = -EPERM;
985 if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
988 if (dev == itn->fb_tunnel_dev) {
989 err = -ENOENT;
990 t = ip_tunnel_find(itn, p, itn->fb_tunnel_dev->type);
991 if (!t)
993 err = -EPERM;
994 if (t == netdev_priv(itn->fb_tunnel_dev))
996 dev = t->dev;
1003 err = -EINVAL;
1019 strscpy(kp->name, p.name);
1020 kp->link = p.link;
1021 ip_tunnel_flags_from_be16(kp->i_flags, p.i_flags);
1022 ip_tunnel_flags_from_be16(kp->o_flags, p.o_flags);
1023 kp->i_key = p.i_key;
1024 kp->o_key = p.o_key;
1025 memcpy(&kp->iph, &p.iph, min(sizeof(kp->iph), sizeof(p.iph)));
1035 if (!ip_tunnel_flags_is_be16_compat(kp->i_flags) ||
1036 !ip_tunnel_flags_is_be16_compat(kp->o_flags))
1041 strscpy(p.name, kp->name);
1042 p.link = kp->link;
1043 p.i_flags = ip_tunnel_flags_to_be16(kp->i_flags);
1044 p.o_flags = ip_tunnel_flags_to_be16(kp->o_flags);
1045 p.i_key = kp->i_key;
1046 p.o_key = kp->o_key;
1047 memcpy(&p.iph, &kp->iph, min(sizeof(p.iph), sizeof(kp->iph)));
1060 return -EFAULT;
1061 err = dev->netdev_ops->ndo_tunnel_ctl(dev, &p, cmd);
1063 return -EFAULT;
1071 int t_hlen = tunnel->hlen + sizeof(struct iphdr);
1072 int max_mtu = IP_MAX_MTU - t_hlen;
1074 if (dev->type == ARPHRD_ETHER)
1075 max_mtu -= dev->hard_header_len;
1078 return -EINVAL;
1082 return -EINVAL;
1087 WRITE_ONCE(dev->mtu, new_mtu);
1102 gro_cells_destroy(&tunnel->gro_cells);
1103 dst_cache_destroy(&tunnel->dst_cache);
1106 void ip_tunnel_dellink(struct net_device *dev, struct list_head *head)
1111 itn = net_generic(tunnel->net, tunnel->ip_tnl_net_id);
1113 if (itn->fb_tunnel_dev != dev) {
1115 unregister_netdevice_queue(dev, head);
1124 return READ_ONCE(tunnel->net);
1132 return READ_ONCE(tunnel->parms.link);
1143 itn->rtnl_link_ops = ops;
1145 INIT_HLIST_HEAD(&itn->tunnels[i]);
1151 itn->type = it_init_net->type;
1152 itn->fb_tunnel_dev = NULL;
1161 itn->fb_tunnel_dev = __ip_tunnel_create(net, ops, &parms);
1165 if (!IS_ERR(itn->fb_tunnel_dev)) {
1166 itn->fb_tunnel_dev->netns_immutable = true;
1167 itn->fb_tunnel_dev->mtu = ip_tunnel_bind_dev(itn->fb_tunnel_dev);
1168 ip_tunnel_add(itn, netdev_priv(itn->fb_tunnel_dev));
1169 itn->type = itn->fb_tunnel_dev->type;
1173 return PTR_ERR_OR_ZERO(itn->fb_tunnel_dev);
1179 struct list_head *head)
1188 if (dev->rtnl_link_ops == ops)
1189 unregister_netdevice_queue(dev, head);
1192 struct ip_tunnel *t;
1194 struct hlist_head *thead = &itn->tunnels[h];
1196 hlist_for_each_entry_safe(t, n, thead, hash_node)
1200 if (!net_eq(dev_net(t->dev), net))
1201 unregister_netdevice_queue(t->dev, head);
1216 itn = net_generic(net, nt->ip_tnl_net_id);
1218 if (nt->collect_md) {
1219 if (rtnl_dereference(itn->collect_md_tun))
1220 return -EEXIST;
1222 if (ip_tunnel_find(itn, p, dev->type))
1223 return -EEXIST;
1226 nt->net = net;
1227 nt->parms = *p;
1228 nt->fwmark = fwmark;
1233 if (dev->type == ARPHRD_ETHER && !tb[IFLA_ADDRESS])
1238 unsigned int max = IP_MAX_MTU - (nt->hlen + sizeof(struct iphdr));
1240 if (dev->type == ARPHRD_ETHER)
1241 max -= dev->hard_header_len;
1243 mtu = clamp(dev->mtu, (unsigned int)ETH_MIN_MTU, max);
1263 struct ip_tunnel *t;
1265 struct net *net = tunnel->net;
1266 struct ip_tunnel_net *itn = net_generic(net, tunnel->ip_tnl_net_id);
1268 if (dev == itn->fb_tunnel_dev)
1269 return -EINVAL;
1271 t = ip_tunnel_find(itn, p, dev->type);
1273 if (t) {
1274 if (t->dev != dev)
1275 return -EEXIST;
1277 t = tunnel;
1279 if (dev->type != ARPHRD_ETHER) {
1282 if (ipv4_is_multicast(p->iph.daddr))
1284 else if (p->iph.daddr)
1287 if ((dev->flags ^ nflags) &
1289 return -EINVAL;
1293 ip_tunnel_update(itn, t, dev, p, !tb[IFLA_MTU], fwmark);
1301 struct iphdr *iph = &tunnel->parms.iph;
1304 dev->needs_free_netdev = true;
1305 dev->priv_destructor = ip_tunnel_dev_free;
1306 dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
1308 err = dst_cache_init(&tunnel->dst_cache, GFP_KERNEL);
1312 err = gro_cells_init(&tunnel->gro_cells, dev);
1314 dst_cache_destroy(&tunnel->dst_cache);
1318 tunnel->dev = dev;
1319 strscpy(tunnel->parms.name, dev->name);
1320 iph->version = 4;
1321 iph->ihl = 5;
1323 if (tunnel->collect_md)
1333 struct net *net = tunnel->net;
1336 itn = net_generic(net, tunnel->ip_tnl_net_id);
1338 if (itn->fb_tunnel_dev == dev)
1339 WRITE_ONCE(itn->fb_tunnel_dev, NULL);
1341 dst_cache_reset(&tunnel->dst_cache);
1349 tunnel->ip_tnl_net_id = net_id;