1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * ip_vs_xmit.c: various packet transmitters for IPVS
4 *
5 * Authors: Wensong Zhang <wensong@linuxvirtualserver.org>
6 * Julian Anastasov <ja@ssi.bg>
7 *
8 * Changes:
9 *
10 * Description of forwarding methods:
11 * - all transmitters are called from LOCAL_IN (remote clients) and
12 * LOCAL_OUT (local clients) but for ICMP can be called from FORWARD
13 * - not all connections have destination server, for example,
14 * connections in backup server when fwmark is used
15 * - bypass connections use daddr from packet
16 * - we can use dst without ref while sending in RCU section, we use
17 * ref when returning NF_ACCEPT for NAT-ed packet via loopback
18 * LOCAL_OUT rules:
19 * - skb->dev is NULL, skb->protocol is not set (both are set in POST_ROUTING)
20 * - skb->pkt_type is not set yet
21 * - the only place where we can see skb->sk != NULL
22 */
23
24 #define pr_fmt(fmt) "IPVS: " fmt
25
26 #include <linux/kernel.h>
27 #include <linux/slab.h>
28 #include <linux/tcp.h> /* for tcphdr */
29 #include <net/ip.h>
30 #include <net/gue.h>
31 #include <net/gre.h>
32 #include <net/tcp.h> /* for csum_tcpudp_magic */
33 #include <net/udp.h>
34 #include <net/icmp.h> /* for icmp_send */
35 #include <net/route.h> /* for ip_route_output */
36 #include <net/ipv6.h>
37 #include <net/ip6_route.h>
38 #include <net/ip_tunnels.h>
39 #include <net/ip6_checksum.h>
40 #include <net/addrconf.h>
41 #include <linux/icmpv6.h>
42 #include <linux/netfilter.h>
43 #include <linux/netfilter_ipv4.h>
44
45 #include <net/ip_vs.h>
46
47 enum {
48 IP_VS_RT_MODE_LOCAL = 1, /* Allow local dest */
49 IP_VS_RT_MODE_NON_LOCAL = 2, /* Allow non-local dest */
50 IP_VS_RT_MODE_RDR = 4, /* Allow redirect from remote daddr to
51 * local
52 */
53 IP_VS_RT_MODE_CONNECT = 8, /* Always bind route to saddr */
54 IP_VS_RT_MODE_KNOWN_NH = 16,/* Route via remote addr */
55 IP_VS_RT_MODE_TUNNEL = 32,/* Tunnel mode */
56 };
57
ip_vs_dest_dst_alloc(void)58 static inline struct ip_vs_dest_dst *ip_vs_dest_dst_alloc(void)
59 {
60 return kmalloc_obj(struct ip_vs_dest_dst, GFP_ATOMIC);
61 }
62
ip_vs_dest_dst_free(struct ip_vs_dest_dst * dest_dst)63 static inline void ip_vs_dest_dst_free(struct ip_vs_dest_dst *dest_dst)
64 {
65 kfree(dest_dst);
66 }
67
68 /*
69 * Destination cache to speed up outgoing route lookup
70 */
71 static inline void
__ip_vs_dst_set(struct ip_vs_dest * dest,struct ip_vs_dest_dst * dest_dst,struct dst_entry * dst,u32 dst_cookie)72 __ip_vs_dst_set(struct ip_vs_dest *dest, struct ip_vs_dest_dst *dest_dst,
73 struct dst_entry *dst, u32 dst_cookie)
74 {
75 struct ip_vs_dest_dst *old;
76
77 old = rcu_dereference_protected(dest->dest_dst,
78 lockdep_is_held(&dest->dst_lock));
79
80 if (dest_dst) {
81 dest_dst->dst_cache = dst;
82 dest_dst->dst_cookie = dst_cookie;
83 }
84 rcu_assign_pointer(dest->dest_dst, dest_dst);
85
86 if (old)
87 call_rcu(&old->rcu_head, ip_vs_dest_dst_rcu_free);
88 }
89
90 static inline struct ip_vs_dest_dst *
__ip_vs_dst_check(struct ip_vs_dest * dest)91 __ip_vs_dst_check(struct ip_vs_dest *dest)
92 {
93 struct ip_vs_dest_dst *dest_dst = rcu_dereference(dest->dest_dst);
94 struct dst_entry *dst;
95
96 if (!dest_dst)
97 return NULL;
98 dst = dest_dst->dst_cache;
99 if (READ_ONCE(dst->obsolete) &&
100 dst->ops->check(dst, dest_dst->dst_cookie) == NULL)
101 return NULL;
102 return dest_dst;
103 }
104
105 /* Based on ip_exceeds_mtu(). */
ip_vs_exceeds_mtu(const struct sk_buff * skb,unsigned int mtu)106 static bool ip_vs_exceeds_mtu(const struct sk_buff *skb, unsigned int mtu)
107 {
108 if (skb->len <= mtu)
109 return false;
110
111 if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu))
112 return false;
113
114 return true;
115 }
116
117 static inline bool
__mtu_check_toobig_v6(const struct sk_buff * skb,u32 mtu)118 __mtu_check_toobig_v6(const struct sk_buff *skb, u32 mtu)
119 {
120 if (IP6CB(skb)->frag_max_size) {
121 /* frag_max_size tell us that, this packet have been
122 * defragmented by netfilter IPv6 conntrack module.
123 */
124 if (IP6CB(skb)->frag_max_size > mtu)
125 return true; /* largest fragment violate MTU */
126 } else if (ip_vs_exceeds_mtu(skb, mtu))
127 return true; /* Packet size violate MTU size */
128
129 return false;
130 }
131
132 /* Get route to daddr, optionally bind route to saddr */
do_output_route4(struct net * net,__be32 daddr,int rt_mode,__be32 * ret_saddr)133 static struct rtable *do_output_route4(struct net *net, __be32 daddr,
134 int rt_mode, __be32 *ret_saddr)
135 {
136 struct flowi4 fl4;
137 struct rtable *rt;
138
139 memset(&fl4, 0, sizeof(fl4));
140 fl4.daddr = daddr;
141 fl4.flowi4_flags = (rt_mode & IP_VS_RT_MODE_KNOWN_NH) ?
142 FLOWI_FLAG_KNOWN_NH : 0;
143
144 retry:
145 rt = ip_route_output_key(net, &fl4);
146 if (IS_ERR(rt)) {
147 IP_VS_DBG_RL("ip_route_output error, dest: %pI4\n", &daddr);
148 return NULL;
149 }
150 if (rt_mode & IP_VS_RT_MODE_CONNECT && fl4.saddr) {
151 ip_rt_put(rt);
152 flowi4_update_output(&fl4, 0, daddr, fl4.saddr);
153 rt_mode = 0;
154 goto retry;
155 }
156 if (ret_saddr)
157 *ret_saddr = fl4.saddr;
158 return rt;
159 }
160
161 #ifdef CONFIG_IP_VS_IPV6
__ip_vs_is_local_route6(struct rt6_info * rt)162 static inline int __ip_vs_is_local_route6(struct rt6_info *rt)
163 {
164 return rt->dst.dev && rt->dst.dev->flags & IFF_LOOPBACK;
165 }
166 #endif
167
crosses_local_route_boundary(int skb_af,struct sk_buff * skb,int rt_mode,bool new_rt_is_local)168 static inline bool crosses_local_route_boundary(int skb_af, struct sk_buff *skb,
169 int rt_mode,
170 bool new_rt_is_local)
171 {
172 bool rt_mode_allow_local = !!(rt_mode & IP_VS_RT_MODE_LOCAL);
173 bool rt_mode_allow_non_local = !!(rt_mode & IP_VS_RT_MODE_NON_LOCAL);
174 bool rt_mode_allow_redirect = !!(rt_mode & IP_VS_RT_MODE_RDR);
175 bool source_is_loopback;
176 bool old_rt_is_local;
177
178 #ifdef CONFIG_IP_VS_IPV6
179 if (skb_af == AF_INET6) {
180 int addr_type = ipv6_addr_type(&ipv6_hdr(skb)->saddr);
181
182 source_is_loopback =
183 (!skb->dev || skb->dev->flags & IFF_LOOPBACK) &&
184 (addr_type & IPV6_ADDR_LOOPBACK);
185 old_rt_is_local = __ip_vs_is_local_route6(
186 dst_rt6_info(skb_dst(skb)));
187 } else
188 #endif
189 {
190 source_is_loopback = ipv4_is_loopback(ip_hdr(skb)->saddr);
191 old_rt_is_local = skb_rtable(skb)->rt_flags & RTCF_LOCAL;
192 }
193
194 if (unlikely(new_rt_is_local)) {
195 if (!rt_mode_allow_local)
196 return true;
197 if (!rt_mode_allow_redirect && !old_rt_is_local)
198 return true;
199 } else {
200 if (!rt_mode_allow_non_local)
201 return true;
202 if (source_is_loopback)
203 return true;
204 }
205 return false;
206 }
207
maybe_update_pmtu(int skb_af,struct sk_buff * skb,int mtu)208 static inline void maybe_update_pmtu(int skb_af, struct sk_buff *skb, int mtu)
209 {
210 struct sock *sk = skb->sk;
211 struct rtable *ort = skb_rtable(skb);
212
213 if (!skb->dev && sk && sk_fullsock(sk))
214 ort->dst.ops->update_pmtu(&ort->dst, sk, NULL, mtu, true);
215 }
216
ensure_mtu_is_adequate(struct netns_ipvs * ipvs,int skb_af,int rt_mode,struct ip_vs_iphdr * ipvsh,struct sk_buff * skb,int mtu)217 static inline bool ensure_mtu_is_adequate(struct netns_ipvs *ipvs, int skb_af,
218 int rt_mode,
219 struct ip_vs_iphdr *ipvsh,
220 struct sk_buff *skb, int mtu)
221 {
222 #ifdef CONFIG_IP_VS_IPV6
223 if (skb_af == AF_INET6) {
224 struct net *net = ipvs->net;
225
226 if (unlikely(__mtu_check_toobig_v6(skb, mtu))) {
227 if (!skb->dev)
228 skb->dev = net->loopback_dev;
229 /* only send ICMP too big on first fragment */
230 if (!ipvsh->fragoffs && !ip_vs_iph_icmp(ipvsh))
231 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
232 IP_VS_DBG(1, "frag needed for %pI6c\n",
233 &ipv6_hdr(skb)->saddr);
234 return false;
235 }
236 } else
237 #endif
238 {
239 /* If we're going to tunnel the packet and pmtu discovery
240 * is disabled, we'll just fragment it anyway
241 */
242 if ((rt_mode & IP_VS_RT_MODE_TUNNEL) && !sysctl_pmtu_disc(ipvs))
243 return true;
244
245 if (unlikely(ip_hdr(skb)->frag_off & htons(IP_DF) &&
246 ip_vs_exceeds_mtu(skb, mtu) &&
247 !ip_vs_iph_icmp(ipvsh))) {
248 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED,
249 htonl(mtu));
250 IP_VS_DBG(1, "frag needed for %pI4\n",
251 &ip_hdr(skb)->saddr);
252 return false;
253 }
254 }
255
256 return true;
257 }
258
decrement_ttl(struct netns_ipvs * ipvs,int skb_af,struct sk_buff * skb)259 static inline bool decrement_ttl(struct netns_ipvs *ipvs,
260 int skb_af,
261 struct sk_buff *skb)
262 {
263 struct net *net = ipvs->net;
264
265 #ifdef CONFIG_IP_VS_IPV6
266 if (skb_af == AF_INET6) {
267 struct dst_entry *dst = skb_dst(skb);
268
269 /* check and decrement ttl */
270 if (ipv6_hdr(skb)->hop_limit <= 1) {
271 struct inet6_dev *idev = __in6_dev_get_safely(skb->dev);
272
273 /* Force OUTPUT device used as source address */
274 skb->dev = dst->dev;
275 icmpv6_send(skb, ICMPV6_TIME_EXCEED,
276 ICMPV6_EXC_HOPLIMIT, 0);
277 IP6_INC_STATS(net, idev, IPSTATS_MIB_INHDRERRORS);
278
279 return false;
280 }
281
282 /* don't propagate ttl change to cloned packets */
283 if (skb_ensure_writable(skb, sizeof(struct ipv6hdr)))
284 return false;
285
286 ipv6_hdr(skb)->hop_limit--;
287 } else
288 #endif
289 {
290 if (ip_hdr(skb)->ttl <= 1) {
291 /* Tell the sender its packet died... */
292 IP_INC_STATS(net, IPSTATS_MIB_INHDRERRORS);
293 icmp_send(skb, ICMP_TIME_EXCEEDED, ICMP_EXC_TTL, 0);
294 return false;
295 }
296
297 /* don't propagate ttl change to cloned packets */
298 if (skb_ensure_writable(skb, sizeof(struct iphdr)))
299 return false;
300
301 /* Decrease ttl */
302 ip_decrease_ttl(ip_hdr(skb));
303 }
304
305 return true;
306 }
307
308 /* rt has device that is down */
rt_dev_is_down(const struct net_device * dev)309 static bool rt_dev_is_down(const struct net_device *dev)
310 {
311 return dev && !netif_running(dev);
312 }
313
314 /* Get route to destination or remote server */
315 static int
__ip_vs_get_out_rt(struct netns_ipvs * ipvs,int skb_af,struct sk_buff * skb,struct ip_vs_dest * dest,__be32 daddr,int rt_mode,__be32 * ret_saddr,struct ip_vs_iphdr * ipvsh)316 __ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb,
317 struct ip_vs_dest *dest,
318 __be32 daddr, int rt_mode, __be32 *ret_saddr,
319 struct ip_vs_iphdr *ipvsh)
320 {
321 struct net *net = ipvs->net;
322 struct ip_vs_dest_dst *dest_dst;
323 struct rtable *rt; /* Route to the other host */
324 int mtu;
325 int local, noref = 1;
326
327 if (dest) {
328 dest_dst = __ip_vs_dst_check(dest);
329 if (likely(dest_dst)) {
330 rt = dst_rtable(dest_dst->dst_cache);
331 if (ret_saddr)
332 *ret_saddr = dest_dst->dst_saddr.ip;
333 } else {
334 dest_dst = ip_vs_dest_dst_alloc();
335 spin_lock_bh(&dest->dst_lock);
336 if (!dest_dst) {
337 __ip_vs_dst_set(dest, NULL, NULL, 0);
338 spin_unlock_bh(&dest->dst_lock);
339 goto err_unreach;
340 }
341 rt = do_output_route4(net, dest->addr.ip, rt_mode,
342 &dest_dst->dst_saddr.ip);
343 if (!rt) {
344 __ip_vs_dst_set(dest, NULL, NULL, 0);
345 spin_unlock_bh(&dest->dst_lock);
346 ip_vs_dest_dst_free(dest_dst);
347 goto err_unreach;
348 }
349 /* It is forbidden to attach dest->dest_dst if
350 * device is going down or if server is removed and
351 * stored in dest_trash.
352 */
353 if (!rt_dev_is_down(dst_dev_rcu(&rt->dst)) &&
354 dest->cflags & IP_VS_DEST_CF_AVAILABLE)
355 __ip_vs_dst_set(dest, dest_dst, &rt->dst, 0);
356 else
357 noref = 0;
358 spin_unlock_bh(&dest->dst_lock);
359 IP_VS_DBG(10, "new dst %pI4, src %pI4, refcnt=%d\n",
360 &dest->addr.ip, &dest_dst->dst_saddr.ip,
361 rcuref_read(&rt->dst.__rcuref));
362 if (ret_saddr)
363 *ret_saddr = dest_dst->dst_saddr.ip;
364 if (!noref)
365 ip_vs_dest_dst_free(dest_dst);
366 }
367 } else {
368 noref = 0;
369
370 /* For such unconfigured boxes avoid many route lookups
371 * for performance reasons because we do not remember saddr
372 */
373 rt_mode &= ~IP_VS_RT_MODE_CONNECT;
374 rt = do_output_route4(net, daddr, rt_mode, ret_saddr);
375 if (!rt)
376 goto err_unreach;
377 }
378
379 local = (rt->rt_flags & RTCF_LOCAL) ? 1 : 0;
380 if (unlikely(crosses_local_route_boundary(skb_af, skb, rt_mode,
381 local))) {
382 IP_VS_DBG_RL("We are crossing local and non-local addresses"
383 " daddr=%pI4\n", &daddr);
384 goto err_put;
385 }
386
387 if (unlikely(local)) {
388 /* skb to local stack, preserve old route */
389 if (!noref)
390 ip_rt_put(rt);
391 return local;
392 }
393
394 if (!decrement_ttl(ipvs, skb_af, skb))
395 goto err_put;
396
397 if (likely(!(rt_mode & IP_VS_RT_MODE_TUNNEL))) {
398 mtu = dst_mtu(&rt->dst);
399 } else {
400 mtu = dst_mtu(&rt->dst) - sizeof(struct iphdr);
401 if (!dest)
402 goto err_put;
403 if (dest->tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) {
404 mtu -= sizeof(struct udphdr) + sizeof(struct guehdr);
405 if ((dest->tun_flags &
406 IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM) &&
407 skb->ip_summed == CHECKSUM_PARTIAL)
408 mtu -= GUE_PLEN_REMCSUM + GUE_LEN_PRIV;
409 } else if (dest->tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GRE) {
410 IP_TUNNEL_DECLARE_FLAGS(tflags) = { };
411
412 if (dest->tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM)
413 __set_bit(IP_TUNNEL_CSUM_BIT, tflags);
414 mtu -= gre_calc_hlen(tflags);
415 }
416 if (mtu < 68) {
417 IP_VS_DBG_RL("%s(): mtu less than 68\n", __func__);
418 goto err_put;
419 }
420 maybe_update_pmtu(skb_af, skb, mtu);
421 }
422
423 if (!ensure_mtu_is_adequate(ipvs, skb_af, rt_mode, ipvsh, skb, mtu))
424 goto err_put;
425
426 skb_dst_drop(skb);
427 if (noref)
428 skb_dst_set_noref(skb, &rt->dst);
429 else
430 skb_dst_set(skb, &rt->dst);
431
432 return local;
433
434 err_put:
435 if (!noref)
436 ip_rt_put(rt);
437 return -1;
438
439 err_unreach:
440 if (!skb->dev)
441 skb->dev = skb_dst(skb)->dev;
442
443 dst_link_failure(skb);
444 return -1;
445 }
446
447 #ifdef CONFIG_IP_VS_IPV6
448 static struct dst_entry *
__ip_vs_route_output_v6(struct net * net,struct in6_addr * daddr,struct in6_addr * ret_saddr,int do_xfrm,int rt_mode)449 __ip_vs_route_output_v6(struct net *net, struct in6_addr *daddr,
450 struct in6_addr *ret_saddr, int do_xfrm, int rt_mode)
451 {
452 struct dst_entry *dst;
453 struct flowi6 fl6 = {
454 .daddr = *daddr,
455 };
456
457 if (rt_mode & IP_VS_RT_MODE_KNOWN_NH)
458 fl6.flowi6_flags = FLOWI_FLAG_KNOWN_NH;
459
460 dst = ip6_route_output(net, NULL, &fl6);
461 if (dst->error)
462 goto out_err;
463 if (!ret_saddr)
464 return dst;
465 if (ipv6_addr_any(&fl6.saddr) &&
466 ipv6_dev_get_saddr(net, ip6_dst_idev(dst)->dev,
467 &fl6.daddr, 0, &fl6.saddr) < 0)
468 goto out_err;
469 if (do_xfrm) {
470 dst = xfrm_lookup(net, dst, flowi6_to_flowi(&fl6), NULL, 0);
471 if (IS_ERR(dst)) {
472 dst = NULL;
473 goto out_err;
474 }
475 }
476 *ret_saddr = fl6.saddr;
477 return dst;
478
479 out_err:
480 dst_release(dst);
481 IP_VS_DBG_RL("ip6_route_output error, dest: %pI6\n", daddr);
482 return NULL;
483 }
484
485 /*
486 * Get route to destination or remote server
487 */
488 static int
__ip_vs_get_out_rt_v6(struct netns_ipvs * ipvs,int skb_af,struct sk_buff * skb,struct ip_vs_dest * dest,struct in6_addr * daddr,struct in6_addr * ret_saddr,struct ip_vs_iphdr * ipvsh,int do_xfrm,int rt_mode)489 __ip_vs_get_out_rt_v6(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb,
490 struct ip_vs_dest *dest,
491 struct in6_addr *daddr, struct in6_addr *ret_saddr,
492 struct ip_vs_iphdr *ipvsh, int do_xfrm, int rt_mode)
493 {
494 struct net *net = ipvs->net;
495 struct ip_vs_dest_dst *dest_dst;
496 struct rt6_info *rt; /* Route to the other host */
497 struct dst_entry *dst;
498 int mtu;
499 int local, noref = 1;
500
501 if (dest) {
502 dest_dst = __ip_vs_dst_check(dest);
503 if (likely(dest_dst)) {
504 rt = dst_rt6_info(dest_dst->dst_cache);
505 if (ret_saddr)
506 *ret_saddr = dest_dst->dst_saddr.in6;
507 } else {
508 u32 cookie;
509
510 dest_dst = ip_vs_dest_dst_alloc();
511 spin_lock_bh(&dest->dst_lock);
512 if (!dest_dst) {
513 __ip_vs_dst_set(dest, NULL, NULL, 0);
514 spin_unlock_bh(&dest->dst_lock);
515 goto err_unreach;
516 }
517 dst = __ip_vs_route_output_v6(net, &dest->addr.in6,
518 &dest_dst->dst_saddr.in6,
519 do_xfrm, rt_mode);
520 if (!dst) {
521 __ip_vs_dst_set(dest, NULL, NULL, 0);
522 spin_unlock_bh(&dest->dst_lock);
523 ip_vs_dest_dst_free(dest_dst);
524 goto err_unreach;
525 }
526 rt = dst_rt6_info(dst);
527 cookie = rt6_get_cookie(rt);
528 /* It is forbidden to attach dest->dest_dst if
529 * device is going down or if server is removed and
530 * stored in dest_trash.
531 */
532 if (!rt_dev_is_down(dst_dev_rcu(&rt->dst)) &&
533 dest->cflags & IP_VS_DEST_CF_AVAILABLE)
534 __ip_vs_dst_set(dest, dest_dst, &rt->dst, cookie);
535 else
536 noref = 0;
537 spin_unlock_bh(&dest->dst_lock);
538 IP_VS_DBG(10, "new dst %pI6, src %pI6, refcnt=%d\n",
539 &dest->addr.in6, &dest_dst->dst_saddr.in6,
540 rcuref_read(&rt->dst.__rcuref));
541 if (ret_saddr)
542 *ret_saddr = dest_dst->dst_saddr.in6;
543 if (!noref)
544 ip_vs_dest_dst_free(dest_dst);
545 }
546 } else {
547 noref = 0;
548 dst = __ip_vs_route_output_v6(net, daddr, ret_saddr, do_xfrm,
549 rt_mode);
550 if (!dst)
551 goto err_unreach;
552 rt = dst_rt6_info(dst);
553 }
554
555 local = __ip_vs_is_local_route6(rt);
556
557 if (unlikely(crosses_local_route_boundary(skb_af, skb, rt_mode,
558 local))) {
559 IP_VS_DBG_RL("We are crossing local and non-local addresses"
560 " daddr=%pI6\n", daddr);
561 goto err_put;
562 }
563
564 if (unlikely(local)) {
565 /* skb to local stack, preserve old route */
566 if (!noref)
567 dst_release(&rt->dst);
568 return local;
569 }
570
571 if (!decrement_ttl(ipvs, skb_af, skb))
572 goto err_put;
573
574 /* MTU checking */
575 if (likely(!(rt_mode & IP_VS_RT_MODE_TUNNEL)))
576 mtu = dst_mtu(&rt->dst);
577 else {
578 mtu = dst_mtu(&rt->dst) - sizeof(struct ipv6hdr);
579 if (!dest)
580 goto err_put;
581 if (dest->tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) {
582 mtu -= sizeof(struct udphdr) + sizeof(struct guehdr);
583 if ((dest->tun_flags &
584 IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM) &&
585 skb->ip_summed == CHECKSUM_PARTIAL)
586 mtu -= GUE_PLEN_REMCSUM + GUE_LEN_PRIV;
587 } else if (dest->tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GRE) {
588 IP_TUNNEL_DECLARE_FLAGS(tflags) = { };
589
590 if (dest->tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM)
591 __set_bit(IP_TUNNEL_CSUM_BIT, tflags);
592 mtu -= gre_calc_hlen(tflags);
593 }
594 if (mtu < IPV6_MIN_MTU) {
595 IP_VS_DBG_RL("%s(): mtu less than %d\n", __func__,
596 IPV6_MIN_MTU);
597 goto err_put;
598 }
599 maybe_update_pmtu(skb_af, skb, mtu);
600 }
601
602 if (!ensure_mtu_is_adequate(ipvs, skb_af, rt_mode, ipvsh, skb, mtu))
603 goto err_put;
604
605 skb_dst_drop(skb);
606 if (noref)
607 skb_dst_set_noref(skb, &rt->dst);
608 else
609 skb_dst_set(skb, &rt->dst);
610
611 return local;
612
613 err_put:
614 if (!noref)
615 dst_release(&rt->dst);
616 return -1;
617
618 err_unreach:
619 /* The ip6_link_failure function requires the dev field to be set
620 * in order to get the net (further for the sake of fwmark
621 * reflection).
622 */
623 if (!skb->dev)
624 skb->dev = skb_dst(skb)->dev;
625
626 dst_link_failure(skb);
627 return -1;
628 }
629 #endif
630
631
632 /* return NF_ACCEPT to allow forwarding or other NF_xxx on error */
ip_vs_tunnel_xmit_prepare(struct sk_buff * skb,struct ip_vs_conn * cp)633 static inline int ip_vs_tunnel_xmit_prepare(struct sk_buff *skb,
634 struct ip_vs_conn *cp)
635 {
636 int ret = NF_ACCEPT;
637
638 skb->ipvs_property = 1;
639 if (unlikely(cp->flags & IP_VS_CONN_F_NFCT))
640 ret = ip_vs_confirm_conntrack(skb);
641 if (ret == NF_ACCEPT) {
642 nf_reset_ct(skb);
643 skb_forward_csum(skb);
644 if (skb->dev)
645 skb_clear_tstamp(skb);
646 }
647 return ret;
648 }
649
650 /* In the event of a remote destination, it's possible that we would have
651 * matches against an old socket (particularly a TIME-WAIT socket). This
652 * causes havoc down the line (ip_local_out et. al. expect regular sockets
653 * and invalid memory accesses will happen) so simply drop the association
654 * in this case.
655 */
ip_vs_drop_early_demux_sk(struct sk_buff * skb)656 static inline void ip_vs_drop_early_demux_sk(struct sk_buff *skb)
657 {
658 /* If dev is set, the packet came from the LOCAL_IN callback and
659 * not from a local TCP socket.
660 */
661 if (skb->dev)
662 skb_orphan(skb);
663 }
664
665 /* return NF_STOLEN (sent) or NF_ACCEPT if local=1 (not sent) */
ip_vs_nat_send_or_cont(int pf,struct sk_buff * skb,struct ip_vs_conn * cp,int local)666 static inline int ip_vs_nat_send_or_cont(int pf, struct sk_buff *skb,
667 struct ip_vs_conn *cp, int local)
668 {
669 int ret = NF_STOLEN;
670
671 skb->ipvs_property = 1;
672 if (likely(!(cp->flags & IP_VS_CONN_F_NFCT)))
673 ip_vs_notrack(skb);
674 else
675 ip_vs_update_conntrack(skb, cp, 1);
676
677 /* Remove the early_demux association unless it's bound for the
678 * exact same port and address on this host after translation.
679 */
680 if (!local || cp->vport != cp->dport ||
681 !ip_vs_addr_equal(cp->af, &cp->vaddr, &cp->daddr))
682 ip_vs_drop_early_demux_sk(skb);
683
684 if (!local) {
685 skb_forward_csum(skb);
686 if (skb->dev)
687 skb_clear_tstamp(skb);
688 NF_HOOK(pf, NF_INET_LOCAL_OUT, cp->ipvs->net, NULL, skb,
689 NULL, skb_dst(skb)->dev, dst_output);
690 } else
691 ret = NF_ACCEPT;
692
693 return ret;
694 }
695
696 /* return NF_STOLEN (sent) or NF_ACCEPT if local=1 (not sent) */
ip_vs_send_or_cont(int pf,struct sk_buff * skb,struct ip_vs_conn * cp,int local)697 static inline int ip_vs_send_or_cont(int pf, struct sk_buff *skb,
698 struct ip_vs_conn *cp, int local)
699 {
700 int ret = NF_STOLEN;
701
702 skb->ipvs_property = 1;
703 if (likely(!(cp->flags & IP_VS_CONN_F_NFCT)))
704 ip_vs_notrack(skb);
705 if (!local) {
706 ip_vs_drop_early_demux_sk(skb);
707 skb_forward_csum(skb);
708 if (skb->dev)
709 skb_clear_tstamp(skb);
710 NF_HOOK(pf, NF_INET_LOCAL_OUT, cp->ipvs->net, NULL, skb,
711 NULL, skb_dst(skb)->dev, dst_output);
712 } else
713 ret = NF_ACCEPT;
714 return ret;
715 }
716
717
718 /*
719 * NULL transmitter (do nothing except return NF_ACCEPT)
720 */
721 int
ip_vs_null_xmit(struct sk_buff * skb,struct ip_vs_conn * cp,struct ip_vs_protocol * pp,struct ip_vs_iphdr * ipvsh)722 ip_vs_null_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
723 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
724 {
725 /* we do not touch skb and do not need pskb ptr */
726 return ip_vs_send_or_cont(NFPROTO_IPV4, skb, cp, 1);
727 }
728
729
730 /*
731 * Bypass transmitter
732 * Let packets bypass the destination when the destination is not
733 * available, it may be only used in transparent cache cluster.
734 */
735 int
ip_vs_bypass_xmit(struct sk_buff * skb,struct ip_vs_conn * cp,struct ip_vs_protocol * pp,struct ip_vs_iphdr * ipvsh)736 ip_vs_bypass_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
737 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
738 {
739 if (__ip_vs_get_out_rt(cp->ipvs, cp->af, skb, NULL, ip_hdr(skb)->daddr,
740 IP_VS_RT_MODE_NON_LOCAL, NULL, ipvsh) < 0)
741 goto tx_error;
742
743 ip_send_check(ip_hdr(skb));
744
745 /* Another hack: avoid icmp_send in ip_fragment */
746 skb->ignore_df = 1;
747
748 ip_vs_send_or_cont(NFPROTO_IPV4, skb, cp, 0);
749
750 return NF_STOLEN;
751
752 tx_error:
753 kfree_skb(skb);
754 return NF_STOLEN;
755 }
756
757 #ifdef CONFIG_IP_VS_IPV6
758 int
ip_vs_bypass_xmit_v6(struct sk_buff * skb,struct ip_vs_conn * cp,struct ip_vs_protocol * pp,struct ip_vs_iphdr * ipvsh)759 ip_vs_bypass_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
760 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
761 {
762 struct ipv6hdr *iph = ipv6_hdr(skb);
763
764 if (__ip_vs_get_out_rt_v6(cp->ipvs, cp->af, skb, NULL,
765 &iph->daddr, NULL,
766 ipvsh, 0, IP_VS_RT_MODE_NON_LOCAL) < 0)
767 goto tx_error;
768
769 /* Another hack: avoid icmp_send in ip_fragment */
770 skb->ignore_df = 1;
771
772 ip_vs_send_or_cont(NFPROTO_IPV6, skb, cp, 0);
773
774 return NF_STOLEN;
775
776 tx_error:
777 kfree_skb(skb);
778 return NF_STOLEN;
779 }
780 #endif
781
782 /*
783 * NAT transmitter (only for outside-to-inside nat forwarding)
784 * Not used for related ICMP
785 */
786 int
ip_vs_nat_xmit(struct sk_buff * skb,struct ip_vs_conn * cp,struct ip_vs_protocol * pp,struct ip_vs_iphdr * ipvsh)787 ip_vs_nat_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
788 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
789 {
790 struct rtable *rt; /* Route to the other host */
791 int local, rc, was_input;
792
793 /* check if it is a connection of no-client-port */
794 if (unlikely(cp->flags & IP_VS_CONN_F_NO_CPORT)) {
795 __be16 _pt, *p;
796
797 p = skb_header_pointer(skb, ipvsh->len, sizeof(_pt), &_pt);
798 if (p == NULL)
799 goto tx_error;
800 ip_vs_conn_fill_cport(cp, *p);
801 IP_VS_DBG(10, "filled cport=%d\n", ntohs(*p));
802 }
803
804 was_input = rt_is_input_route(skb_rtable(skb));
805 local = __ip_vs_get_out_rt(cp->ipvs, cp->af, skb, cp->dest, cp->daddr.ip,
806 IP_VS_RT_MODE_LOCAL |
807 IP_VS_RT_MODE_NON_LOCAL |
808 IP_VS_RT_MODE_RDR, NULL, ipvsh);
809 if (local < 0)
810 goto tx_error;
811 rt = skb_rtable(skb);
812 /*
813 * Avoid duplicate tuple in reply direction for NAT traffic
814 * to local address when connection is sync-ed
815 */
816 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
817 if (cp->flags & IP_VS_CONN_F_SYNC && local) {
818 enum ip_conntrack_info ctinfo;
819 struct nf_conn *ct = nf_ct_get(skb, &ctinfo);
820
821 if (ct) {
822 IP_VS_DBG_RL_PKT(10, AF_INET, pp, skb, ipvsh->off,
823 "ip_vs_nat_xmit(): "
824 "stopping DNAT to local address");
825 goto tx_error;
826 }
827 }
828 #endif
829
830 /* From world but DNAT to loopback address? */
831 if (local && ipv4_is_loopback(cp->daddr.ip) && was_input) {
832 IP_VS_DBG_RL_PKT(1, AF_INET, pp, skb, ipvsh->off,
833 "ip_vs_nat_xmit(): stopping DNAT to loopback "
834 "address");
835 goto tx_error;
836 }
837
838 /* copy-on-write the packet before mangling it */
839 if (skb_ensure_writable(skb, sizeof(struct iphdr)))
840 goto tx_error;
841
842 if (skb_cow(skb, rt->dst.dev->hard_header_len))
843 goto tx_error;
844
845 /* mangle the packet */
846 if (pp->dnat_handler && !pp->dnat_handler(skb, pp, cp, ipvsh))
847 goto tx_error;
848 ip_hdr(skb)->daddr = cp->daddr.ip;
849 ip_send_check(ip_hdr(skb));
850
851 IP_VS_DBG_PKT(10, AF_INET, pp, skb, ipvsh->off, "After DNAT");
852
853 /* FIXME: when application helper enlarges the packet and the length
854 is larger than the MTU of outgoing device, there will be still
855 MTU problem. */
856
857 /* Another hack: avoid icmp_send in ip_fragment */
858 skb->ignore_df = 1;
859
860 rc = ip_vs_nat_send_or_cont(NFPROTO_IPV4, skb, cp, local);
861
862 return rc;
863
864 tx_error:
865 kfree_skb(skb);
866 return NF_STOLEN;
867 }
868
869 #ifdef CONFIG_IP_VS_IPV6
870 int
ip_vs_nat_xmit_v6(struct sk_buff * skb,struct ip_vs_conn * cp,struct ip_vs_protocol * pp,struct ip_vs_iphdr * ipvsh)871 ip_vs_nat_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
872 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
873 {
874 struct rt6_info *rt; /* Route to the other host */
875 int local, rc;
876
877 /* check if it is a connection of no-client-port */
878 if (unlikely(cp->flags & IP_VS_CONN_F_NO_CPORT && !ipvsh->fragoffs)) {
879 __be16 _pt, *p;
880 p = skb_header_pointer(skb, ipvsh->len, sizeof(_pt), &_pt);
881 if (p == NULL)
882 goto tx_error;
883 ip_vs_conn_fill_cport(cp, *p);
884 IP_VS_DBG(10, "filled cport=%d\n", ntohs(*p));
885 }
886
887 local = __ip_vs_get_out_rt_v6(cp->ipvs, cp->af, skb, cp->dest,
888 &cp->daddr.in6,
889 NULL, ipvsh, 0,
890 IP_VS_RT_MODE_LOCAL |
891 IP_VS_RT_MODE_NON_LOCAL |
892 IP_VS_RT_MODE_RDR);
893 if (local < 0)
894 goto tx_error;
895 rt = dst_rt6_info(skb_dst(skb));
896 /*
897 * Avoid duplicate tuple in reply direction for NAT traffic
898 * to local address when connection is sync-ed
899 */
900 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
901 if (cp->flags & IP_VS_CONN_F_SYNC && local) {
902 enum ip_conntrack_info ctinfo;
903 struct nf_conn *ct = nf_ct_get(skb, &ctinfo);
904
905 if (ct) {
906 IP_VS_DBG_RL_PKT(10, AF_INET6, pp, skb, ipvsh->off,
907 "ip_vs_nat_xmit_v6(): "
908 "stopping DNAT to local address");
909 goto tx_error;
910 }
911 }
912 #endif
913
914 /* From world but DNAT to loopback address? */
915 if (local && skb->dev && !(skb->dev->flags & IFF_LOOPBACK) &&
916 ipv6_addr_type(&cp->daddr.in6) & IPV6_ADDR_LOOPBACK) {
917 IP_VS_DBG_RL_PKT(1, AF_INET6, pp, skb, ipvsh->off,
918 "ip_vs_nat_xmit_v6(): "
919 "stopping DNAT to loopback address");
920 goto tx_error;
921 }
922
923 /* copy-on-write the packet before mangling it */
924 if (skb_ensure_writable(skb, sizeof(struct ipv6hdr)))
925 goto tx_error;
926
927 if (skb_cow(skb, rt->dst.dev->hard_header_len))
928 goto tx_error;
929
930 /* mangle the packet */
931 if (pp->dnat_handler && !pp->dnat_handler(skb, pp, cp, ipvsh))
932 goto tx_error;
933 ipv6_hdr(skb)->daddr = cp->daddr.in6;
934
935 IP_VS_DBG_PKT(10, AF_INET6, pp, skb, ipvsh->off, "After DNAT");
936
937 /* FIXME: when application helper enlarges the packet and the length
938 is larger than the MTU of outgoing device, there will be still
939 MTU problem. */
940
941 /* Another hack: avoid icmp_send in ip_fragment */
942 skb->ignore_df = 1;
943
944 rc = ip_vs_nat_send_or_cont(NFPROTO_IPV6, skb, cp, local);
945
946 return rc;
947
948 tx_error:
949 kfree_skb(skb);
950 return NF_STOLEN;
951 }
952 #endif
953
954 /* When forwarding a packet, we must ensure that we've got enough headroom
955 * for the encapsulation packet in the skb. This also gives us an
956 * opportunity to figure out what the payload_len, dsfield, ttl, and df
957 * values should be, so that we won't need to look at the old ip header
958 * again
959 */
960 static struct sk_buff *
ip_vs_prepare_tunneled_skb(struct sk_buff * skb,int skb_af,unsigned int max_headroom,__u8 * next_protocol,__u32 * payload_len,__u8 * dsfield,__u8 * ttl,__be16 * df)961 ip_vs_prepare_tunneled_skb(struct sk_buff *skb, int skb_af,
962 unsigned int max_headroom, __u8 *next_protocol,
963 __u32 *payload_len, __u8 *dsfield, __u8 *ttl,
964 __be16 *df)
965 {
966 struct sk_buff *new_skb = NULL;
967 struct iphdr *old_iph = NULL;
968 __u8 old_dsfield;
969 #ifdef CONFIG_IP_VS_IPV6
970 struct ipv6hdr *old_ipv6h = NULL;
971 #endif
972
973 ip_vs_drop_early_demux_sk(skb);
974
975 if (skb_headroom(skb) < max_headroom || skb_cloned(skb)) {
976 new_skb = skb_realloc_headroom(skb, max_headroom);
977 if (!new_skb)
978 goto error;
979 if (skb->sk)
980 skb_set_owner_w(new_skb, skb->sk);
981 consume_skb(skb);
982 skb = new_skb;
983 }
984
985 #ifdef CONFIG_IP_VS_IPV6
986 if (skb_af == AF_INET6) {
987 old_ipv6h = ipv6_hdr(skb);
988 *next_protocol = IPPROTO_IPV6;
989 if (payload_len)
990 *payload_len =
991 ipv6_payload_len(skb, old_ipv6h) +
992 sizeof(*old_ipv6h);
993 old_dsfield = ipv6_get_dsfield(old_ipv6h);
994 *ttl = old_ipv6h->hop_limit;
995 if (df)
996 *df = 0;
997 } else
998 #endif
999 {
1000 old_iph = ip_hdr(skb);
1001 /* Copy DF, reset fragment offset and MF */
1002 if (df)
1003 *df = (old_iph->frag_off & htons(IP_DF));
1004 *next_protocol = IPPROTO_IPIP;
1005
1006 /* fix old IP header checksum */
1007 ip_send_check(old_iph);
1008 old_dsfield = ipv4_get_dsfield(old_iph);
1009 *ttl = old_iph->ttl;
1010 if (payload_len)
1011 *payload_len = skb_ip_totlen(skb);
1012 }
1013
1014 /* Implement full-functionality option for ECN encapsulation */
1015 *dsfield = INET_ECN_encapsulate(old_dsfield, old_dsfield);
1016
1017 return skb;
1018 error:
1019 kfree_skb(skb);
1020 return ERR_PTR(-ENOMEM);
1021 }
1022
__tun_gso_type_mask(int encaps_af,int orig_af)1023 static inline int __tun_gso_type_mask(int encaps_af, int orig_af)
1024 {
1025 switch (encaps_af) {
1026 case AF_INET:
1027 return SKB_GSO_IPXIP4;
1028 case AF_INET6:
1029 return SKB_GSO_IPXIP6;
1030 default:
1031 return 0;
1032 }
1033 }
1034
1035 static int
ipvs_gue_encap(struct net * net,struct sk_buff * skb,struct ip_vs_conn * cp,__u8 * next_protocol)1036 ipvs_gue_encap(struct net *net, struct sk_buff *skb,
1037 struct ip_vs_conn *cp, __u8 *next_protocol)
1038 {
1039 __be16 dport;
1040 __be16 sport = udp_flow_src_port(net, skb, 0, 0, false);
1041 struct udphdr *udph; /* Our new UDP header */
1042 struct guehdr *gueh; /* Our new GUE header */
1043 size_t hdrlen, optlen = 0;
1044 void *data;
1045 bool need_priv = false;
1046
1047 if ((cp->dest->tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM) &&
1048 skb->ip_summed == CHECKSUM_PARTIAL) {
1049 optlen += GUE_PLEN_REMCSUM + GUE_LEN_PRIV;
1050 need_priv = true;
1051 }
1052
1053 hdrlen = sizeof(struct guehdr) + optlen;
1054
1055 skb_push(skb, hdrlen);
1056
1057 gueh = (struct guehdr *)skb->data;
1058
1059 gueh->control = 0;
1060 gueh->version = 0;
1061 gueh->hlen = optlen >> 2;
1062 gueh->flags = 0;
1063 gueh->proto_ctype = *next_protocol;
1064
1065 data = &gueh[1];
1066
1067 if (need_priv) {
1068 __be32 *flags = data;
1069 u16 csum_start = skb_checksum_start_offset(skb);
1070 __be16 *pd;
1071
1072 gueh->flags |= GUE_FLAG_PRIV;
1073 *flags = 0;
1074 data += GUE_LEN_PRIV;
1075
1076 if (csum_start < hdrlen)
1077 return -EINVAL;
1078
1079 csum_start -= hdrlen;
1080 pd = data;
1081 pd[0] = htons(csum_start);
1082 pd[1] = htons(csum_start + skb->csum_offset);
1083
1084 if (!skb_is_gso(skb)) {
1085 skb->ip_summed = CHECKSUM_NONE;
1086 skb->encapsulation = 0;
1087 }
1088
1089 *flags |= GUE_PFLAG_REMCSUM;
1090 data += GUE_PLEN_REMCSUM;
1091 }
1092
1093 skb_push(skb, sizeof(struct udphdr));
1094 skb_reset_transport_header(skb);
1095
1096 udph = udp_hdr(skb);
1097
1098 dport = cp->dest->tun_port;
1099 udph->dest = dport;
1100 udph->source = sport;
1101 udp_set_len(udph, skb->len);
1102 udph->check = 0;
1103
1104 *next_protocol = IPPROTO_UDP;
1105
1106 return 0;
1107 }
1108
1109 static void
ipvs_gre_encap(struct net * net,struct sk_buff * skb,struct ip_vs_conn * cp,__u8 * next_protocol)1110 ipvs_gre_encap(struct net *net, struct sk_buff *skb,
1111 struct ip_vs_conn *cp, __u8 *next_protocol)
1112 {
1113 __be16 proto = *next_protocol == IPPROTO_IPIP ?
1114 htons(ETH_P_IP) : htons(ETH_P_IPV6);
1115 IP_TUNNEL_DECLARE_FLAGS(tflags) = { };
1116 size_t hdrlen;
1117
1118 if (cp->dest->tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM)
1119 __set_bit(IP_TUNNEL_CSUM_BIT, tflags);
1120
1121 hdrlen = gre_calc_hlen(tflags);
1122 gre_build_header(skb, hdrlen, tflags, proto, 0, 0);
1123
1124 *next_protocol = IPPROTO_GRE;
1125 }
1126
1127 /*
1128 * IP Tunneling transmitter
1129 *
1130 * This function encapsulates the packet in a new IP packet, its
1131 * destination will be set to cp->daddr. Most code of this function
1132 * is taken from ipip.c.
1133 *
1134 * It is used in VS/TUN cluster. The load balancer selects a real
1135 * server from a cluster based on a scheduling algorithm,
1136 * encapsulates the request packet and forwards it to the selected
1137 * server. For example, all real servers are configured with
1138 * "ifconfig tunl0 <Virtual IP Address> up". When the server receives
1139 * the encapsulated packet, it will decapsulate the packet, processe
1140 * the request and return the response packets directly to the client
1141 * without passing the load balancer. This can greatly increase the
1142 * scalability of virtual server.
1143 *
1144 * Used for ANY protocol
1145 */
1146 int
ip_vs_tunnel_xmit(struct sk_buff * skb,struct ip_vs_conn * cp,struct ip_vs_protocol * pp,struct ip_vs_iphdr * ipvsh)1147 ip_vs_tunnel_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
1148 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
1149 {
1150 struct netns_ipvs *ipvs = cp->ipvs;
1151 struct net *net = ipvs->net;
1152 struct rtable *rt; /* Route to the other host */
1153 __be32 saddr; /* Source for tunnel */
1154 struct net_device *tdev; /* Device to other host */
1155 __u8 next_protocol = 0;
1156 __u8 dsfield = 0;
1157 __u8 ttl = 0;
1158 __be16 df = 0;
1159 __be16 *dfp = NULL;
1160 struct iphdr *iph; /* Our new IP header */
1161 unsigned int max_headroom; /* The extra header space needed */
1162 int ret, local;
1163 int tun_type, gso_type;
1164 int tun_flags;
1165
1166 local = __ip_vs_get_out_rt(ipvs, cp->af, skb, cp->dest, cp->daddr.ip,
1167 IP_VS_RT_MODE_LOCAL |
1168 IP_VS_RT_MODE_NON_LOCAL |
1169 IP_VS_RT_MODE_CONNECT |
1170 IP_VS_RT_MODE_TUNNEL, &saddr, ipvsh);
1171 if (local < 0)
1172 goto tx_error;
1173 if (local)
1174 return ip_vs_send_or_cont(NFPROTO_IPV4, skb, cp, 1);
1175
1176 rt = skb_rtable(skb);
1177 tdev = rt->dst.dev;
1178
1179 /*
1180 * Okay, now see if we can stuff it in the buffer as-is.
1181 */
1182 max_headroom = LL_RESERVED_SPACE(tdev) + sizeof(struct iphdr);
1183
1184 tun_type = cp->dest->tun_type;
1185 tun_flags = cp->dest->tun_flags;
1186
1187 if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) {
1188 size_t gue_hdrlen, gue_optlen = 0;
1189
1190 if ((tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM) &&
1191 skb->ip_summed == CHECKSUM_PARTIAL) {
1192 gue_optlen += GUE_PLEN_REMCSUM + GUE_LEN_PRIV;
1193 }
1194 gue_hdrlen = sizeof(struct guehdr) + gue_optlen;
1195
1196 max_headroom += sizeof(struct udphdr) + gue_hdrlen;
1197 } else if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GRE) {
1198 IP_TUNNEL_DECLARE_FLAGS(tflags) = { };
1199 size_t gre_hdrlen;
1200
1201 if (tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM)
1202 __set_bit(IP_TUNNEL_CSUM_BIT, tflags);
1203 gre_hdrlen = gre_calc_hlen(tflags);
1204
1205 max_headroom += gre_hdrlen;
1206 }
1207
1208 /* We only care about the df field if sysctl_pmtu_disc(ipvs) is set */
1209 dfp = sysctl_pmtu_disc(ipvs) ? &df : NULL;
1210 skb = ip_vs_prepare_tunneled_skb(skb, cp->af, max_headroom,
1211 &next_protocol, NULL, &dsfield,
1212 &ttl, dfp);
1213 if (IS_ERR(skb))
1214 return NF_STOLEN;
1215
1216 gso_type = __tun_gso_type_mask(AF_INET, cp->af);
1217 if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) {
1218 if ((tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM) ||
1219 (tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM))
1220 gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1221 else
1222 gso_type |= SKB_GSO_UDP_TUNNEL;
1223 if ((tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM) &&
1224 skb->ip_summed == CHECKSUM_PARTIAL) {
1225 gso_type |= SKB_GSO_TUNNEL_REMCSUM;
1226 }
1227 } else if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GRE) {
1228 if (tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM)
1229 gso_type |= SKB_GSO_GRE_CSUM;
1230 else
1231 gso_type |= SKB_GSO_GRE;
1232 }
1233
1234 if (iptunnel_handle_offloads(skb, gso_type))
1235 goto tx_error;
1236
1237 skb->transport_header = skb->network_header;
1238
1239 skb_set_inner_ipproto(skb, next_protocol);
1240 skb_set_inner_mac_header(skb, skb_inner_network_offset(skb));
1241
1242 if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) {
1243 bool check = false;
1244
1245 if (ipvs_gue_encap(net, skb, cp, &next_protocol))
1246 goto tx_error;
1247
1248 if ((tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM) ||
1249 (tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM))
1250 check = true;
1251
1252 udp_set_csum(!check, skb, saddr, cp->daddr.ip, skb->len);
1253 } else if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GRE)
1254 ipvs_gre_encap(net, skb, cp, &next_protocol);
1255
1256 skb_push(skb, sizeof(struct iphdr));
1257 skb_reset_network_header(skb);
1258 memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
1259
1260 /*
1261 * Push down and install the IPIP header.
1262 */
1263 iph = ip_hdr(skb);
1264 iph->version = 4;
1265 iph->ihl = sizeof(struct iphdr)>>2;
1266 iph->frag_off = df;
1267 iph->protocol = next_protocol;
1268 iph->tos = dsfield;
1269 iph->daddr = cp->daddr.ip;
1270 iph->saddr = saddr;
1271 iph->ttl = ttl;
1272 ip_select_ident(net, skb, NULL);
1273
1274 /* Another hack: avoid icmp_send in ip_fragment */
1275 skb->ignore_df = 1;
1276
1277 ret = ip_vs_tunnel_xmit_prepare(skb, cp);
1278 if (ret == NF_ACCEPT)
1279 ip_local_out(net, skb->sk, skb);
1280 else if (ret == NF_DROP)
1281 kfree_skb(skb);
1282
1283 return NF_STOLEN;
1284
1285 tx_error:
1286 kfree_skb(skb);
1287 return NF_STOLEN;
1288 }
1289
1290 #ifdef CONFIG_IP_VS_IPV6
1291 int
ip_vs_tunnel_xmit_v6(struct sk_buff * skb,struct ip_vs_conn * cp,struct ip_vs_protocol * pp,struct ip_vs_iphdr * ipvsh)1292 ip_vs_tunnel_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
1293 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
1294 {
1295 struct netns_ipvs *ipvs = cp->ipvs;
1296 struct net *net = ipvs->net;
1297 struct rt6_info *rt; /* Route to the other host */
1298 struct in6_addr saddr; /* Source for tunnel */
1299 struct net_device *tdev; /* Device to other host */
1300 __u8 next_protocol = 0;
1301 __u32 payload_len = 0;
1302 __u8 dsfield = 0;
1303 __u8 ttl = 0;
1304 struct ipv6hdr *iph; /* Our new IP header */
1305 unsigned int max_headroom; /* The extra header space needed */
1306 int ret, local;
1307 int tun_type, gso_type;
1308 int tun_flags;
1309
1310 local = __ip_vs_get_out_rt_v6(ipvs, cp->af, skb, cp->dest,
1311 &cp->daddr.in6,
1312 &saddr, ipvsh, 1,
1313 IP_VS_RT_MODE_LOCAL |
1314 IP_VS_RT_MODE_NON_LOCAL |
1315 IP_VS_RT_MODE_TUNNEL);
1316 if (local < 0)
1317 goto tx_error;
1318 if (local)
1319 return ip_vs_send_or_cont(NFPROTO_IPV6, skb, cp, 1);
1320
1321 rt = dst_rt6_info(skb_dst(skb));
1322 tdev = rt->dst.dev;
1323
1324 /*
1325 * Okay, now see if we can stuff it in the buffer as-is.
1326 */
1327 max_headroom = LL_RESERVED_SPACE(tdev) + sizeof(struct ipv6hdr);
1328
1329 tun_type = cp->dest->tun_type;
1330 tun_flags = cp->dest->tun_flags;
1331
1332 if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) {
1333 size_t gue_hdrlen, gue_optlen = 0;
1334
1335 if ((tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM) &&
1336 skb->ip_summed == CHECKSUM_PARTIAL) {
1337 gue_optlen += GUE_PLEN_REMCSUM + GUE_LEN_PRIV;
1338 }
1339 gue_hdrlen = sizeof(struct guehdr) + gue_optlen;
1340
1341 max_headroom += sizeof(struct udphdr) + gue_hdrlen;
1342 } else if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GRE) {
1343 IP_TUNNEL_DECLARE_FLAGS(tflags) = { };
1344 size_t gre_hdrlen;
1345
1346 if (tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM)
1347 __set_bit(IP_TUNNEL_CSUM_BIT, tflags);
1348 gre_hdrlen = gre_calc_hlen(tflags);
1349
1350 max_headroom += gre_hdrlen;
1351 }
1352
1353 skb = ip_vs_prepare_tunneled_skb(skb, cp->af, max_headroom,
1354 &next_protocol, &payload_len,
1355 &dsfield, &ttl, NULL);
1356 if (IS_ERR(skb))
1357 return NF_STOLEN;
1358
1359 gso_type = __tun_gso_type_mask(AF_INET6, cp->af);
1360 if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) {
1361 if ((tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM) ||
1362 (tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM))
1363 gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1364 else
1365 gso_type |= SKB_GSO_UDP_TUNNEL;
1366 if ((tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM) &&
1367 skb->ip_summed == CHECKSUM_PARTIAL) {
1368 gso_type |= SKB_GSO_TUNNEL_REMCSUM;
1369 }
1370 } else if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GRE) {
1371 if (tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM)
1372 gso_type |= SKB_GSO_GRE_CSUM;
1373 else
1374 gso_type |= SKB_GSO_GRE;
1375 }
1376
1377 if (iptunnel_handle_offloads(skb, gso_type))
1378 goto tx_error;
1379
1380 skb->transport_header = skb->network_header;
1381
1382 skb_set_inner_ipproto(skb, next_protocol);
1383 skb_set_inner_mac_header(skb, skb_inner_network_offset(skb));
1384
1385 if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) {
1386 bool check = false;
1387
1388 if (ipvs_gue_encap(net, skb, cp, &next_protocol))
1389 goto tx_error;
1390
1391 if ((tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM) ||
1392 (tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM))
1393 check = true;
1394
1395 udp6_set_csum(!check, skb, &saddr, &cp->daddr.in6, skb->len);
1396 } else if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GRE)
1397 ipvs_gre_encap(net, skb, cp, &next_protocol);
1398
1399 skb_push(skb, sizeof(struct ipv6hdr));
1400 skb_reset_network_header(skb);
1401 memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
1402
1403 /*
1404 * Push down and install the IPIP header.
1405 */
1406 iph = ipv6_hdr(skb);
1407 iph->version = 6;
1408 iph->nexthdr = next_protocol;
1409 iph->payload_len = htons(payload_len);
1410 memset(&iph->flow_lbl, 0, sizeof(iph->flow_lbl));
1411 ipv6_change_dsfield(iph, 0, dsfield);
1412 iph->daddr = cp->daddr.in6;
1413 iph->saddr = saddr;
1414 iph->hop_limit = ttl;
1415
1416 /* Another hack: avoid icmp_send in ip_fragment */
1417 skb->ignore_df = 1;
1418
1419 ret = ip_vs_tunnel_xmit_prepare(skb, cp);
1420 if (ret == NF_ACCEPT)
1421 ip6_local_out(net, skb->sk, skb);
1422 else if (ret == NF_DROP)
1423 kfree_skb(skb);
1424
1425 return NF_STOLEN;
1426
1427 tx_error:
1428 kfree_skb(skb);
1429 return NF_STOLEN;
1430 }
1431 #endif
1432
1433
1434 /*
1435 * Direct Routing transmitter
1436 * Used for ANY protocol
1437 */
1438 int
ip_vs_dr_xmit(struct sk_buff * skb,struct ip_vs_conn * cp,struct ip_vs_protocol * pp,struct ip_vs_iphdr * ipvsh)1439 ip_vs_dr_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
1440 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
1441 {
1442 int local;
1443
1444 local = __ip_vs_get_out_rt(cp->ipvs, cp->af, skb, cp->dest, cp->daddr.ip,
1445 IP_VS_RT_MODE_LOCAL |
1446 IP_VS_RT_MODE_NON_LOCAL |
1447 IP_VS_RT_MODE_KNOWN_NH, NULL, ipvsh);
1448 if (local < 0)
1449 goto tx_error;
1450 if (local)
1451 return ip_vs_send_or_cont(NFPROTO_IPV4, skb, cp, 1);
1452
1453 ip_send_check(ip_hdr(skb));
1454
1455 /* Another hack: avoid icmp_send in ip_fragment */
1456 skb->ignore_df = 1;
1457
1458 ip_vs_send_or_cont(NFPROTO_IPV4, skb, cp, 0);
1459
1460 return NF_STOLEN;
1461
1462 tx_error:
1463 kfree_skb(skb);
1464 return NF_STOLEN;
1465 }
1466
1467 #ifdef CONFIG_IP_VS_IPV6
1468 int
ip_vs_dr_xmit_v6(struct sk_buff * skb,struct ip_vs_conn * cp,struct ip_vs_protocol * pp,struct ip_vs_iphdr * ipvsh)1469 ip_vs_dr_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
1470 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
1471 {
1472 int local;
1473
1474 local = __ip_vs_get_out_rt_v6(cp->ipvs, cp->af, skb, cp->dest,
1475 &cp->daddr.in6,
1476 NULL, ipvsh, 0,
1477 IP_VS_RT_MODE_LOCAL |
1478 IP_VS_RT_MODE_NON_LOCAL |
1479 IP_VS_RT_MODE_KNOWN_NH);
1480 if (local < 0)
1481 goto tx_error;
1482 if (local)
1483 return ip_vs_send_or_cont(NFPROTO_IPV6, skb, cp, 1);
1484
1485 /* Another hack: avoid icmp_send in ip_fragment */
1486 skb->ignore_df = 1;
1487
1488 ip_vs_send_or_cont(NFPROTO_IPV6, skb, cp, 0);
1489
1490 return NF_STOLEN;
1491
1492 tx_error:
1493 kfree_skb(skb);
1494 return NF_STOLEN;
1495 }
1496 #endif
1497
1498
1499 /*
1500 * ICMP packet transmitter
1501 * called by the ip_vs_in_icmp
1502 */
1503 int
ip_vs_icmp_xmit(struct sk_buff * skb,struct ip_vs_conn * cp,struct ip_vs_protocol * pp,unsigned int toff,unsigned int hooknum,struct ip_vs_iphdr * ciph)1504 ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
1505 struct ip_vs_protocol *pp, unsigned int toff,
1506 unsigned int hooknum, struct ip_vs_iphdr *ciph)
1507 {
1508 struct rtable *rt; /* Route to the other host */
1509 int rc;
1510 int local;
1511 int rt_mode, was_input;
1512 bool has_ports = false;
1513 unsigned int wlen;
1514
1515 /* The ICMP packet for VS/TUN, VS/DR and LOCALNODE will be
1516 forwarded directly here, because there is no need to
1517 translate address/port back */
1518 if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) {
1519 if (cp->packet_xmit)
1520 rc = cp->packet_xmit(skb, cp, pp, ciph);
1521 else
1522 rc = NF_ACCEPT;
1523 /* do not touch skb anymore */
1524 atomic_inc(&cp->in_pkts);
1525 return rc;
1526 }
1527
1528 /*
1529 * mangle and send the packet here (only for VS/NAT)
1530 */
1531 was_input = rt_is_input_route(skb_rtable(skb));
1532
1533 /* LOCALNODE from FORWARD hook is not supported */
1534 rt_mode = (hooknum != NF_INET_FORWARD) ?
1535 IP_VS_RT_MODE_LOCAL | IP_VS_RT_MODE_NON_LOCAL |
1536 IP_VS_RT_MODE_RDR : IP_VS_RT_MODE_NON_LOCAL;
1537 local = __ip_vs_get_out_rt(cp->ipvs, cp->af, skb, cp->dest, cp->daddr.ip, rt_mode,
1538 NULL, ciph);
1539 if (local < 0)
1540 goto tx_error;
1541 rt = skb_rtable(skb);
1542
1543 /*
1544 * Avoid duplicate tuple in reply direction for NAT traffic
1545 * to local address when connection is sync-ed
1546 */
1547 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
1548 if (cp->flags & IP_VS_CONN_F_SYNC && local) {
1549 enum ip_conntrack_info ctinfo;
1550 struct nf_conn *ct = nf_ct_get(skb, &ctinfo);
1551
1552 if (ct) {
1553 IP_VS_DBG(10, "%s(): "
1554 "stopping DNAT to local address %pI4\n",
1555 __func__, &cp->daddr.ip);
1556 goto tx_error;
1557 }
1558 }
1559 #endif
1560
1561 /* From world but DNAT to loopback address? */
1562 if (local && ipv4_is_loopback(cp->daddr.ip) && was_input) {
1563 IP_VS_DBG(1, "%s(): "
1564 "stopping DNAT to loopback %pI4\n",
1565 __func__, &cp->daddr.ip);
1566 goto tx_error;
1567 }
1568
1569 wlen = ciph->len;
1570 if (ciph->protocol == IPPROTO_TCP || ciph->protocol == IPPROTO_UDP ||
1571 ciph->protocol == IPPROTO_SCTP) {
1572 wlen += 2 * sizeof(__u16); /* Also mangle ports */
1573 has_ports = true;
1574 }
1575
1576 /* copy-on-write the packet before mangling it */
1577 if (skb_ensure_writable(skb, wlen))
1578 goto tx_error;
1579
1580 if (skb_cow(skb, rt->dst.dev->hard_header_len))
1581 goto tx_error;
1582
1583 if (!ip_vs_nat_icmp(skb, pp, cp, 0, toff, has_ports, ciph))
1584 goto tx_error;
1585
1586 /* Another hack: avoid icmp_send in ip_fragment */
1587 skb->ignore_df = 1;
1588
1589 return ip_vs_nat_send_or_cont(NFPROTO_IPV4, skb, cp, local);
1590
1591 tx_error:
1592 kfree_skb(skb);
1593 rc = NF_STOLEN;
1594 return rc;
1595 }
1596
1597 #ifdef CONFIG_IP_VS_IPV6
1598 int
ip_vs_icmp_xmit_v6(struct sk_buff * skb,struct ip_vs_conn * cp,struct ip_vs_protocol * pp,unsigned int toff,unsigned int hooknum,struct ip_vs_iphdr * ciph)1599 ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
1600 struct ip_vs_protocol *pp, unsigned int toff,
1601 unsigned int hooknum, struct ip_vs_iphdr *ciph)
1602 {
1603 bool has_ports = false;
1604 struct rt6_info *rt; /* Route to the other host */
1605 unsigned int wlen;
1606 int rc;
1607 int local;
1608 int rt_mode;
1609
1610 /* The ICMP packet for VS/TUN, VS/DR and LOCALNODE will be
1611 forwarded directly here, because there is no need to
1612 translate address/port back */
1613 if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) {
1614 if (cp->packet_xmit)
1615 rc = cp->packet_xmit(skb, cp, pp, ciph);
1616 else
1617 rc = NF_ACCEPT;
1618 /* do not touch skb anymore */
1619 atomic_inc(&cp->in_pkts);
1620 return rc;
1621 }
1622
1623 /*
1624 * mangle and send the packet here (only for VS/NAT)
1625 */
1626
1627 /* LOCALNODE from FORWARD hook is not supported */
1628 rt_mode = (hooknum != NF_INET_FORWARD) ?
1629 IP_VS_RT_MODE_LOCAL | IP_VS_RT_MODE_NON_LOCAL |
1630 IP_VS_RT_MODE_RDR : IP_VS_RT_MODE_NON_LOCAL;
1631 local = __ip_vs_get_out_rt_v6(cp->ipvs, cp->af, skb, cp->dest,
1632 &cp->daddr.in6, NULL, ciph, 0, rt_mode);
1633 if (local < 0)
1634 goto tx_error;
1635 rt = dst_rt6_info(skb_dst(skb));
1636 /*
1637 * Avoid duplicate tuple in reply direction for NAT traffic
1638 * to local address when connection is sync-ed
1639 */
1640 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
1641 if (cp->flags & IP_VS_CONN_F_SYNC && local) {
1642 enum ip_conntrack_info ctinfo;
1643 struct nf_conn *ct = nf_ct_get(skb, &ctinfo);
1644
1645 if (ct) {
1646 IP_VS_DBG(10, "%s(): "
1647 "stopping DNAT to local address %pI6\n",
1648 __func__, &cp->daddr.in6);
1649 goto tx_error;
1650 }
1651 }
1652 #endif
1653
1654 /* From world but DNAT to loopback address? */
1655 if (local && skb->dev && !(skb->dev->flags & IFF_LOOPBACK) &&
1656 ipv6_addr_type(&cp->daddr.in6) & IPV6_ADDR_LOOPBACK) {
1657 IP_VS_DBG(1, "%s(): "
1658 "stopping DNAT to loopback %pI6\n",
1659 __func__, &cp->daddr.in6);
1660 goto tx_error;
1661 }
1662
1663 wlen = ciph->len;
1664 if (ciph->protocol == IPPROTO_TCP || ciph->protocol == IPPROTO_UDP ||
1665 ciph->protocol == IPPROTO_SCTP) {
1666 wlen += 2 * sizeof(__u16); /* Also mangle ports */
1667 has_ports = true;
1668 }
1669
1670 /* copy-on-write the packet before mangling it */
1671 if (skb_ensure_writable(skb, wlen))
1672 goto tx_error;
1673
1674 if (skb_cow(skb, rt->dst.dev->hard_header_len))
1675 goto tx_error;
1676
1677 ip_vs_nat_icmp_v6(skb, pp, cp, 0, toff, has_ports, ciph);
1678
1679 /* Another hack: avoid icmp_send in ip_fragment */
1680 skb->ignore_df = 1;
1681
1682 return ip_vs_nat_send_or_cont(NFPROTO_IPV6, skb, cp, local);
1683
1684 tx_error:
1685 kfree_skb(skb);
1686 rc = NF_STOLEN;
1687 return rc;
1688 }
1689 #endif
1690