xref: /linux/net/ipv4/fou_core.c (revision 07fdad3a93756b872da7b53647715c48d0f4a2d0)
1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/module.h>
3 #include <linux/errno.h>
4 #include <linux/socket.h>
5 #include <linux/skbuff.h>
6 #include <linux/ip.h>
7 #include <linux/icmp.h>
8 #include <linux/udp.h>
9 #include <linux/types.h>
10 #include <linux/kernel.h>
11 #include <net/genetlink.h>
12 #include <net/gro.h>
13 #include <net/gue.h>
14 #include <net/fou.h>
15 #include <net/ip.h>
16 #include <net/protocol.h>
17 #include <net/udp.h>
18 #include <net/udp_tunnel.h>
19 #include <uapi/linux/fou.h>
20 #include <uapi/linux/genetlink.h>
21 
22 #include "fou_nl.h"
23 
24 struct fou {
25 	struct socket *sock;
26 	u8 protocol;
27 	u8 flags;
28 	__be16 port;
29 	u8 family;
30 	u16 type;
31 	struct list_head list;
32 	struct rcu_head rcu;
33 };
34 
35 #define FOU_F_REMCSUM_NOPARTIAL BIT(0)
36 
37 struct fou_cfg {
38 	u16 type;
39 	u8 protocol;
40 	u8 flags;
41 	struct udp_port_cfg udp_config;
42 };
43 
44 static unsigned int fou_net_id;
45 
46 struct fou_net {
47 	struct list_head fou_list;
48 	struct mutex fou_lock;
49 };
50 
51 static inline struct fou *fou_from_sock(struct sock *sk)
52 {
53 	return rcu_dereference_sk_user_data(sk);
54 }
55 
56 static int fou_recv_pull(struct sk_buff *skb, struct fou *fou, size_t len)
57 {
58 	/* Remove 'len' bytes from the packet (UDP header and
59 	 * FOU header if present).
60 	 */
61 	if (fou->family == AF_INET)
62 		ip_hdr(skb)->tot_len = htons(ntohs(ip_hdr(skb)->tot_len) - len);
63 	else
64 		ipv6_hdr(skb)->payload_len =
65 		    htons(ntohs(ipv6_hdr(skb)->payload_len) - len);
66 
67 	__skb_pull(skb, len);
68 	skb_postpull_rcsum(skb, udp_hdr(skb), len);
69 	skb_reset_transport_header(skb);
70 	return iptunnel_pull_offloads(skb);
71 }
72 
73 static int fou_udp_recv(struct sock *sk, struct sk_buff *skb)
74 {
75 	struct fou *fou = fou_from_sock(sk);
76 
77 	if (!fou)
78 		return 1;
79 
80 	if (fou_recv_pull(skb, fou, sizeof(struct udphdr)))
81 		goto drop;
82 
83 	return -fou->protocol;
84 
85 drop:
86 	kfree_skb(skb);
87 	return 0;
88 }
89 
90 static struct guehdr *gue_remcsum(struct sk_buff *skb, struct guehdr *guehdr,
91 				  void *data, size_t hdrlen, u8 ipproto,
92 				  bool nopartial)
93 {
94 	__be16 *pd = data;
95 	size_t start = ntohs(pd[0]);
96 	size_t offset = ntohs(pd[1]);
97 	size_t plen = sizeof(struct udphdr) + hdrlen +
98 	    max_t(size_t, offset + sizeof(u16), start);
99 
100 	if (skb->remcsum_offload)
101 		return guehdr;
102 
103 	if (!pskb_may_pull(skb, plen))
104 		return NULL;
105 	guehdr = (struct guehdr *)&udp_hdr(skb)[1];
106 
107 	skb_remcsum_process(skb, (void *)guehdr + hdrlen,
108 			    start, offset, nopartial);
109 
110 	return guehdr;
111 }
112 
113 static int gue_control_message(struct sk_buff *skb, struct guehdr *guehdr)
114 {
115 	/* No support yet */
116 	kfree_skb(skb);
117 	return 0;
118 }
119 
120 static int gue_udp_recv(struct sock *sk, struct sk_buff *skb)
121 {
122 	struct fou *fou = fou_from_sock(sk);
123 	size_t len, optlen, hdrlen;
124 	struct guehdr *guehdr;
125 	void *data;
126 	u16 doffset = 0;
127 	u8 proto_ctype;
128 
129 	if (!fou)
130 		return 1;
131 
132 	len = sizeof(struct udphdr) + sizeof(struct guehdr);
133 	if (!pskb_may_pull(skb, len))
134 		goto drop;
135 
136 	guehdr = (struct guehdr *)&udp_hdr(skb)[1];
137 
138 	switch (guehdr->version) {
139 	case 0: /* Full GUE header present */
140 		break;
141 
142 	case 1: {
143 		/* Direct encapsulation of IPv4 or IPv6 */
144 
145 		int prot;
146 
147 		switch (((struct iphdr *)guehdr)->version) {
148 		case 4:
149 			prot = IPPROTO_IPIP;
150 			break;
151 		case 6:
152 			prot = IPPROTO_IPV6;
153 			break;
154 		default:
155 			goto drop;
156 		}
157 
158 		if (fou_recv_pull(skb, fou, sizeof(struct udphdr)))
159 			goto drop;
160 
161 		return -prot;
162 	}
163 
164 	default: /* Undefined version */
165 		goto drop;
166 	}
167 
168 	optlen = guehdr->hlen << 2;
169 	len += optlen;
170 
171 	if (!pskb_may_pull(skb, len))
172 		goto drop;
173 
174 	/* guehdr may change after pull */
175 	guehdr = (struct guehdr *)&udp_hdr(skb)[1];
176 
177 	if (validate_gue_flags(guehdr, optlen))
178 		goto drop;
179 
180 	hdrlen = sizeof(struct guehdr) + optlen;
181 
182 	if (fou->family == AF_INET)
183 		ip_hdr(skb)->tot_len = htons(ntohs(ip_hdr(skb)->tot_len) - len);
184 	else
185 		ipv6_hdr(skb)->payload_len =
186 		    htons(ntohs(ipv6_hdr(skb)->payload_len) - len);
187 
188 	/* Pull csum through the guehdr now . This can be used if
189 	 * there is a remote checksum offload.
190 	 */
191 	skb_postpull_rcsum(skb, udp_hdr(skb), len);
192 
193 	data = &guehdr[1];
194 
195 	if (guehdr->flags & GUE_FLAG_PRIV) {
196 		__be32 flags = *(__be32 *)(data + doffset);
197 
198 		doffset += GUE_LEN_PRIV;
199 
200 		if (flags & GUE_PFLAG_REMCSUM) {
201 			guehdr = gue_remcsum(skb, guehdr, data + doffset,
202 					     hdrlen, guehdr->proto_ctype,
203 					     !!(fou->flags &
204 						FOU_F_REMCSUM_NOPARTIAL));
205 			if (!guehdr)
206 				goto drop;
207 
208 			data = &guehdr[1];
209 
210 			doffset += GUE_PLEN_REMCSUM;
211 		}
212 	}
213 
214 	if (unlikely(guehdr->control))
215 		return gue_control_message(skb, guehdr);
216 
217 	proto_ctype = guehdr->proto_ctype;
218 	__skb_pull(skb, sizeof(struct udphdr) + hdrlen);
219 	skb_reset_transport_header(skb);
220 
221 	if (iptunnel_pull_offloads(skb))
222 		goto drop;
223 
224 	return -proto_ctype;
225 
226 drop:
227 	kfree_skb(skb);
228 	return 0;
229 }
230 
231 static const struct net_offload *fou_gro_ops(const struct sock *sk,
232 					     int proto)
233 {
234 	const struct net_offload __rcu **offloads;
235 
236 	/* FOU doesn't allow IPv4 on IPv6 sockets. */
237 	offloads = sk->sk_family == AF_INET6 ? inet6_offloads : inet_offloads;
238 	return rcu_dereference(offloads[proto]);
239 }
240 
241 static struct sk_buff *fou_gro_receive(struct sock *sk,
242 				       struct list_head *head,
243 				       struct sk_buff *skb)
244 {
245 	struct fou *fou = fou_from_sock(sk);
246 	const struct net_offload *ops;
247 	struct sk_buff *pp = NULL;
248 
249 	if (!fou)
250 		goto out;
251 
252 	/* We can clear the encap_mark for FOU as we are essentially doing
253 	 * one of two possible things.  We are either adding an L4 tunnel
254 	 * header to the outer L3 tunnel header, or we are simply
255 	 * treating the GRE tunnel header as though it is a UDP protocol
256 	 * specific header such as VXLAN or GENEVE.
257 	 */
258 	NAPI_GRO_CB(skb)->encap_mark = 0;
259 
260 	/* Flag this frame as already having an outer encap header */
261 	NAPI_GRO_CB(skb)->is_fou = 1;
262 
263 	ops = fou_gro_ops(sk, fou->protocol);
264 	if (!ops || !ops->callbacks.gro_receive)
265 		goto out;
266 
267 	pp = call_gro_receive(ops->callbacks.gro_receive, head, skb);
268 
269 out:
270 	return pp;
271 }
272 
273 static int fou_gro_complete(struct sock *sk, struct sk_buff *skb,
274 			    int nhoff)
275 {
276 	struct fou *fou = fou_from_sock(sk);
277 	const struct net_offload *ops;
278 	int err;
279 
280 	if (!fou) {
281 		err = -ENOENT;
282 		goto out;
283 	}
284 
285 	ops = fou_gro_ops(sk, fou->protocol);
286 	if (WARN_ON(!ops || !ops->callbacks.gro_complete)) {
287 		err = -ENOSYS;
288 		goto out;
289 	}
290 
291 	err = ops->callbacks.gro_complete(skb, nhoff);
292 
293 	skb_set_inner_mac_header(skb, nhoff);
294 
295 out:
296 	return err;
297 }
298 
299 static struct guehdr *gue_gro_remcsum(struct sk_buff *skb, unsigned int off,
300 				      struct guehdr *guehdr, void *data,
301 				      size_t hdrlen, struct gro_remcsum *grc,
302 				      bool nopartial)
303 {
304 	__be16 *pd = data;
305 	size_t start = ntohs(pd[0]);
306 	size_t offset = ntohs(pd[1]);
307 
308 	if (skb->remcsum_offload)
309 		return guehdr;
310 
311 	if (!NAPI_GRO_CB(skb)->csum_valid)
312 		return NULL;
313 
314 	guehdr = skb_gro_remcsum_process(skb, (void *)guehdr, off, hdrlen,
315 					 start, offset, grc, nopartial);
316 
317 	skb->remcsum_offload = 1;
318 
319 	return guehdr;
320 }
321 
322 static struct sk_buff *gue_gro_receive(struct sock *sk,
323 				       struct list_head *head,
324 				       struct sk_buff *skb)
325 {
326 	const struct net_offload *ops;
327 	struct sk_buff *pp = NULL;
328 	struct sk_buff *p;
329 	struct guehdr *guehdr;
330 	size_t len, optlen, hdrlen, off;
331 	void *data;
332 	u16 doffset = 0;
333 	int flush = 1;
334 	struct fou *fou = fou_from_sock(sk);
335 	struct gro_remcsum grc;
336 	u8 proto;
337 
338 	skb_gro_remcsum_init(&grc);
339 
340 	if (!fou)
341 		goto out;
342 
343 	off = skb_gro_offset(skb);
344 	len = off + sizeof(*guehdr);
345 
346 	guehdr = skb_gro_header(skb, len, off);
347 	if (unlikely(!guehdr))
348 		goto out;
349 
350 	switch (guehdr->version) {
351 	case 0:
352 		break;
353 	case 1:
354 		switch (((struct iphdr *)guehdr)->version) {
355 		case 4:
356 			proto = IPPROTO_IPIP;
357 			break;
358 		case 6:
359 			proto = IPPROTO_IPV6;
360 			break;
361 		default:
362 			goto out;
363 		}
364 		goto next_proto;
365 	default:
366 		goto out;
367 	}
368 
369 	optlen = guehdr->hlen << 2;
370 	len += optlen;
371 
372 	if (!skb_gro_may_pull(skb, len)) {
373 		guehdr = skb_gro_header_slow(skb, len, off);
374 		if (unlikely(!guehdr))
375 			goto out;
376 	}
377 
378 	if (unlikely(guehdr->control) || guehdr->version != 0 ||
379 	    validate_gue_flags(guehdr, optlen))
380 		goto out;
381 
382 	hdrlen = sizeof(*guehdr) + optlen;
383 
384 	/* Adjust NAPI_GRO_CB(skb)->csum to account for guehdr,
385 	 * this is needed if there is a remote checkcsum offload.
386 	 */
387 	skb_gro_postpull_rcsum(skb, guehdr, hdrlen);
388 
389 	data = &guehdr[1];
390 
391 	if (guehdr->flags & GUE_FLAG_PRIV) {
392 		__be32 flags = *(__be32 *)(data + doffset);
393 
394 		doffset += GUE_LEN_PRIV;
395 
396 		if (flags & GUE_PFLAG_REMCSUM) {
397 			guehdr = gue_gro_remcsum(skb, off, guehdr,
398 						 data + doffset, hdrlen, &grc,
399 						 !!(fou->flags &
400 						    FOU_F_REMCSUM_NOPARTIAL));
401 
402 			if (!guehdr)
403 				goto out;
404 
405 			data = &guehdr[1];
406 
407 			doffset += GUE_PLEN_REMCSUM;
408 		}
409 	}
410 
411 	skb_gro_pull(skb, hdrlen);
412 
413 	list_for_each_entry(p, head, list) {
414 		const struct guehdr *guehdr2;
415 
416 		if (!NAPI_GRO_CB(p)->same_flow)
417 			continue;
418 
419 		guehdr2 = (struct guehdr *)(p->data + off);
420 
421 		/* Compare base GUE header to be equal (covers
422 		 * hlen, version, proto_ctype, and flags.
423 		 */
424 		if (guehdr->word != guehdr2->word) {
425 			NAPI_GRO_CB(p)->same_flow = 0;
426 			continue;
427 		}
428 
429 		/* Compare optional fields are the same. */
430 		if (guehdr->hlen && memcmp(&guehdr[1], &guehdr2[1],
431 					   guehdr->hlen << 2)) {
432 			NAPI_GRO_CB(p)->same_flow = 0;
433 			continue;
434 		}
435 	}
436 
437 	proto = guehdr->proto_ctype;
438 
439 next_proto:
440 
441 	/* We can clear the encap_mark for GUE as we are essentially doing
442 	 * one of two possible things.  We are either adding an L4 tunnel
443 	 * header to the outer L3 tunnel header, or we are simply
444 	 * treating the GRE tunnel header as though it is a UDP protocol
445 	 * specific header such as VXLAN or GENEVE.
446 	 */
447 	NAPI_GRO_CB(skb)->encap_mark = 0;
448 
449 	/* Flag this frame as already having an outer encap header */
450 	NAPI_GRO_CB(skb)->is_fou = 1;
451 
452 	ops = fou_gro_ops(sk, proto);
453 	if (!ops || !ops->callbacks.gro_receive)
454 		goto out;
455 
456 	pp = call_gro_receive(ops->callbacks.gro_receive, head, skb);
457 	flush = 0;
458 
459 out:
460 	skb_gro_flush_final_remcsum(skb, pp, flush, &grc);
461 
462 	return pp;
463 }
464 
465 static int gue_gro_complete(struct sock *sk, struct sk_buff *skb, int nhoff)
466 {
467 	struct guehdr *guehdr = (struct guehdr *)(skb->data + nhoff);
468 	const struct net_offload *ops;
469 	unsigned int guehlen = 0;
470 	u8 proto;
471 	int err = -ENOENT;
472 
473 	switch (guehdr->version) {
474 	case 0:
475 		proto = guehdr->proto_ctype;
476 		guehlen = sizeof(*guehdr) + (guehdr->hlen << 2);
477 		break;
478 	case 1:
479 		switch (((struct iphdr *)guehdr)->version) {
480 		case 4:
481 			proto = IPPROTO_IPIP;
482 			break;
483 		case 6:
484 			proto = IPPROTO_IPV6;
485 			break;
486 		default:
487 			return err;
488 		}
489 		break;
490 	default:
491 		return err;
492 	}
493 
494 	ops = fou_gro_ops(sk, proto);
495 	if (WARN_ON(!ops || !ops->callbacks.gro_complete))
496 		goto out;
497 
498 	err = ops->callbacks.gro_complete(skb, nhoff + guehlen);
499 
500 	skb_set_inner_mac_header(skb, nhoff + guehlen);
501 
502 out:
503 	return err;
504 }
505 
506 static bool fou_cfg_cmp(struct fou *fou, struct fou_cfg *cfg)
507 {
508 	struct sock *sk = fou->sock->sk;
509 	struct udp_port_cfg *udp_cfg = &cfg->udp_config;
510 
511 	if (fou->family != udp_cfg->family ||
512 	    fou->port != udp_cfg->local_udp_port ||
513 	    sk->sk_dport != udp_cfg->peer_udp_port ||
514 	    sk->sk_bound_dev_if != udp_cfg->bind_ifindex)
515 		return false;
516 
517 	if (fou->family == AF_INET) {
518 		if (sk->sk_rcv_saddr != udp_cfg->local_ip.s_addr ||
519 		    sk->sk_daddr != udp_cfg->peer_ip.s_addr)
520 			return false;
521 		else
522 			return true;
523 #if IS_ENABLED(CONFIG_IPV6)
524 	} else {
525 		if (ipv6_addr_cmp(&sk->sk_v6_rcv_saddr, &udp_cfg->local_ip6) ||
526 		    ipv6_addr_cmp(&sk->sk_v6_daddr, &udp_cfg->peer_ip6))
527 			return false;
528 		else
529 			return true;
530 #endif
531 	}
532 
533 	return false;
534 }
535 
536 static int fou_add_to_port_list(struct net *net, struct fou *fou,
537 				struct fou_cfg *cfg)
538 {
539 	struct fou_net *fn = net_generic(net, fou_net_id);
540 	struct fou *fout;
541 
542 	mutex_lock(&fn->fou_lock);
543 	list_for_each_entry(fout, &fn->fou_list, list) {
544 		if (fou_cfg_cmp(fout, cfg)) {
545 			mutex_unlock(&fn->fou_lock);
546 			return -EALREADY;
547 		}
548 	}
549 
550 	list_add(&fou->list, &fn->fou_list);
551 	mutex_unlock(&fn->fou_lock);
552 
553 	return 0;
554 }
555 
556 static void fou_release(struct fou *fou)
557 {
558 	struct socket *sock = fou->sock;
559 
560 	list_del(&fou->list);
561 	udp_tunnel_sock_release(sock);
562 
563 	kfree_rcu(fou, rcu);
564 }
565 
566 static int fou_create(struct net *net, struct fou_cfg *cfg,
567 		      struct socket **sockp)
568 {
569 	struct socket *sock = NULL;
570 	struct fou *fou = NULL;
571 	struct sock *sk;
572 	struct udp_tunnel_sock_cfg tunnel_cfg;
573 	int err;
574 
575 	/* Open UDP socket */
576 	err = udp_sock_create(net, &cfg->udp_config, &sock);
577 	if (err < 0)
578 		goto error;
579 
580 	/* Allocate FOU port structure */
581 	fou = kzalloc(sizeof(*fou), GFP_KERNEL);
582 	if (!fou) {
583 		err = -ENOMEM;
584 		goto error;
585 	}
586 
587 	sk = sock->sk;
588 
589 	fou->port = cfg->udp_config.local_udp_port;
590 	fou->family = cfg->udp_config.family;
591 	fou->flags = cfg->flags;
592 	fou->type = cfg->type;
593 	fou->sock = sock;
594 
595 	memset(&tunnel_cfg, 0, sizeof(tunnel_cfg));
596 	tunnel_cfg.encap_type = 1;
597 	tunnel_cfg.sk_user_data = fou;
598 	tunnel_cfg.encap_destroy = NULL;
599 
600 	/* Initial for fou type */
601 	switch (cfg->type) {
602 	case FOU_ENCAP_DIRECT:
603 		tunnel_cfg.encap_rcv = fou_udp_recv;
604 		tunnel_cfg.gro_receive = fou_gro_receive;
605 		tunnel_cfg.gro_complete = fou_gro_complete;
606 		fou->protocol = cfg->protocol;
607 		break;
608 	case FOU_ENCAP_GUE:
609 		tunnel_cfg.encap_rcv = gue_udp_recv;
610 		tunnel_cfg.gro_receive = gue_gro_receive;
611 		tunnel_cfg.gro_complete = gue_gro_complete;
612 		break;
613 	default:
614 		err = -EINVAL;
615 		goto error;
616 	}
617 
618 	setup_udp_tunnel_sock(net, sock, &tunnel_cfg);
619 
620 	sk->sk_allocation = GFP_ATOMIC;
621 
622 	err = fou_add_to_port_list(net, fou, cfg);
623 	if (err)
624 		goto error;
625 
626 	if (sockp)
627 		*sockp = sock;
628 
629 	return 0;
630 
631 error:
632 	kfree(fou);
633 	if (sock)
634 		udp_tunnel_sock_release(sock);
635 
636 	return err;
637 }
638 
639 static int fou_destroy(struct net *net, struct fou_cfg *cfg)
640 {
641 	struct fou_net *fn = net_generic(net, fou_net_id);
642 	int err = -EINVAL;
643 	struct fou *fou;
644 
645 	mutex_lock(&fn->fou_lock);
646 	list_for_each_entry(fou, &fn->fou_list, list) {
647 		if (fou_cfg_cmp(fou, cfg)) {
648 			fou_release(fou);
649 			err = 0;
650 			break;
651 		}
652 	}
653 	mutex_unlock(&fn->fou_lock);
654 
655 	return err;
656 }
657 
658 static struct genl_family fou_nl_family;
659 
660 static int parse_nl_config(struct genl_info *info,
661 			   struct fou_cfg *cfg)
662 {
663 	bool has_local = false, has_peer = false;
664 	struct nlattr *attr;
665 	int ifindex;
666 	__be16 port;
667 
668 	memset(cfg, 0, sizeof(*cfg));
669 
670 	cfg->udp_config.family = AF_INET;
671 
672 	if (info->attrs[FOU_ATTR_AF]) {
673 		u8 family = nla_get_u8(info->attrs[FOU_ATTR_AF]);
674 
675 		switch (family) {
676 		case AF_INET:
677 			break;
678 		case AF_INET6:
679 			cfg->udp_config.ipv6_v6only = 1;
680 			break;
681 		default:
682 			return -EAFNOSUPPORT;
683 		}
684 
685 		cfg->udp_config.family = family;
686 	}
687 
688 	if (info->attrs[FOU_ATTR_PORT]) {
689 		port = nla_get_be16(info->attrs[FOU_ATTR_PORT]);
690 		cfg->udp_config.local_udp_port = port;
691 	}
692 
693 	if (info->attrs[FOU_ATTR_IPPROTO])
694 		cfg->protocol = nla_get_u8(info->attrs[FOU_ATTR_IPPROTO]);
695 
696 	if (info->attrs[FOU_ATTR_TYPE])
697 		cfg->type = nla_get_u8(info->attrs[FOU_ATTR_TYPE]);
698 
699 	if (info->attrs[FOU_ATTR_REMCSUM_NOPARTIAL])
700 		cfg->flags |= FOU_F_REMCSUM_NOPARTIAL;
701 
702 	if (cfg->udp_config.family == AF_INET) {
703 		if (info->attrs[FOU_ATTR_LOCAL_V4]) {
704 			attr = info->attrs[FOU_ATTR_LOCAL_V4];
705 			cfg->udp_config.local_ip.s_addr = nla_get_in_addr(attr);
706 			has_local = true;
707 		}
708 
709 		if (info->attrs[FOU_ATTR_PEER_V4]) {
710 			attr = info->attrs[FOU_ATTR_PEER_V4];
711 			cfg->udp_config.peer_ip.s_addr = nla_get_in_addr(attr);
712 			has_peer = true;
713 		}
714 #if IS_ENABLED(CONFIG_IPV6)
715 	} else {
716 		if (info->attrs[FOU_ATTR_LOCAL_V6]) {
717 			attr = info->attrs[FOU_ATTR_LOCAL_V6];
718 			cfg->udp_config.local_ip6 = nla_get_in6_addr(attr);
719 			has_local = true;
720 		}
721 
722 		if (info->attrs[FOU_ATTR_PEER_V6]) {
723 			attr = info->attrs[FOU_ATTR_PEER_V6];
724 			cfg->udp_config.peer_ip6 = nla_get_in6_addr(attr);
725 			has_peer = true;
726 		}
727 #endif
728 	}
729 
730 	if (has_peer) {
731 		if (info->attrs[FOU_ATTR_PEER_PORT]) {
732 			port = nla_get_be16(info->attrs[FOU_ATTR_PEER_PORT]);
733 			cfg->udp_config.peer_udp_port = port;
734 		} else {
735 			return -EINVAL;
736 		}
737 	}
738 
739 	if (info->attrs[FOU_ATTR_IFINDEX]) {
740 		if (!has_local)
741 			return -EINVAL;
742 
743 		ifindex = nla_get_s32(info->attrs[FOU_ATTR_IFINDEX]);
744 
745 		cfg->udp_config.bind_ifindex = ifindex;
746 	}
747 
748 	return 0;
749 }
750 
751 int fou_nl_add_doit(struct sk_buff *skb, struct genl_info *info)
752 {
753 	struct net *net = genl_info_net(info);
754 	struct fou_cfg cfg;
755 	int err;
756 
757 	err = parse_nl_config(info, &cfg);
758 	if (err)
759 		return err;
760 
761 	return fou_create(net, &cfg, NULL);
762 }
763 
764 int fou_nl_del_doit(struct sk_buff *skb, struct genl_info *info)
765 {
766 	struct net *net = genl_info_net(info);
767 	struct fou_cfg cfg;
768 	int err;
769 
770 	err = parse_nl_config(info, &cfg);
771 	if (err)
772 		return err;
773 
774 	return fou_destroy(net, &cfg);
775 }
776 
777 static int fou_fill_info(struct fou *fou, struct sk_buff *msg)
778 {
779 	struct sock *sk = fou->sock->sk;
780 
781 	if (nla_put_u8(msg, FOU_ATTR_AF, fou->sock->sk->sk_family) ||
782 	    nla_put_be16(msg, FOU_ATTR_PORT, fou->port) ||
783 	    nla_put_be16(msg, FOU_ATTR_PEER_PORT, sk->sk_dport) ||
784 	    nla_put_u8(msg, FOU_ATTR_IPPROTO, fou->protocol) ||
785 	    nla_put_u8(msg, FOU_ATTR_TYPE, fou->type) ||
786 	    nla_put_s32(msg, FOU_ATTR_IFINDEX, sk->sk_bound_dev_if))
787 		return -1;
788 
789 	if (fou->flags & FOU_F_REMCSUM_NOPARTIAL)
790 		if (nla_put_flag(msg, FOU_ATTR_REMCSUM_NOPARTIAL))
791 			return -1;
792 
793 	if (fou->sock->sk->sk_family == AF_INET) {
794 		if (nla_put_in_addr(msg, FOU_ATTR_LOCAL_V4, sk->sk_rcv_saddr))
795 			return -1;
796 
797 		if (nla_put_in_addr(msg, FOU_ATTR_PEER_V4, sk->sk_daddr))
798 			return -1;
799 #if IS_ENABLED(CONFIG_IPV6)
800 	} else {
801 		if (nla_put_in6_addr(msg, FOU_ATTR_LOCAL_V6,
802 				     &sk->sk_v6_rcv_saddr))
803 			return -1;
804 
805 		if (nla_put_in6_addr(msg, FOU_ATTR_PEER_V6, &sk->sk_v6_daddr))
806 			return -1;
807 #endif
808 	}
809 
810 	return 0;
811 }
812 
813 static int fou_dump_info(struct fou *fou, u32 portid, u32 seq,
814 			 u32 flags, struct sk_buff *skb, u8 cmd)
815 {
816 	void *hdr;
817 
818 	hdr = genlmsg_put(skb, portid, seq, &fou_nl_family, flags, cmd);
819 	if (!hdr)
820 		return -ENOMEM;
821 
822 	if (fou_fill_info(fou, skb) < 0)
823 		goto nla_put_failure;
824 
825 	genlmsg_end(skb, hdr);
826 	return 0;
827 
828 nla_put_failure:
829 	genlmsg_cancel(skb, hdr);
830 	return -EMSGSIZE;
831 }
832 
833 int fou_nl_get_doit(struct sk_buff *skb, struct genl_info *info)
834 {
835 	struct net *net = genl_info_net(info);
836 	struct fou_net *fn = net_generic(net, fou_net_id);
837 	struct sk_buff *msg;
838 	struct fou_cfg cfg;
839 	struct fou *fout;
840 	__be16 port;
841 	u8 family;
842 	int ret;
843 
844 	ret = parse_nl_config(info, &cfg);
845 	if (ret)
846 		return ret;
847 	port = cfg.udp_config.local_udp_port;
848 	if (port == 0)
849 		return -EINVAL;
850 
851 	family = cfg.udp_config.family;
852 	if (family != AF_INET && family != AF_INET6)
853 		return -EINVAL;
854 
855 	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
856 	if (!msg)
857 		return -ENOMEM;
858 
859 	ret = -ESRCH;
860 	mutex_lock(&fn->fou_lock);
861 	list_for_each_entry(fout, &fn->fou_list, list) {
862 		if (fou_cfg_cmp(fout, &cfg)) {
863 			ret = fou_dump_info(fout, info->snd_portid,
864 					    info->snd_seq, 0, msg,
865 					    info->genlhdr->cmd);
866 			break;
867 		}
868 	}
869 	mutex_unlock(&fn->fou_lock);
870 	if (ret < 0)
871 		goto out_free;
872 
873 	return genlmsg_reply(msg, info);
874 
875 out_free:
876 	nlmsg_free(msg);
877 	return ret;
878 }
879 
880 int fou_nl_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
881 {
882 	struct net *net = sock_net(skb->sk);
883 	struct fou_net *fn = net_generic(net, fou_net_id);
884 	struct fou *fout;
885 	int idx = 0, ret;
886 
887 	mutex_lock(&fn->fou_lock);
888 	list_for_each_entry(fout, &fn->fou_list, list) {
889 		if (idx++ < cb->args[0])
890 			continue;
891 		ret = fou_dump_info(fout, NETLINK_CB(cb->skb).portid,
892 				    cb->nlh->nlmsg_seq, NLM_F_MULTI,
893 				    skb, FOU_CMD_GET);
894 		if (ret)
895 			break;
896 	}
897 	mutex_unlock(&fn->fou_lock);
898 
899 	cb->args[0] = idx;
900 	return skb->len;
901 }
902 
903 static struct genl_family fou_nl_family __ro_after_init = {
904 	.hdrsize	= 0,
905 	.name		= FOU_GENL_NAME,
906 	.version	= FOU_GENL_VERSION,
907 	.maxattr	= FOU_ATTR_MAX,
908 	.policy		= fou_nl_policy,
909 	.netnsok	= true,
910 	.module		= THIS_MODULE,
911 	.small_ops	= fou_nl_ops,
912 	.n_small_ops	= ARRAY_SIZE(fou_nl_ops),
913 	.resv_start_op	= FOU_CMD_GET + 1,
914 };
915 
916 size_t fou_encap_hlen(struct ip_tunnel_encap *e)
917 {
918 	return sizeof(struct udphdr);
919 }
920 EXPORT_SYMBOL(fou_encap_hlen);
921 
922 size_t gue_encap_hlen(struct ip_tunnel_encap *e)
923 {
924 	size_t len;
925 	bool need_priv = false;
926 
927 	len = sizeof(struct udphdr) + sizeof(struct guehdr);
928 
929 	if (e->flags & TUNNEL_ENCAP_FLAG_REMCSUM) {
930 		len += GUE_PLEN_REMCSUM;
931 		need_priv = true;
932 	}
933 
934 	len += need_priv ? GUE_LEN_PRIV : 0;
935 
936 	return len;
937 }
938 EXPORT_SYMBOL(gue_encap_hlen);
939 
940 int __fou_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e,
941 		       u8 *protocol, __be16 *sport, int type)
942 {
943 	int err;
944 
945 	err = iptunnel_handle_offloads(skb, type);
946 	if (err)
947 		return err;
948 
949 	*sport = e->sport ? : udp_flow_src_port(dev_net(skb->dev),
950 						skb, 0, 0, false);
951 
952 	return 0;
953 }
954 EXPORT_SYMBOL(__fou_build_header);
955 
956 int __gue_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e,
957 		       u8 *protocol, __be16 *sport, int type)
958 {
959 	struct guehdr *guehdr;
960 	size_t hdrlen, optlen = 0;
961 	void *data;
962 	bool need_priv = false;
963 	int err;
964 
965 	if ((e->flags & TUNNEL_ENCAP_FLAG_REMCSUM) &&
966 	    skb->ip_summed == CHECKSUM_PARTIAL) {
967 		optlen += GUE_PLEN_REMCSUM;
968 		type |= SKB_GSO_TUNNEL_REMCSUM;
969 		need_priv = true;
970 	}
971 
972 	optlen += need_priv ? GUE_LEN_PRIV : 0;
973 
974 	err = iptunnel_handle_offloads(skb, type);
975 	if (err)
976 		return err;
977 
978 	/* Get source port (based on flow hash) before skb_push */
979 	*sport = e->sport ? : udp_flow_src_port(dev_net(skb->dev),
980 						skb, 0, 0, false);
981 
982 	hdrlen = sizeof(struct guehdr) + optlen;
983 
984 	skb_push(skb, hdrlen);
985 
986 	guehdr = (struct guehdr *)skb->data;
987 
988 	guehdr->control = 0;
989 	guehdr->version = 0;
990 	guehdr->hlen = optlen >> 2;
991 	guehdr->flags = 0;
992 	guehdr->proto_ctype = *protocol;
993 
994 	data = &guehdr[1];
995 
996 	if (need_priv) {
997 		__be32 *flags = data;
998 
999 		guehdr->flags |= GUE_FLAG_PRIV;
1000 		*flags = 0;
1001 		data += GUE_LEN_PRIV;
1002 
1003 		if (type & SKB_GSO_TUNNEL_REMCSUM) {
1004 			u16 csum_start = skb_checksum_start_offset(skb);
1005 			__be16 *pd = data;
1006 
1007 			if (csum_start < hdrlen)
1008 				return -EINVAL;
1009 
1010 			csum_start -= hdrlen;
1011 			pd[0] = htons(csum_start);
1012 			pd[1] = htons(csum_start + skb->csum_offset);
1013 
1014 			if (!skb_is_gso(skb)) {
1015 				skb->ip_summed = CHECKSUM_NONE;
1016 				skb->encapsulation = 0;
1017 			}
1018 
1019 			*flags |= GUE_PFLAG_REMCSUM;
1020 			data += GUE_PLEN_REMCSUM;
1021 		}
1022 
1023 	}
1024 
1025 	return 0;
1026 }
1027 EXPORT_SYMBOL(__gue_build_header);
1028 
1029 #ifdef CONFIG_NET_FOU_IP_TUNNELS
1030 
1031 static void fou_build_udp(struct sk_buff *skb, struct ip_tunnel_encap *e,
1032 			  struct flowi4 *fl4, u8 *protocol, __be16 sport)
1033 {
1034 	struct udphdr *uh;
1035 
1036 	skb_push(skb, sizeof(struct udphdr));
1037 	skb_reset_transport_header(skb);
1038 
1039 	uh = udp_hdr(skb);
1040 
1041 	uh->dest = e->dport;
1042 	uh->source = sport;
1043 	uh->len = htons(skb->len);
1044 	udp_set_csum(!(e->flags & TUNNEL_ENCAP_FLAG_CSUM), skb,
1045 		     fl4->saddr, fl4->daddr, skb->len);
1046 
1047 	*protocol = IPPROTO_UDP;
1048 }
1049 
1050 static int fou_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e,
1051 			    u8 *protocol, struct flowi4 *fl4)
1052 {
1053 	int type = e->flags & TUNNEL_ENCAP_FLAG_CSUM ? SKB_GSO_UDP_TUNNEL_CSUM :
1054 						       SKB_GSO_UDP_TUNNEL;
1055 	__be16 sport;
1056 	int err;
1057 
1058 	err = __fou_build_header(skb, e, protocol, &sport, type);
1059 	if (err)
1060 		return err;
1061 
1062 	fou_build_udp(skb, e, fl4, protocol, sport);
1063 
1064 	return 0;
1065 }
1066 
1067 static int gue_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e,
1068 			    u8 *protocol, struct flowi4 *fl4)
1069 {
1070 	int type = e->flags & TUNNEL_ENCAP_FLAG_CSUM ? SKB_GSO_UDP_TUNNEL_CSUM :
1071 						       SKB_GSO_UDP_TUNNEL;
1072 	__be16 sport;
1073 	int err;
1074 
1075 	err = __gue_build_header(skb, e, protocol, &sport, type);
1076 	if (err)
1077 		return err;
1078 
1079 	fou_build_udp(skb, e, fl4, protocol, sport);
1080 
1081 	return 0;
1082 }
1083 
1084 static int gue_err_proto_handler(int proto, struct sk_buff *skb, u32 info)
1085 {
1086 	const struct net_protocol *ipprot = rcu_dereference(inet_protos[proto]);
1087 
1088 	if (ipprot && ipprot->err_handler) {
1089 		if (!ipprot->err_handler(skb, info))
1090 			return 0;
1091 	}
1092 
1093 	return -ENOENT;
1094 }
1095 
1096 static int gue_err(struct sk_buff *skb, u32 info)
1097 {
1098 	int transport_offset = skb_transport_offset(skb);
1099 	struct guehdr *guehdr;
1100 	size_t len, optlen;
1101 	int ret;
1102 
1103 	len = sizeof(struct udphdr) + sizeof(struct guehdr);
1104 	if (!pskb_may_pull(skb, transport_offset + len))
1105 		return -EINVAL;
1106 
1107 	guehdr = (struct guehdr *)&udp_hdr(skb)[1];
1108 
1109 	switch (guehdr->version) {
1110 	case 0: /* Full GUE header present */
1111 		break;
1112 	case 1: {
1113 		/* Direct encapsulation of IPv4 or IPv6 */
1114 		skb_set_transport_header(skb, -(int)sizeof(struct icmphdr));
1115 
1116 		switch (((struct iphdr *)guehdr)->version) {
1117 		case 4:
1118 			ret = gue_err_proto_handler(IPPROTO_IPIP, skb, info);
1119 			goto out;
1120 #if IS_ENABLED(CONFIG_IPV6)
1121 		case 6:
1122 			ret = gue_err_proto_handler(IPPROTO_IPV6, skb, info);
1123 			goto out;
1124 #endif
1125 		default:
1126 			ret = -EOPNOTSUPP;
1127 			goto out;
1128 		}
1129 	}
1130 	default: /* Undefined version */
1131 		return -EOPNOTSUPP;
1132 	}
1133 
1134 	if (guehdr->control)
1135 		return -ENOENT;
1136 
1137 	optlen = guehdr->hlen << 2;
1138 
1139 	if (!pskb_may_pull(skb, transport_offset + len + optlen))
1140 		return -EINVAL;
1141 
1142 	guehdr = (struct guehdr *)&udp_hdr(skb)[1];
1143 	if (validate_gue_flags(guehdr, optlen))
1144 		return -EINVAL;
1145 
1146 	/* Handling exceptions for direct UDP encapsulation in GUE would lead to
1147 	 * recursion. Besides, this kind of encapsulation can't even be
1148 	 * configured currently. Discard this.
1149 	 */
1150 	if (guehdr->proto_ctype == IPPROTO_UDP ||
1151 	    guehdr->proto_ctype == IPPROTO_UDPLITE)
1152 		return -EOPNOTSUPP;
1153 
1154 	skb_set_transport_header(skb, -(int)sizeof(struct icmphdr));
1155 	ret = gue_err_proto_handler(guehdr->proto_ctype, skb, info);
1156 
1157 out:
1158 	skb_set_transport_header(skb, transport_offset);
1159 	return ret;
1160 }
1161 
1162 
1163 static const struct ip_tunnel_encap_ops fou_iptun_ops = {
1164 	.encap_hlen = fou_encap_hlen,
1165 	.build_header = fou_build_header,
1166 	.err_handler = gue_err,
1167 };
1168 
1169 static const struct ip_tunnel_encap_ops gue_iptun_ops = {
1170 	.encap_hlen = gue_encap_hlen,
1171 	.build_header = gue_build_header,
1172 	.err_handler = gue_err,
1173 };
1174 
1175 static int ip_tunnel_encap_add_fou_ops(void)
1176 {
1177 	int ret;
1178 
1179 	ret = ip_tunnel_encap_add_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU);
1180 	if (ret < 0) {
1181 		pr_err("can't add fou ops\n");
1182 		return ret;
1183 	}
1184 
1185 	ret = ip_tunnel_encap_add_ops(&gue_iptun_ops, TUNNEL_ENCAP_GUE);
1186 	if (ret < 0) {
1187 		pr_err("can't add gue ops\n");
1188 		ip_tunnel_encap_del_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU);
1189 		return ret;
1190 	}
1191 
1192 	return 0;
1193 }
1194 
1195 static void ip_tunnel_encap_del_fou_ops(void)
1196 {
1197 	ip_tunnel_encap_del_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU);
1198 	ip_tunnel_encap_del_ops(&gue_iptun_ops, TUNNEL_ENCAP_GUE);
1199 }
1200 
1201 #else
1202 
1203 static int ip_tunnel_encap_add_fou_ops(void)
1204 {
1205 	return 0;
1206 }
1207 
1208 static void ip_tunnel_encap_del_fou_ops(void)
1209 {
1210 }
1211 
1212 #endif
1213 
1214 static __net_init int fou_init_net(struct net *net)
1215 {
1216 	struct fou_net *fn = net_generic(net, fou_net_id);
1217 
1218 	INIT_LIST_HEAD(&fn->fou_list);
1219 	mutex_init(&fn->fou_lock);
1220 	return 0;
1221 }
1222 
1223 static __net_exit void fou_exit_net(struct net *net)
1224 {
1225 	struct fou_net *fn = net_generic(net, fou_net_id);
1226 	struct fou *fou, *next;
1227 
1228 	/* Close all the FOU sockets */
1229 	mutex_lock(&fn->fou_lock);
1230 	list_for_each_entry_safe(fou, next, &fn->fou_list, list)
1231 		fou_release(fou);
1232 	mutex_unlock(&fn->fou_lock);
1233 }
1234 
1235 static struct pernet_operations fou_net_ops = {
1236 	.init = fou_init_net,
1237 	.exit = fou_exit_net,
1238 	.id   = &fou_net_id,
1239 	.size = sizeof(struct fou_net),
1240 };
1241 
1242 static int __init fou_init(void)
1243 {
1244 	int ret;
1245 
1246 	ret = register_pernet_device(&fou_net_ops);
1247 	if (ret)
1248 		goto exit;
1249 
1250 	ret = genl_register_family(&fou_nl_family);
1251 	if (ret < 0)
1252 		goto unregister;
1253 
1254 	ret = register_fou_bpf();
1255 	if (ret < 0)
1256 		goto kfunc_failed;
1257 
1258 	ret = ip_tunnel_encap_add_fou_ops();
1259 	if (ret == 0)
1260 		return 0;
1261 
1262 kfunc_failed:
1263 	genl_unregister_family(&fou_nl_family);
1264 unregister:
1265 	unregister_pernet_device(&fou_net_ops);
1266 exit:
1267 	return ret;
1268 }
1269 
1270 static void __exit fou_fini(void)
1271 {
1272 	ip_tunnel_encap_del_fou_ops();
1273 	genl_unregister_family(&fou_nl_family);
1274 	unregister_pernet_device(&fou_net_ops);
1275 }
1276 
1277 module_init(fou_init);
1278 module_exit(fou_fini);
1279 MODULE_AUTHOR("Tom Herbert <therbert@google.com>");
1280 MODULE_LICENSE("GPL");
1281 MODULE_DESCRIPTION("Foo over UDP");
1282