16 static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb,  in __skb_udp_tunnel_segment()  argument
18 	struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb,  in __skb_udp_tunnel_segment()  argument
22 	int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb);  in __skb_udp_tunnel_segment()
25 	struct udphdr *uh = udp_hdr(skb);  in __skb_udp_tunnel_segment()
26 	u16 mac_offset = skb->mac_header;  in __skb_udp_tunnel_segment()
27 	__be16 protocol = skb->protocol;  in __skb_udp_tunnel_segment()
28 	u16 mac_len = skb->mac_len;  in __skb_udp_tunnel_segment()
33 	if (unlikely(!pskb_may_pull(skb, tnl_hlen)))  in __skb_udp_tunnel_segment()
42 	if (skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL)  in __skb_udp_tunnel_segment()
45 		partial = (__force __wsum)htonl(skb->len);  in __skb_udp_tunnel_segment()
49 	skb->encapsulation = 0;  in __skb_udp_tunnel_segment()
50 	SKB_GSO_CB(skb)->encap_level = 0;  in __skb_udp_tunnel_segment()
51 	__skb_pull(skb, tnl_hlen);  in __skb_udp_tunnel_segment()
52 	skb_reset_mac_header(skb);  in __skb_udp_tunnel_segment()
53 	skb_set_network_header(skb, skb_inner_network_offset(skb));  in __skb_udp_tunnel_segment()
54 	skb_set_transport_header(skb, skb_inner_transport_offset(skb));  in __skb_udp_tunnel_segment()
55 	skb->mac_len = skb_inner_network_offset(skb);  in __skb_udp_tunnel_segment()
56 	skb->protocol = new_protocol;  in __skb_udp_tunnel_segment()
58 	need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM);  in __skb_udp_tunnel_segment()
59 	skb->encap_hdr_csum = need_csum;  in __skb_udp_tunnel_segment()
61 	remcsum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_TUNNEL_REMCSUM);  in __skb_udp_tunnel_segment()
62 	skb->remcsum_offload = remcsum;  in __skb_udp_tunnel_segment()
64 	need_ipsec = skb_dst(skb) && dst_xfrm(skb_dst(skb));  in __skb_udp_tunnel_segment()
68 			  (skb->dev->features &  in __skb_udp_tunnel_segment()
72 	features &= skb->dev->hw_enc_features;  in __skb_udp_tunnel_segment()
87 	segs = gso_inner_segment(skb, features);  in __skb_udp_tunnel_segment()
89 		skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset,  in __skb_udp_tunnel_segment()
96 	outer_hlen = skb_tnl_header_len(skb);  in __skb_udp_tunnel_segment()
98 	skb = segs;  in __skb_udp_tunnel_segment()
103 			skb->ip_summed = CHECKSUM_NONE;  in __skb_udp_tunnel_segment()
106 		if (skb->ip_summed == CHECKSUM_PARTIAL) {  in __skb_udp_tunnel_segment()
107 			skb_reset_inner_headers(skb);  in __skb_udp_tunnel_segment()
108 			skb->encapsulation = 1;  in __skb_udp_tunnel_segment()
111 		skb->mac_len = mac_len;  in __skb_udp_tunnel_segment()
112 		skb->protocol = protocol;  in __skb_udp_tunnel_segment()
114 		__skb_push(skb, outer_hlen);  in __skb_udp_tunnel_segment()
115 		skb_reset_mac_header(skb);  in __skb_udp_tunnel_segment()
116 		skb_set_network_header(skb, mac_len);  in __skb_udp_tunnel_segment()
117 		skb_set_transport_header(skb, udp_offset);  in __skb_udp_tunnel_segment()
118 		len = skb->len - udp_offset;  in __skb_udp_tunnel_segment()
119 		uh = udp_hdr(skb);  in __skb_udp_tunnel_segment()
125 		if (gso_partial && skb_is_gso(skb)) {  in __skb_udp_tunnel_segment()
126 			uh->len = htons(skb_shinfo(skb)->gso_size +  in __skb_udp_tunnel_segment()
127 					SKB_GSO_CB(skb)->data_offset +  in __skb_udp_tunnel_segment()
128 					skb->head - (unsigned char *)uh);  in __skb_udp_tunnel_segment()
139 		if (skb->encapsulation || !offload_csum) {  in __skb_udp_tunnel_segment()
140 			uh->check = gso_make_checksum(skb, ~uh->check);  in __skb_udp_tunnel_segment()
144 			skb->ip_summed = CHECKSUM_PARTIAL;  in __skb_udp_tunnel_segment()
145 			skb->csum_start = skb_transport_header(skb) - skb->head;  in __skb_udp_tunnel_segment()
146 			skb->csum_offset = offsetof(struct udphdr, check);  in __skb_udp_tunnel_segment()
148 	} while ((skb = skb->next));  in __skb_udp_tunnel_segment()
153 struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb,  in skb_udp_tunnel_segment()  argument
158 	__be16 protocol = skb->protocol;  in skb_udp_tunnel_segment()
161 	struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb,  in skb_udp_tunnel_segment()
166 	switch (skb->inner_protocol_type) {  in skb_udp_tunnel_segment()
168 		protocol = skb->inner_protocol;  in skb_udp_tunnel_segment()
173 		ops = rcu_dereference(offloads[skb->inner_ipproto]);  in skb_udp_tunnel_segment()
182 	segs = __skb_udp_tunnel_segment(skb, features, gso_inner_segment,  in skb_udp_tunnel_segment()
250 static struct sk_buff *__udp_gso_segment_list(struct sk_buff *skb,  in __udp_gso_segment_list()  argument
254 	unsigned int mss = skb_shinfo(skb)->gso_size;  in __udp_gso_segment_list()
256 	skb = skb_segment_list(skb, features, skb_mac_header_len(skb));  in __udp_gso_segment_list()
257 	if (IS_ERR(skb))  in __udp_gso_segment_list()
258 		return skb;  in __udp_gso_segment_list()
260 	udp_hdr(skb)->len = htons(sizeof(struct udphdr) + mss);  in __udp_gso_segment_list()
262 	return is_ipv6 ? skb : __udpv4_gso_segment_list_csum(skb);  in __udp_gso_segment_list()
417 static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb,  in udp4_ufo_fragment()  argument
426 	if (skb->encapsulation &&  in udp4_ufo_fragment()
427 	    (skb_shinfo(skb)->gso_type &  in udp4_ufo_fragment()
429 		segs = skb_udp_tunnel_segment(skb, features, false);  in udp4_ufo_fragment()
433 	if (!(skb_shinfo(skb)->gso_type & (SKB_GSO_UDP | SKB_GSO_UDP_L4)))  in udp4_ufo_fragment()
436 	if (!pskb_may_pull(skb, sizeof(struct udphdr)))  in udp4_ufo_fragment()
439 	if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4)  in udp4_ufo_fragment()
440 		return __udp_gso_segment(skb, features, false);  in udp4_ufo_fragment()
442 	mss = skb_shinfo(skb)->gso_size;  in udp4_ufo_fragment()
443 	if (unlikely(skb->len <= mss))  in udp4_ufo_fragment()
451 	uh = udp_hdr(skb);  in udp4_ufo_fragment()
452 	iph = ip_hdr(skb);  in udp4_ufo_fragment()
455 	csum = skb_checksum(skb, 0, skb->len, 0);  in udp4_ufo_fragment()
456 	uh->check = udp_v4_check(skb->len, iph->saddr, iph->daddr, csum);  in udp4_ufo_fragment()
460 	skb->ip_summed = CHECKSUM_UNNECESSARY;  in udp4_ufo_fragment()
466 	if (!skb->encap_hdr_csum)  in udp4_ufo_fragment()
472 	segs = skb_segment(skb, features);  in udp4_ufo_fragment()
480 					       struct sk_buff *skb)  in udp_gro_receive_segment()  argument
482 	struct udphdr *uh = udp_gro_udphdr(skb);  in udp_gro_receive_segment()
492 		NAPI_GRO_CB(skb)->flush = 1;  in udp_gro_receive_segment()
498 	if (ulen <= sizeof(*uh) || ulen != skb_gro_len(skb)) {  in udp_gro_receive_segment()
499 		NAPI_GRO_CB(skb)->flush = 1;  in udp_gro_receive_segment()
503 	skb_gro_pull(skb, sizeof(struct udphdr));  in udp_gro_receive_segment()
517 		if (NAPI_GRO_CB(skb)->is_flist != NAPI_GRO_CB(p)->is_flist) {  in udp_gro_receive_segment()
518 			NAPI_GRO_CB(skb)->flush = 1;  in udp_gro_receive_segment()
533 			if (NAPI_GRO_CB(skb)->is_flist) {  in udp_gro_receive_segment()
534 				if (!pskb_may_pull(skb, skb_gro_offset(skb))) {  in udp_gro_receive_segment()
535 					NAPI_GRO_CB(skb)->flush = 1;  in udp_gro_receive_segment()
538 				if ((skb->ip_summed != p->ip_summed) ||  in udp_gro_receive_segment()
539 				    (skb->csum_level != p->csum_level)) {  in udp_gro_receive_segment()
540 					NAPI_GRO_CB(skb)->flush = 1;  in udp_gro_receive_segment()
543 				ret = skb_gro_receive_list(p, skb);  in udp_gro_receive_segment()
545 				skb_gro_postpull_rcsum(skb, uh,  in udp_gro_receive_segment()
548 				ret = skb_gro_receive(p, skb);  in udp_gro_receive_segment()
563 struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,  in udp_gro_receive()  argument
569 	unsigned int off = skb_gro_offset(skb);  in udp_gro_receive()
577 	NAPI_GRO_CB(skb)->is_flist = 0;  in udp_gro_receive()
582 		if (skb->encapsulation)  in udp_gro_receive()
585 		if (skb->dev->features & NETIF_F_GRO_FRAGLIST)  in udp_gro_receive()
586 			NAPI_GRO_CB(skb)->is_flist = sk ? !udp_test_bit(GRO_ENABLED, sk) : 1;  in udp_gro_receive()
588 		if ((!sk && (skb->dev->features & NETIF_F_GRO_UDP_FWD)) ||  in udp_gro_receive()
589 		    (sk && udp_test_bit(GRO_ENABLED, sk)) || NAPI_GRO_CB(skb)->is_flist)  in udp_gro_receive()
590 			return call_gro_receive(udp_gro_receive_segment, head, skb);  in udp_gro_receive()
596 	if (NAPI_GRO_CB(skb)->encap_mark ||  in udp_gro_receive()
597 	    (uh->check && skb->ip_summed != CHECKSUM_PARTIAL &&  in udp_gro_receive()
598 	     NAPI_GRO_CB(skb)->csum_cnt == 0 &&  in udp_gro_receive()
599 	     !NAPI_GRO_CB(skb)->csum_valid))  in udp_gro_receive()
603 	NAPI_GRO_CB(skb)->encap_mark = 1;  in udp_gro_receive()
623 	skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */  in udp_gro_receive()
624 	skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr));  in udp_gro_receive()
625 	pp = call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);  in udp_gro_receive()
628 	skb_gro_flush_final(skb, pp, flush);  in udp_gro_receive()
633 static struct sock *udp4_gro_lookup_skb(struct sk_buff *skb, __be16 sport,  in udp4_gro_lookup_skb()  argument
636 	const struct iphdr *iph = skb_gro_network_header(skb);  in udp4_gro_lookup_skb()
637 	struct net *net = dev_net(skb->dev);  in udp4_gro_lookup_skb()
640 	inet_get_iif_sdif(skb, &iif, &sdif);  in udp4_gro_lookup_skb()
648 struct sk_buff *udp4_gro_receive(struct list_head *head, struct sk_buff *skb)  in udp4_gro_receive()  argument
650 	struct udphdr *uh = udp_gro_udphdr(skb);  in udp4_gro_receive()
658 	if (NAPI_GRO_CB(skb)->flush)  in udp4_gro_receive()
661 	if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check,  in udp4_gro_receive()
665 		skb_gro_checksum_try_convert(skb, IPPROTO_UDP,  in udp4_gro_receive()
668 	NAPI_GRO_CB(skb)->is_ipv6 = 0;  in udp4_gro_receive()
671 		sk = udp4_gro_lookup_skb(skb, uh->source, uh->dest);  in udp4_gro_receive()
673 	pp = udp_gro_receive(head, skb, uh, sk);  in udp4_gro_receive()
677 	NAPI_GRO_CB(skb)->flush = 1;  in udp4_gro_receive()
681 static int udp_gro_complete_segment(struct sk_buff *skb)  in udp_gro_complete_segment()  argument
683 	struct udphdr *uh = udp_hdr(skb);  in udp_gro_complete_segment()
685 	skb->csum_start = (unsigned char *)uh - skb->head;  in udp_gro_complete_segment()
686 	skb->csum_offset = offsetof(struct udphdr, check);  in udp_gro_complete_segment()
687 	skb->ip_summed = CHECKSUM_PARTIAL;  in udp_gro_complete_segment()
689 	skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;  in udp_gro_complete_segment()
690 	skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_L4;  in udp_gro_complete_segment()
692 	if (skb->encapsulation)  in udp_gro_complete_segment()
693 		skb->inner_transport_header = skb->transport_header;  in udp_gro_complete_segment()
698 int udp_gro_complete(struct sk_buff *skb, int nhoff,  in udp_gro_complete()  argument
701 	__be16 newlen = htons(skb->len - nhoff);  in udp_gro_complete()
702 	struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);  in udp_gro_complete()
709 				udp4_lib_lookup_skb, skb, uh->source, uh->dest);  in udp_gro_complete()
711 		skb_shinfo(skb)->gso_type = uh->check ? SKB_GSO_UDP_TUNNEL_CSUM  in udp_gro_complete()
717 		NAPI_GRO_CB(skb)->encap_mark = 0;  in udp_gro_complete()
722 		skb->encapsulation = 1;  in udp_gro_complete()
723 		err = udp_sk(sk)->gro_complete(sk, skb,  in udp_gro_complete()
726 		err = udp_gro_complete_segment(skb);  in udp_gro_complete()
729 	if (skb->remcsum_offload)  in udp_gro_complete()
730 		skb_shinfo(skb)->gso_type |= SKB_GSO_TUNNEL_REMCSUM;  in udp_gro_complete()
736 INDIRECT_CALLABLE_SCOPE int udp4_gro_complete(struct sk_buff *skb, int nhoff)  in udp4_gro_complete()  argument
738 	const u16 offset = NAPI_GRO_CB(skb)->network_offsets[skb->encapsulation];  in udp4_gro_complete()
739 	const struct iphdr *iph = (struct iphdr *)(skb->data + offset);  in udp4_gro_complete()
740 	struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);  in udp4_gro_complete()
743 	if (NAPI_GRO_CB(skb)->is_flist && !NAPI_GRO_CB(skb)->encap_mark) {  in udp4_gro_complete()
744 		uh->len = htons(skb->len - nhoff);  in udp4_gro_complete()
746 		skb_shinfo(skb)->gso_type |= (SKB_GSO_FRAGLIST|SKB_GSO_UDP_L4);  in udp4_gro_complete()
747 		skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;  in udp4_gro_complete()
749 		__skb_incr_checksum_unnecessary(skb);  in udp4_gro_complete()
755 		uh->check = ~udp_v4_check(skb->len - nhoff, iph->saddr,  in udp4_gro_complete()
758 	return udp_gro_complete(skb, nhoff, udp4_lib_lookup_skb);  in udp4_gro_complete()