1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * IPV6 GSO/GRO offload support
4 * Linux INET6 implementation
5 *
6 * TCPv6 GSO/GRO support
7 */
8 #include <linux/indirect_call_wrapper.h>
9 #include <linux/skbuff.h>
10 #include <net/inet6_hashtables.h>
11 #include <net/gro.h>
12 #include <net/protocol.h>
13 #include <net/tcp.h>
14 #include <net/ip6_checksum.h>
15 #include "ip6_offload.h"
16
tcp6_check_fraglist_gro(struct list_head * head,struct sk_buff * skb,struct tcphdr * th)17 static void tcp6_check_fraglist_gro(struct list_head *head, struct sk_buff *skb,
18 struct tcphdr *th)
19 {
20 #if IS_ENABLED(CONFIG_IPV6)
21 const struct ipv6hdr *hdr;
22 struct sk_buff *p;
23 struct sock *sk;
24 struct net *net;
25 int iif, sdif;
26
27 p = tcp_gro_lookup(head, th);
28 if (p) {
29 /* flist GRO applies to consecutive non-GSO skbs */
30 if (!skb_is_gso(skb) || !NAPI_GRO_CB(p)->is_flist) {
31 NAPI_GRO_CB(skb)->is_flist = NAPI_GRO_CB(p)->is_flist;
32 return;
33 }
34
35 /* Fall back to the regular GRO path */
36 if (NAPI_GRO_CB(p)->count == 1)
37 NAPI_GRO_CB(p)->is_flist = 0;
38
39 NAPI_GRO_CB(skb)->is_flist = 0;
40
41 return;
42 }
43
44 inet6_get_iif_sdif(skb, &iif, &sdif);
45 hdr = skb_gro_network_header(skb);
46 net = dev_net_rcu(skb->dev);
47 sk = __inet6_lookup_established(net, &hdr->saddr, th->source,
48 &hdr->daddr, ntohs(th->dest),
49 iif, sdif);
50 NAPI_GRO_CB(skb)->is_flist = !sk && !skb_is_gso(skb);
51 if (sk)
52 sock_gen_put(sk);
53 #endif /* IS_ENABLED(CONFIG_IPV6) */
54 }
55
tcp6_gro_receive(struct list_head * head,struct sk_buff * skb)56 static __always_inline struct sk_buff *tcp6_gro_receive(struct list_head *head,
57 struct sk_buff *skb)
58 {
59 struct tcphdr *th;
60
61 /* Don't bother verifying checksum if we're going to flush anyway. */
62 if (!NAPI_GRO_CB(skb)->flush &&
63 skb_gro_checksum_validate(skb, IPPROTO_TCP,
64 ip6_gro_compute_pseudo))
65 goto flush;
66
67 th = tcp_gro_pull_header(skb);
68 if (!th)
69 goto flush;
70
71 if (unlikely(skb->dev->features & NETIF_F_GRO_FRAGLIST))
72 tcp6_check_fraglist_gro(head, skb, th);
73
74 return tcp_gro_receive(head, skb, th);
75
76 flush:
77 NAPI_GRO_CB(skb)->flush = 1;
78 return NULL;
79 }
80
tcp6_gro_complete(struct sk_buff * skb,int thoff)81 static __always_inline int tcp6_gro_complete(struct sk_buff *skb, int thoff)
82 {
83 const u16 offset = NAPI_GRO_CB(skb)->network_offsets[skb->encapsulation];
84 const struct ipv6hdr *iph = (struct ipv6hdr *)(skb->data + offset);
85 struct tcphdr *th = tcp_hdr(skb);
86
87 if (unlikely(NAPI_GRO_CB(skb)->is_flist)) {
88 skb_shinfo(skb)->gso_type |= SKB_GSO_FRAGLIST | SKB_GSO_TCPV6;
89 skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
90
91 __skb_incr_checksum_unnecessary(skb);
92
93 return 0;
94 }
95
96 th->check = ~tcp_v6_check(skb->len - thoff, &iph->saddr,
97 &iph->daddr, 0);
98 skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV6;
99
100 tcp_gro_complete(skb);
101 return 0;
102 }
103
__tcpv6_gso_segment_csum(struct sk_buff * seg,struct in6_addr * oldip,const struct in6_addr * newip,__be16 * oldport,__be16 newport)104 static void __tcpv6_gso_segment_csum(struct sk_buff *seg,
105 struct in6_addr *oldip,
106 const struct in6_addr *newip,
107 __be16 *oldport, __be16 newport)
108 {
109 struct tcphdr *th = tcp_hdr(seg);
110
111 if (!ipv6_addr_equal(oldip, newip)) {
112 inet_proto_csum_replace16(&th->check, seg,
113 oldip->s6_addr32,
114 newip->s6_addr32,
115 true);
116 *oldip = *newip;
117 }
118
119 if (*oldport == newport)
120 return;
121
122 inet_proto_csum_replace2(&th->check, seg, *oldport, newport, false);
123 *oldport = newport;
124 }
125
__tcpv6_gso_segment_list_csum(struct sk_buff * segs)126 static struct sk_buff *__tcpv6_gso_segment_list_csum(struct sk_buff *segs)
127 {
128 const struct tcphdr *th;
129 const struct ipv6hdr *iph;
130 struct sk_buff *seg;
131 struct tcphdr *th2;
132 struct ipv6hdr *iph2;
133
134 seg = segs;
135 th = tcp_hdr(seg);
136 iph = ipv6_hdr(seg);
137 th2 = tcp_hdr(seg->next);
138 iph2 = ipv6_hdr(seg->next);
139
140 if (!(*(const u32 *)&th->source ^ *(const u32 *)&th2->source) &&
141 ipv6_addr_equal(&iph->saddr, &iph2->saddr) &&
142 ipv6_addr_equal(&iph->daddr, &iph2->daddr))
143 return segs;
144
145 while ((seg = seg->next)) {
146 th2 = tcp_hdr(seg);
147 iph2 = ipv6_hdr(seg);
148
149 __tcpv6_gso_segment_csum(seg, &iph2->saddr, &iph->saddr,
150 &th2->source, th->source);
151 __tcpv6_gso_segment_csum(seg, &iph2->daddr, &iph->daddr,
152 &th2->dest, th->dest);
153 }
154
155 return segs;
156 }
157
__tcp6_gso_segment_list(struct sk_buff * skb,netdev_features_t features)158 static struct sk_buff *__tcp6_gso_segment_list(struct sk_buff *skb,
159 netdev_features_t features)
160 {
161 skb = skb_segment_list(skb, features, skb_mac_header_len(skb));
162 if (IS_ERR(skb))
163 return skb;
164
165 return __tcpv6_gso_segment_list_csum(skb);
166 }
167
tcp6_gso_segment(struct sk_buff * skb,netdev_features_t features)168 static struct sk_buff *tcp6_gso_segment(struct sk_buff *skb,
169 netdev_features_t features)
170 {
171 struct tcphdr *th;
172
173 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6))
174 return ERR_PTR(-EINVAL);
175
176 if (!pskb_may_pull(skb, sizeof(*th)))
177 return ERR_PTR(-EINVAL);
178
179 if (skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST) {
180 struct tcphdr *th = tcp_hdr(skb);
181
182 if ((skb_pagelen(skb) - th->doff * 4 == skb_shinfo(skb)->gso_size) &&
183 !(skb_shinfo(skb)->gso_type & SKB_GSO_DODGY))
184 return __tcp6_gso_segment_list(skb, features);
185
186 skb->ip_summed = CHECKSUM_NONE;
187 }
188
189 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
190 const struct ipv6hdr *ipv6h = ipv6_hdr(skb);
191 struct tcphdr *th = tcp_hdr(skb);
192
193 /* Set up pseudo header, usually expect stack to have done
194 * this.
195 */
196
197 th->check = 0;
198 skb->ip_summed = CHECKSUM_PARTIAL;
199 __tcp_v6_send_check(skb, &ipv6h->saddr, &ipv6h->daddr);
200 }
201
202 return tcp_gso_segment(skb, features);
203 }
204
tcpv6_offload_init(void)205 int __init tcpv6_offload_init(void)
206 {
207 net_hotdata.tcpv6_offload = (struct net_offload) {
208 .callbacks = {
209 .gso_segment = tcp6_gso_segment,
210 .gro_receive = tcp6_gro_receive,
211 .gro_complete = tcp6_gro_complete,
212 },
213 };
214 return inet6_add_offload(&net_hotdata.tcpv6_offload, IPPROTO_TCP);
215 }
216