xref: /linux/include/net/gro.h (revision 460ea8980511c01c1551012b9a6ec6a06d02da59) 
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 
3 #ifndef _NET_IPV6_GRO_H
4 #define _NET_IPV6_GRO_H
5 
6 #include <linux/indirect_call_wrapper.h>
7 #include <linux/ip.h>
8 #include <linux/ipv6.h>
9 #include <net/ip6_checksum.h>
10 #include <linux/skbuff.h>
11 #include <net/udp.h>
12 
13 struct napi_gro_cb {
14 	union {
15 		struct {
16 			/* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */
17 			void	*frag0;
18 
19 			/* Length of frag0. */
20 			unsigned int frag0_len;
21 		};
22 
23 		struct {
24 			/* used in skb_gro_receive() slow path */
25 			struct sk_buff *last;
26 
27 			/* jiffies when first packet was created/queued */
28 			unsigned long age;
29 		};
30 	};
31 
32 	/* This indicates where we are processing relative to skb->data. */
33 	int	data_offset;
34 
35 	/* This is non-zero if the packet cannot be merged with the new skb. */
36 	u16	flush;
37 
38 	/* Save the IP ID here and check when we get to the transport layer */
39 	u16	flush_id;
40 
41 	/* Number of segments aggregated. */
42 	u16	count;
43 
44 	/* Used in ipv6_gro_receive() and foo-over-udp */
45 	u16	proto;
46 
47 /* Used in napi_gro_cb::free */
48 #define NAPI_GRO_FREE             1
49 #define NAPI_GRO_FREE_STOLEN_HEAD 2
50 	/* portion of the cb set to zero at every gro iteration */
51 	struct_group(zeroed,
52 
53 		/* Start offset for remote checksum offload */
54 		u16	gro_remcsum_start;
55 
56 		/* This is non-zero if the packet may be of the same flow. */
57 		u8	same_flow:1;
58 
59 		/* Used in tunnel GRO receive */
60 		u8	encap_mark:1;
61 
62 		/* GRO checksum is valid */
63 		u8	csum_valid:1;
64 
65 		/* Number of checksums via CHECKSUM_UNNECESSARY */
66 		u8	csum_cnt:3;
67 
68 		/* Free the skb? */
69 		u8	free:2;
70 
71 		/* Used in foo-over-udp, set in udp[46]_gro_receive */
72 		u8	is_ipv6:1;
73 
74 		/* Used in GRE, set in fou/gue_gro_receive */
75 		u8	is_fou:1;
76 
77 		/* Used to determine if flush_id can be ignored */
78 		u8	is_atomic:1;
79 
80 		/* Number of gro_receive callbacks this packet already went through */
81 		u8 recursion_counter:4;
82 
83 		/* GRO is done by frag_list pointer chaining. */
84 		u8	is_flist:1;
85 	);
86 
87 	/* used to support CHECKSUM_COMPLETE for tunneling protocols */
88 	__wsum	csum;
89 };
90 
91 #define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb)
92 
93 #define GRO_RECURSION_LIMIT 15
94 static inline int gro_recursion_inc_test(struct sk_buff *skb)
95 {
96 	return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT;
97 }
98 
99 typedef struct sk_buff *(*gro_receive_t)(struct list_head *, struct sk_buff *);
100 static inline struct sk_buff *call_gro_receive(gro_receive_t cb,
101 					       struct list_head *head,
102 					       struct sk_buff *skb)
103 {
104 	if (unlikely(gro_recursion_inc_test(skb))) {
105 		NAPI_GRO_CB(skb)->flush |= 1;
106 		return NULL;
107 	}
108 
109 	return cb(head, skb);
110 }
111 
112 typedef struct sk_buff *(*gro_receive_sk_t)(struct sock *, struct list_head *,
113 					    struct sk_buff *);
114 static inline struct sk_buff *call_gro_receive_sk(gro_receive_sk_t cb,
115 						  struct sock *sk,
116 						  struct list_head *head,
117 						  struct sk_buff *skb)
118 {
119 	if (unlikely(gro_recursion_inc_test(skb))) {
120 		NAPI_GRO_CB(skb)->flush |= 1;
121 		return NULL;
122 	}
123 
124 	return cb(sk, head, skb);
125 }
126 
127 static inline unsigned int skb_gro_offset(const struct sk_buff *skb)
128 {
129 	return NAPI_GRO_CB(skb)->data_offset;
130 }
131 
132 static inline unsigned int skb_gro_len(const struct sk_buff *skb)
133 {
134 	return skb->len - NAPI_GRO_CB(skb)->data_offset;
135 }
136 
137 static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len)
138 {
139 	NAPI_GRO_CB(skb)->data_offset += len;
140 }
141 
142 static inline void *skb_gro_header_fast(struct sk_buff *skb,
143 					unsigned int offset)
144 {
145 	return NAPI_GRO_CB(skb)->frag0 + offset;
146 }
147 
148 static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen)
149 {
150 	return NAPI_GRO_CB(skb)->frag0_len < hlen;
151 }
152 
153 static inline void skb_gro_frag0_invalidate(struct sk_buff *skb)
154 {
155 	NAPI_GRO_CB(skb)->frag0 = NULL;
156 	NAPI_GRO_CB(skb)->frag0_len = 0;
157 }
158 
159 static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,
160 					unsigned int offset)
161 {
162 	if (!pskb_may_pull(skb, hlen))
163 		return NULL;
164 
165 	skb_gro_frag0_invalidate(skb);
166 	return skb->data + offset;
167 }
168 
169 static inline void *skb_gro_header(struct sk_buff *skb,
170 					unsigned int hlen, unsigned int offset)
171 {
172 	void *ptr;
173 
174 	ptr = skb_gro_header_fast(skb, offset);
175 	if (skb_gro_header_hard(skb, hlen))
176 		ptr = skb_gro_header_slow(skb, hlen, offset);
177 	return ptr;
178 }
179 
180 static inline void *skb_gro_network_header(struct sk_buff *skb)
181 {
182 	return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) +
183 	       skb_network_offset(skb);
184 }
185 
186 static inline __wsum inet_gro_compute_pseudo(struct sk_buff *skb, int proto)
187 {
188 	const struct iphdr *iph = skb_gro_network_header(skb);
189 
190 	return csum_tcpudp_nofold(iph->saddr, iph->daddr,
191 				  skb_gro_len(skb), proto, 0);
192 }
193 
194 static inline void skb_gro_postpull_rcsum(struct sk_buff *skb,
195 					const void *start, unsigned int len)
196 {
197 	if (NAPI_GRO_CB(skb)->csum_valid)
198 		NAPI_GRO_CB(skb)->csum = wsum_negate(csum_partial(start, len,
199 						wsum_negate(NAPI_GRO_CB(skb)->csum)));
200 }
201 
202 /* GRO checksum functions. These are logical equivalents of the normal
203  * checksum functions (in skbuff.h) except that they operate on the GRO
204  * offsets and fields in sk_buff.
205  */
206 
207 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb);
208 
209 static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb)
210 {
211 	return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb));
212 }
213 
214 static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb,
215 						      bool zero_okay,
216 						      __sum16 check)
217 {
218 	return ((skb->ip_summed != CHECKSUM_PARTIAL ||
219 		skb_checksum_start_offset(skb) <
220 		 skb_gro_offset(skb)) &&
221 		!skb_at_gro_remcsum_start(skb) &&
222 		NAPI_GRO_CB(skb)->csum_cnt == 0 &&
223 		(!zero_okay || check));
224 }
225 
226 static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb,
227 							   __wsum psum)
228 {
229 	if (NAPI_GRO_CB(skb)->csum_valid &&
230 	    !csum_fold(csum_add(psum, NAPI_GRO_CB(skb)->csum)))
231 		return 0;
232 
233 	NAPI_GRO_CB(skb)->csum = psum;
234 
235 	return __skb_gro_checksum_complete(skb);
236 }
237 
238 static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb)
239 {
240 	if (NAPI_GRO_CB(skb)->csum_cnt > 0) {
241 		/* Consume a checksum from CHECKSUM_UNNECESSARY */
242 		NAPI_GRO_CB(skb)->csum_cnt--;
243 	} else {
244 		/* Update skb for CHECKSUM_UNNECESSARY and csum_level when we
245 		 * verified a new top level checksum or an encapsulated one
246 		 * during GRO. This saves work if we fallback to normal path.
247 		 */
248 		__skb_incr_checksum_unnecessary(skb);
249 	}
250 }
251 
252 #define __skb_gro_checksum_validate(skb, proto, zero_okay, check,	\
253 				    compute_pseudo)			\
254 ({									\
255 	__sum16 __ret = 0;						\
256 	if (__skb_gro_checksum_validate_needed(skb, zero_okay, check))	\
257 		__ret = __skb_gro_checksum_validate_complete(skb,	\
258 				compute_pseudo(skb, proto));		\
259 	if (!__ret)							\
260 		skb_gro_incr_csum_unnecessary(skb);			\
261 	__ret;								\
262 })
263 
264 #define skb_gro_checksum_validate(skb, proto, compute_pseudo)		\
265 	__skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo)
266 
267 #define skb_gro_checksum_validate_zero_check(skb, proto, check,		\
268 					     compute_pseudo)		\
269 	__skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo)
270 
271 #define skb_gro_checksum_simple_validate(skb)				\
272 	__skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo)
273 
274 static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb)
275 {
276 	return (NAPI_GRO_CB(skb)->csum_cnt == 0 &&
277 		!NAPI_GRO_CB(skb)->csum_valid);
278 }
279 
280 static inline void __skb_gro_checksum_convert(struct sk_buff *skb,
281 					      __wsum pseudo)
282 {
283 	NAPI_GRO_CB(skb)->csum = ~pseudo;
284 	NAPI_GRO_CB(skb)->csum_valid = 1;
285 }
286 
287 #define skb_gro_checksum_try_convert(skb, proto, compute_pseudo)	\
288 do {									\
289 	if (__skb_gro_checksum_convert_check(skb))			\
290 		__skb_gro_checksum_convert(skb, 			\
291 					   compute_pseudo(skb, proto));	\
292 } while (0)
293 
294 struct gro_remcsum {
295 	int offset;
296 	__wsum delta;
297 };
298 
299 static inline void skb_gro_remcsum_init(struct gro_remcsum *grc)
300 {
301 	grc->offset = 0;
302 	grc->delta = 0;
303 }
304 
305 static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr,
306 					    unsigned int off, size_t hdrlen,
307 					    int start, int offset,
308 					    struct gro_remcsum *grc,
309 					    bool nopartial)
310 {
311 	__wsum delta;
312 	size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start);
313 
314 	BUG_ON(!NAPI_GRO_CB(skb)->csum_valid);
315 
316 	if (!nopartial) {
317 		NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start;
318 		return ptr;
319 	}
320 
321 	ptr = skb_gro_header(skb, off + plen, off);
322 	if (!ptr)
323 		return NULL;
324 
325 	delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum,
326 			       start, offset);
327 
328 	/* Adjust skb->csum since we changed the packet */
329 	NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta);
330 
331 	grc->offset = off + hdrlen + offset;
332 	grc->delta = delta;
333 
334 	return ptr;
335 }
336 
337 static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb,
338 					   struct gro_remcsum *grc)
339 {
340 	void *ptr;
341 	size_t plen = grc->offset + sizeof(u16);
342 
343 	if (!grc->delta)
344 		return;
345 
346 	ptr = skb_gro_header(skb, plen, grc->offset);
347 	if (!ptr)
348 		return;
349 
350 	remcsum_unadjust((__sum16 *)ptr, grc->delta);
351 }
352 
353 #ifdef CONFIG_XFRM_OFFLOAD
354 static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)
355 {
356 	if (PTR_ERR(pp) != -EINPROGRESS)
357 		NAPI_GRO_CB(skb)->flush |= flush;
358 }
359 static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb,
360 					       struct sk_buff *pp,
361 					       int flush,
362 					       struct gro_remcsum *grc)
363 {
364 	if (PTR_ERR(pp) != -EINPROGRESS) {
365 		NAPI_GRO_CB(skb)->flush |= flush;
366 		skb_gro_remcsum_cleanup(skb, grc);
367 		skb->remcsum_offload = 0;
368 	}
369 }
370 #else
371 static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)
372 {
373 	NAPI_GRO_CB(skb)->flush |= flush;
374 }
375 static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb,
376 					       struct sk_buff *pp,
377 					       int flush,
378 					       struct gro_remcsum *grc)
379 {
380 	NAPI_GRO_CB(skb)->flush |= flush;
381 	skb_gro_remcsum_cleanup(skb, grc);
382 	skb->remcsum_offload = 0;
383 }
384 #endif
385 
386 INDIRECT_CALLABLE_DECLARE(struct sk_buff *ipv6_gro_receive(struct list_head *,
387 							   struct sk_buff *));
388 INDIRECT_CALLABLE_DECLARE(int ipv6_gro_complete(struct sk_buff *, int));
389 INDIRECT_CALLABLE_DECLARE(struct sk_buff *inet_gro_receive(struct list_head *,
390 							   struct sk_buff *));
391 INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int));
392 
393 INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp4_gro_receive(struct list_head *,
394 							   struct sk_buff *));
395 INDIRECT_CALLABLE_DECLARE(int udp4_gro_complete(struct sk_buff *, int));
396 
397 INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp6_gro_receive(struct list_head *,
398 							   struct sk_buff *));
399 INDIRECT_CALLABLE_DECLARE(int udp6_gro_complete(struct sk_buff *, int));
400 
401 #define indirect_call_gro_receive_inet(cb, f2, f1, head, skb)	\
402 ({								\
403 	unlikely(gro_recursion_inc_test(skb)) ?			\
404 		NAPI_GRO_CB(skb)->flush |= 1, NULL :		\
405 		INDIRECT_CALL_INET(cb, f2, f1, head, skb);	\
406 })
407 
408 struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,
409 				struct udphdr *uh, struct sock *sk);
410 int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup);
411 
412 static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb)
413 {
414 	struct udphdr *uh;
415 	unsigned int hlen, off;
416 
417 	off  = skb_gro_offset(skb);
418 	hlen = off + sizeof(*uh);
419 	uh   = skb_gro_header(skb, hlen, off);
420 
421 	return uh;
422 }
423 
424 static inline __wsum ip6_gro_compute_pseudo(struct sk_buff *skb, int proto)
425 {
426 	const struct ipv6hdr *iph = skb_gro_network_header(skb);
427 
428 	return ~csum_unfold(csum_ipv6_magic(&iph->saddr, &iph->daddr,
429 					    skb_gro_len(skb), proto, 0));
430 }
431 
432 int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb);
433 
434 /* Pass the currently batched GRO_NORMAL SKBs up to the stack. */
435 static inline void gro_normal_list(struct napi_struct *napi)
436 {
437 	if (!napi->rx_count)
438 		return;
439 	netif_receive_skb_list_internal(&napi->rx_list);
440 	INIT_LIST_HEAD(&napi->rx_list);
441 	napi->rx_count = 0;
442 }
443 
444 /* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded,
445  * pass the whole batch up to the stack.
446  */
447 static inline void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs)
448 {
449 	list_add_tail(&skb->list, &napi->rx_list);
450 	napi->rx_count += segs;
451 	if (napi->rx_count >= READ_ONCE(gro_normal_batch))
452 		gro_normal_list(napi);
453 }
454 
455 extern struct list_head offload_base;
456 
457 #endif /* _NET_IPV6_GRO_H */
458