xref: /linux/net/core/skbuff.c (revision 7e1392a42ea288d70a266f6553fe06f57f8647ea)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  *	Routines having to do with the 'struct sk_buff' memory handlers.
4  *
5  *	Authors:	Alan Cox <alan@lxorguk.ukuu.org.uk>
6  *			Florian La Roche <rzsfl@rz.uni-sb.de>
7  *
8  *	Fixes:
9  *		Alan Cox	:	Fixed the worst of the load
10  *					balancer bugs.
11  *		Dave Platt	:	Interrupt stacking fix.
12  *	Richard Kooijman	:	Timestamp fixes.
13  *		Alan Cox	:	Changed buffer format.
14  *		Alan Cox	:	destructor hook for AF_UNIX etc.
15  *		Linus Torvalds	:	Better skb_clone.
16  *		Alan Cox	:	Added skb_copy.
17  *		Alan Cox	:	Added all the changed routines Linus
18  *					only put in the headers
19  *		Ray VanTassle	:	Fixed --skb->lock in free
20  *		Alan Cox	:	skb_copy copy arp field
21  *		Andi Kleen	:	slabified it.
22  *		Robert Olsson	:	Removed skb_head_pool
23  *
24  *	NOTE:
25  *		The __skb_ routines should be called with interrupts
26  *	disabled, or you better be *real* sure that the operation is atomic
27  *	with respect to whatever list is being frobbed (e.g. via lock_sock()
28  *	or via disabling bottom half handlers, etc).
29  */
30 
31 /*
32  *	The functions in this file will not compile correctly with gcc 2.4.x
33  */
34 
35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36 
37 #include <linux/module.h>
38 #include <linux/types.h>
39 #include <linux/kernel.h>
40 #include <linux/mm.h>
41 #include <linux/interrupt.h>
42 #include <linux/in.h>
43 #include <linux/inet.h>
44 #include <linux/slab.h>
45 #include <linux/tcp.h>
46 #include <linux/udp.h>
47 #include <linux/sctp.h>
48 #include <linux/netdevice.h>
49 #ifdef CONFIG_NET_CLS_ACT
50 #include <net/pkt_sched.h>
51 #endif
52 #include <linux/string.h>
53 #include <linux/skbuff.h>
54 #include <linux/skbuff_ref.h>
55 #include <linux/splice.h>
56 #include <linux/cache.h>
57 #include <linux/rtnetlink.h>
58 #include <linux/init.h>
59 #include <linux/scatterlist.h>
60 #include <linux/errqueue.h>
61 #include <linux/prefetch.h>
62 #include <linux/bitfield.h>
63 #include <linux/if_vlan.h>
64 #include <linux/mpls.h>
65 #include <linux/kcov.h>
66 #include <linux/iov_iter.h>
67 #include <linux/crc32.h>
68 #include <kunit/visibility.h>
69 
70 #include <net/protocol.h>
71 #include <net/dst.h>
72 #include <net/sock.h>
73 #include <net/checksum.h>
74 #include <net/gro.h>
75 #include <net/gso.h>
76 #include <net/hotdata.h>
77 #include <net/ip6_checksum.h>
78 #include <net/xfrm.h>
79 #include <net/mpls.h>
80 #include <net/mptcp.h>
81 #include <net/mctp.h>
82 #include <net/tcp.h>
83 #include <net/can.h>
84 #include <net/page_pool/helpers.h>
85 #include <net/psp/types.h>
86 #include <net/dropreason.h>
87 #include <net/xdp_sock.h>
88 
89 #include <linux/uaccess.h>
90 #include <trace/events/skb.h>
91 #include <linux/highmem.h>
92 #include <linux/capability.h>
93 #include <linux/user_namespace.h>
94 #include <linux/indirect_call_wrapper.h>
95 #include <linux/textsearch.h>
96 
97 #include "dev.h"
98 #include "devmem.h"
99 #include "net-sysfs.h"
100 #include "netmem_priv.h"
101 
102 #ifdef CONFIG_SKB_EXTENSIONS
103 static struct kmem_cache *skbuff_ext_cache __ro_after_init;
104 #endif
105 
106 #define GRO_MAX_HEAD_PAD (GRO_MAX_HEAD + NET_SKB_PAD + NET_IP_ALIGN)
107 #define SKB_SMALL_HEAD_SIZE SKB_HEAD_ALIGN(max(MAX_TCP_HEADER, \
108 					       GRO_MAX_HEAD_PAD))
109 
110 /* SKB_SMALL_HEAD_CACHE_SIZE is the size used for the skbuff_small_head
111  * kmem_cache. The non-power-of-2 padding is kept for historical reasons and
112  * to avoid potential collisions with generic kmalloc bucket sizes.
113  */
114 #define SKB_SMALL_HEAD_CACHE_SIZE					\
115 	(is_power_of_2(SKB_SMALL_HEAD_SIZE) ?			\
116 		(SKB_SMALL_HEAD_SIZE + L1_CACHE_BYTES) :	\
117 		SKB_SMALL_HEAD_SIZE)
118 
119 #define SKB_SMALL_HEAD_HEADROOM						\
120 	SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE)
121 
122 /* kcm_write_msgs() relies on casting paged frags to bio_vec to use
123  * iov_iter_bvec(). These static asserts ensure the cast is valid is long as the
124  * netmem is a page.
125  */
126 static_assert(offsetof(struct bio_vec, bv_page) ==
127 	      offsetof(skb_frag_t, netmem));
128 static_assert(sizeof_field(struct bio_vec, bv_page) ==
129 	      sizeof_field(skb_frag_t, netmem));
130 
131 static_assert(offsetof(struct bio_vec, bv_len) == offsetof(skb_frag_t, len));
132 static_assert(sizeof_field(struct bio_vec, bv_len) ==
133 	      sizeof_field(skb_frag_t, len));
134 
135 static_assert(offsetof(struct bio_vec, bv_offset) ==
136 	      offsetof(skb_frag_t, offset));
137 static_assert(sizeof_field(struct bio_vec, bv_offset) ==
138 	      sizeof_field(skb_frag_t, offset));
139 
140 #undef FN
141 #define FN(reason) [SKB_DROP_REASON_##reason] = #reason,
142 static const char * const drop_reasons[] = {
143 	[SKB_CONSUMED] = "CONSUMED",
144 	DEFINE_DROP_REASON(FN, FN)
145 };
146 
147 static const struct drop_reason_list drop_reasons_core = {
148 	.reasons = drop_reasons,
149 	.n_reasons = ARRAY_SIZE(drop_reasons),
150 };
151 
152 const struct drop_reason_list __rcu *
153 drop_reasons_by_subsys[SKB_DROP_REASON_SUBSYS_NUM] = {
154 	[SKB_DROP_REASON_SUBSYS_CORE] = RCU_INITIALIZER(&drop_reasons_core),
155 };
156 EXPORT_SYMBOL(drop_reasons_by_subsys);
157 
158 /**
159  * drop_reasons_register_subsys - register another drop reason subsystem
160  * @subsys: the subsystem to register, must not be the core
161  * @list: the list of drop reasons within the subsystem, must point to
162  *	a statically initialized list
163  */
164 void drop_reasons_register_subsys(enum skb_drop_reason_subsys subsys,
165 				  const struct drop_reason_list *list)
166 {
167 	if (WARN(subsys <= SKB_DROP_REASON_SUBSYS_CORE ||
168 		 subsys >= ARRAY_SIZE(drop_reasons_by_subsys),
169 		 "invalid subsystem %d\n", subsys))
170 		return;
171 
172 	/* must point to statically allocated memory, so INIT is OK */
173 	RCU_INIT_POINTER(drop_reasons_by_subsys[subsys], list);
174 }
175 EXPORT_SYMBOL_GPL(drop_reasons_register_subsys);
176 
177 /**
178  * drop_reasons_unregister_subsys - unregister a drop reason subsystem
179  * @subsys: the subsystem to remove, must not be the core
180  *
181  * Note: This will synchronize_rcu() to ensure no users when it returns.
182  */
183 void drop_reasons_unregister_subsys(enum skb_drop_reason_subsys subsys)
184 {
185 	if (WARN(subsys <= SKB_DROP_REASON_SUBSYS_CORE ||
186 		 subsys >= ARRAY_SIZE(drop_reasons_by_subsys),
187 		 "invalid subsystem %d\n", subsys))
188 		return;
189 
190 	RCU_INIT_POINTER(drop_reasons_by_subsys[subsys], NULL);
191 
192 	synchronize_rcu();
193 }
194 EXPORT_SYMBOL_GPL(drop_reasons_unregister_subsys);
195 
196 /**
197  *	skb_panic - private function for out-of-line support
198  *	@skb:	buffer
199  *	@sz:	size
200  *	@addr:	address
201  *	@msg:	skb_over_panic or skb_under_panic
202  *
203  *	Out-of-line support for skb_put() and skb_push().
204  *	Called via the wrapper skb_over_panic() or skb_under_panic().
205  *	Keep out of line to prevent kernel bloat.
206  *	__builtin_return_address is not used because it is not always reliable.
207  */
208 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
209 		      const char msg[])
210 {
211 	pr_emerg("%s: text:%px len:%d put:%d head:%px data:%px tail:%#lx end:%#lx dev:%s\n",
212 		 msg, addr, skb->len, sz, skb->head, skb->data,
213 		 (unsigned long)skb->tail, (unsigned long)skb->end,
214 		 skb->dev ? skb->dev->name : "<NULL>");
215 	BUG();
216 }
217 
218 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
219 {
220 	skb_panic(skb, sz, addr, __func__);
221 }
222 
223 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
224 {
225 	skb_panic(skb, sz, addr, __func__);
226 }
227 
228 #define NAPI_SKB_CACHE_SIZE	128
229 #define NAPI_SKB_CACHE_BULK	32
230 #define NAPI_SKB_CACHE_FREE	32
231 
232 struct napi_alloc_cache {
233 	local_lock_t bh_lock;
234 	struct page_frag_cache page;
235 	unsigned int skb_count;
236 	void *skb_cache[NAPI_SKB_CACHE_SIZE];
237 };
238 
239 static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
240 static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache) = {
241 	.bh_lock = INIT_LOCAL_LOCK(bh_lock),
242 };
243 
244 void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask)
245 {
246 	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
247 	void *data;
248 
249 	fragsz = SKB_DATA_ALIGN(fragsz);
250 
251 	local_lock_nested_bh(&napi_alloc_cache.bh_lock);
252 	data = __page_frag_alloc_align(&nc->page, fragsz,
253 				       GFP_ATOMIC | __GFP_NOWARN, align_mask);
254 	local_unlock_nested_bh(&napi_alloc_cache.bh_lock);
255 	return data;
256 
257 }
258 EXPORT_SYMBOL(__napi_alloc_frag_align);
259 
260 void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask)
261 {
262 	void *data;
263 
264 	if (in_hardirq() || irqs_disabled()) {
265 		struct page_frag_cache *nc = this_cpu_ptr(&netdev_alloc_cache);
266 
267 		fragsz = SKB_DATA_ALIGN(fragsz);
268 		data = __page_frag_alloc_align(nc, fragsz,
269 					       GFP_ATOMIC | __GFP_NOWARN,
270 					       align_mask);
271 	} else {
272 		local_bh_disable();
273 		data = __napi_alloc_frag_align(fragsz, align_mask);
274 		local_bh_enable();
275 	}
276 	return data;
277 }
278 EXPORT_SYMBOL(__netdev_alloc_frag_align);
279 
280 /* Cache kmem_cache_size(net_hotdata.skbuff_cache) to help the compiler
281  * remove dead code (and skbuff_cache_size) when CONFIG_KASAN is unset.
282  */
283 static u32 skbuff_cache_size __read_mostly;
284 
285 static inline struct sk_buff *napi_skb_cache_get(bool alloc)
286 {
287 	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
288 	struct sk_buff *skb;
289 
290 	local_lock_nested_bh(&napi_alloc_cache.bh_lock);
291 	if (unlikely(!nc->skb_count)) {
292 		if (alloc && kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
293 						   GFP_ATOMIC | __GFP_NOWARN,
294 						   NAPI_SKB_CACHE_BULK,
295 						   nc->skb_cache))
296 			nc->skb_count = NAPI_SKB_CACHE_BULK;
297 		if (unlikely(!nc->skb_count)) {
298 			local_unlock_nested_bh(&napi_alloc_cache.bh_lock);
299 			return NULL;
300 		}
301 	}
302 
303 	skb = nc->skb_cache[--nc->skb_count];
304 	if (nc->skb_count)
305 		prefetch(nc->skb_cache[nc->skb_count - 1]);
306 	local_unlock_nested_bh(&napi_alloc_cache.bh_lock);
307 	kasan_mempool_unpoison_object(skb, skbuff_cache_size);
308 
309 	return skb;
310 }
311 
312 /*
313  * Only clear those fields we need to clear, not those that we will
314  * actually initialise later. Hence, don't put any more fields after
315  * the tail pointer in struct sk_buff!
316  */
317 static inline void skbuff_clear(struct sk_buff *skb)
318 {
319 	/* Replace memset(skb, 0, offsetof(struct sk_buff, tail))
320 	 * with two smaller memset(), with a barrier() between them.
321 	 * This forces the compiler to inline both calls.
322 	 */
323 	BUILD_BUG_ON(offsetof(struct sk_buff, tail) <= 128);
324 	memset(skb, 0, 128);
325 	barrier();
326 	memset((void *)skb + 128, 0, offsetof(struct sk_buff, tail) - 128);
327 }
328 
329 /**
330  * napi_skb_cache_get_bulk - obtain a number of zeroed skb heads from the cache
331  * @skbs: pointer to an at least @n-sized array to fill with skb pointers
332  * @n: number of entries to provide
333  *
334  * Tries to obtain @n &sk_buff entries from the NAPI percpu cache and writes
335  * the pointers into the provided array @skbs. If there are less entries
336  * available, tries to replenish the cache and bulk-allocates the diff from
337  * the MM layer if needed.
338  * The heads are being zeroed with either memset() or %__GFP_ZERO, so they are
339  * ready for {,__}build_skb_around() and don't have any data buffers attached.
340  * Must be called *only* from the BH context.
341  *
342  * Return: number of successfully allocated skbs (@n if no actual allocation
343  *	   needed or kmem_cache_alloc_bulk() didn't fail).
344  */
345 u32 napi_skb_cache_get_bulk(void **skbs, u32 n)
346 {
347 	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
348 	u32 bulk, total = n;
349 
350 	local_lock_nested_bh(&napi_alloc_cache.bh_lock);
351 
352 	if (nc->skb_count >= n)
353 		goto get;
354 
355 	/* No enough cached skbs. Try refilling the cache first */
356 	bulk = min(NAPI_SKB_CACHE_SIZE - nc->skb_count, NAPI_SKB_CACHE_BULK);
357 	if (kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
358 				  GFP_ATOMIC | __GFP_NOWARN, bulk,
359 				  &nc->skb_cache[nc->skb_count]))
360 		nc->skb_count += bulk;
361 	if (likely(nc->skb_count >= n))
362 		goto get;
363 
364 	/* Still not enough. Bulk-allocate the missing part directly, zeroed */
365 	if (kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
366 				  GFP_ATOMIC | __GFP_ZERO | __GFP_NOWARN,
367 				  n - nc->skb_count, &skbs[nc->skb_count]))
368 		n = nc->skb_count;
369 	if (likely(nc->skb_count >= n))
370 		goto get;
371 
372 	/* kmem_cache didn't allocate the number we need, limit the output */
373 	total -= n - nc->skb_count;
374 	n = nc->skb_count;
375 
376 get:
377 	for (u32 base = nc->skb_count - n, i = 0; i < n; i++) {
378 		skbs[i] = nc->skb_cache[base + i];
379 
380 		kasan_mempool_unpoison_object(skbs[i], skbuff_cache_size);
381 		skbuff_clear(skbs[i]);
382 	}
383 
384 	nc->skb_count -= n;
385 	local_unlock_nested_bh(&napi_alloc_cache.bh_lock);
386 
387 	return total;
388 }
389 EXPORT_SYMBOL_GPL(napi_skb_cache_get_bulk);
390 
391 static inline void __finalize_skb_around(struct sk_buff *skb, void *data,
392 					 unsigned int size)
393 {
394 	struct skb_shared_info *shinfo;
395 
396 	size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
397 
398 	/* Assumes caller memset cleared SKB */
399 	skb->truesize = SKB_TRUESIZE(size);
400 	refcount_set(&skb->users, 1);
401 	skb->head = data;
402 	skb->data = data;
403 	skb_reset_tail_pointer(skb);
404 	skb_set_end_offset(skb, size);
405 	skb->mac_header = (typeof(skb->mac_header))~0U;
406 	skb->transport_header = (typeof(skb->transport_header))~0U;
407 	skb->alloc_cpu = raw_smp_processor_id();
408 	/* make sure we initialize shinfo sequentially */
409 	shinfo = skb_shinfo(skb);
410 	memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
411 	atomic_set(&shinfo->dataref, 1);
412 
413 	skb_set_kcov_handle(skb, kcov_common_handle());
414 }
415 
416 static inline void *__slab_build_skb(void *data, unsigned int *size)
417 {
418 	void *resized;
419 
420 	/* Must find the allocation size (and grow it to match). */
421 	*size = ksize(data);
422 	/* krealloc() will immediately return "data" when
423 	 * "ksize(data)" is requested: it is the existing upper
424 	 * bounds. As a result, GFP_ATOMIC will be ignored. Note
425 	 * that this "new" pointer needs to be passed back to the
426 	 * caller for use so the __alloc_size hinting will be
427 	 * tracked correctly.
428 	 */
429 	resized = krealloc(data, *size, GFP_ATOMIC);
430 	WARN_ON_ONCE(resized != data);
431 	return resized;
432 }
433 
434 /* build_skb() variant which can operate on slab buffers.
435  * Note that this should be used sparingly as slab buffers
436  * cannot be combined efficiently by GRO!
437  */
438 struct sk_buff *slab_build_skb(void *data)
439 {
440 	struct sk_buff *skb;
441 	unsigned int size;
442 
443 	skb = kmem_cache_alloc(net_hotdata.skbuff_cache,
444 			       GFP_ATOMIC | __GFP_NOWARN);
445 	if (unlikely(!skb))
446 		return NULL;
447 
448 	skbuff_clear(skb);
449 	data = __slab_build_skb(data, &size);
450 	__finalize_skb_around(skb, data, size);
451 
452 	return skb;
453 }
454 EXPORT_SYMBOL(slab_build_skb);
455 
456 /* Caller must provide SKB that is memset cleared */
457 static void __build_skb_around(struct sk_buff *skb, void *data,
458 			       unsigned int frag_size)
459 {
460 	unsigned int size = frag_size;
461 
462 	/* frag_size == 0 is considered deprecated now. Callers
463 	 * using slab buffer should use slab_build_skb() instead.
464 	 */
465 	if (WARN_ONCE(size == 0, "Use slab_build_skb() instead"))
466 		data = __slab_build_skb(data, &size);
467 
468 	__finalize_skb_around(skb, data, size);
469 }
470 
471 /**
472  * __build_skb - build a network buffer
473  * @data: data buffer provided by caller
474  * @frag_size: size of data (must not be 0)
475  *
476  * Allocate a new &sk_buff. Caller provides space holding head and
477  * skb_shared_info. @data must have been allocated from the page
478  * allocator or vmalloc(). (A @frag_size of 0 to indicate a kmalloc()
479  * allocation is deprecated, and callers should use slab_build_skb()
480  * instead.)
481  * The return is the new skb buffer.
482  * On a failure the return is %NULL, and @data is not freed.
483  * Notes :
484  *  Before IO, driver allocates only data buffer where NIC put incoming frame
485  *  Driver should add room at head (NET_SKB_PAD) and
486  *  MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info))
487  *  After IO, driver calls build_skb(), to allocate sk_buff and populate it
488  *  before giving packet to stack.
489  *  RX rings only contains data buffers, not full skbs.
490  */
491 struct sk_buff *__build_skb(void *data, unsigned int frag_size)
492 {
493 	struct sk_buff *skb;
494 
495 	skb = kmem_cache_alloc(net_hotdata.skbuff_cache,
496 			       GFP_ATOMIC | __GFP_NOWARN);
497 	if (unlikely(!skb))
498 		return NULL;
499 
500 	skbuff_clear(skb);
501 	__build_skb_around(skb, data, frag_size);
502 
503 	return skb;
504 }
505 
506 /* build_skb() is wrapper over __build_skb(), that specifically
507  * takes care of skb->head and skb->pfmemalloc
508  */
509 struct sk_buff *build_skb(void *data, unsigned int frag_size)
510 {
511 	struct sk_buff *skb = __build_skb(data, frag_size);
512 
513 	if (likely(skb && frag_size)) {
514 		skb->head_frag = 1;
515 		skb_propagate_pfmemalloc(virt_to_head_page(data), skb);
516 	}
517 	return skb;
518 }
519 EXPORT_SYMBOL(build_skb);
520 
521 /**
522  * build_skb_around - build a network buffer around provided skb
523  * @skb: sk_buff provide by caller, must be memset cleared
524  * @data: data buffer provided by caller
525  * @frag_size: size of data
526  */
527 struct sk_buff *build_skb_around(struct sk_buff *skb,
528 				 void *data, unsigned int frag_size)
529 {
530 	if (unlikely(!skb))
531 		return NULL;
532 
533 	__build_skb_around(skb, data, frag_size);
534 
535 	if (frag_size) {
536 		skb->head_frag = 1;
537 		skb_propagate_pfmemalloc(virt_to_head_page(data), skb);
538 	}
539 	return skb;
540 }
541 EXPORT_SYMBOL(build_skb_around);
542 
543 /**
544  * __napi_build_skb - build a network buffer
545  * @data: data buffer provided by caller
546  * @frag_size: size of data
547  *
548  * Version of __build_skb() that uses NAPI percpu caches to obtain
549  * skbuff_head instead of inplace allocation.
550  *
551  * Returns a new &sk_buff on success, %NULL on allocation failure.
552  */
553 static struct sk_buff *__napi_build_skb(void *data, unsigned int frag_size)
554 {
555 	struct sk_buff *skb;
556 
557 	skb = napi_skb_cache_get(true);
558 	if (unlikely(!skb))
559 		return NULL;
560 
561 	skbuff_clear(skb);
562 	__build_skb_around(skb, data, frag_size);
563 
564 	return skb;
565 }
566 
567 /**
568  * napi_build_skb - build a network buffer
569  * @data: data buffer provided by caller
570  * @frag_size: size of data
571  *
572  * Version of __napi_build_skb() that takes care of skb->head_frag
573  * and skb->pfmemalloc when the data is a page or page fragment.
574  *
575  * Returns a new &sk_buff on success, %NULL on allocation failure.
576  */
577 struct sk_buff *napi_build_skb(void *data, unsigned int frag_size)
578 {
579 	struct sk_buff *skb = __napi_build_skb(data, frag_size);
580 
581 	if (likely(skb) && frag_size) {
582 		skb->head_frag = 1;
583 		skb_propagate_pfmemalloc(virt_to_head_page(data), skb);
584 	}
585 
586 	return skb;
587 }
588 EXPORT_SYMBOL(napi_build_skb);
589 
590 static void *kmalloc_pfmemalloc(size_t obj_size, gfp_t flags, int node)
591 {
592 	if (!gfp_pfmemalloc_allowed(flags))
593 		return NULL;
594 	if (!obj_size)
595 		return kmem_cache_alloc_node(net_hotdata.skb_small_head_cache,
596 					     flags, node);
597 	return kmalloc_node_track_caller(obj_size, flags, node);
598 }
599 
600 /*
601  * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells
602  * the caller if emergency pfmemalloc reserves are being used. If it is and
603  * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves
604  * may be used. Otherwise, the packet data may be discarded until enough
605  * memory is free
606  */
607 static void *kmalloc_reserve(unsigned int *size, gfp_t flags, int node,
608 			     struct sk_buff *skb)
609 {
610 	size_t obj_size;
611 	void *obj;
612 
613 	obj_size = SKB_HEAD_ALIGN(*size);
614 	if (obj_size <= SKB_SMALL_HEAD_CACHE_SIZE &&
615 	    !(flags & KMALLOC_NOT_NORMAL_BITS)) {
616 		obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache,
617 				flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
618 				node);
619 		*size = SKB_SMALL_HEAD_CACHE_SIZE;
620 		if (likely(obj))
621 			goto out;
622 		/* Try again but now we are using pfmemalloc reserves */
623 		if (skb)
624 			skb->pfmemalloc = true;
625 		return kmalloc_pfmemalloc(0, flags, node);
626 	}
627 
628 	obj_size = kmalloc_size_roundup(obj_size);
629 	/* The following cast might truncate high-order bits of obj_size, this
630 	 * is harmless because kmalloc(obj_size >= 2^32) will fail anyway.
631 	 */
632 	*size = (unsigned int)obj_size;
633 
634 	/*
635 	 * Try a regular allocation, when that fails and we're not entitled
636 	 * to the reserves, fail.
637 	 */
638 	obj = kmalloc_node_track_caller(obj_size,
639 					flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
640 					node);
641 	if (likely(obj))
642 		goto out;
643 
644 	/* Try again but now we are using pfmemalloc reserves */
645 	if (skb)
646 		skb->pfmemalloc = true;
647 	obj = kmalloc_pfmemalloc(obj_size, flags, node);
648 out:
649 	return obj;
650 }
651 
652 /* 	Allocate a new skbuff. We do this ourselves so we can fill in a few
653  *	'private' fields and also do memory statistics to find all the
654  *	[BEEP] leaks.
655  *
656  */
657 
658 /**
659  *	__alloc_skb	-	allocate a network buffer
660  *	@size: size to allocate
661  *	@gfp_mask: allocation mask
662  *	@flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache
663  *		instead of head cache and allocate a cloned (child) skb.
664  *		If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
665  *		allocations in case the data is required for writeback
666  *	@node: numa node to allocate memory on
667  *
668  *	Allocate a new &sk_buff. The returned buffer has no headroom and a
669  *	tail room of at least size bytes. The object has a reference count
670  *	of one. The return is the buffer. On a failure the return is %NULL.
671  *
672  *	Buffers may only be allocated from interrupts using a @gfp_mask of
673  *	%GFP_ATOMIC.
674  */
675 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
676 			    int flags, int node)
677 {
678 	struct sk_buff *skb = NULL;
679 	struct kmem_cache *cache;
680 	u8 *data;
681 
682 	if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
683 		gfp_mask |= __GFP_MEMALLOC;
684 
685 	if (flags & SKB_ALLOC_FCLONE) {
686 		cache = net_hotdata.skbuff_fclone_cache;
687 		goto fallback;
688 	}
689 	cache = net_hotdata.skbuff_cache;
690 	if (unlikely(node != NUMA_NO_NODE && node != numa_mem_id()))
691 		goto fallback;
692 
693 	if (flags & SKB_ALLOC_NAPI) {
694 		skb = napi_skb_cache_get(true);
695 		if (unlikely(!skb))
696 			return NULL;
697 	} else if (!in_hardirq() && !irqs_disabled()) {
698 		local_bh_disable();
699 		skb = napi_skb_cache_get(false);
700 		local_bh_enable();
701 	}
702 
703 	if (!skb) {
704 fallback:
705 		skb = kmem_cache_alloc_node(cache, gfp_mask & ~GFP_DMA, node);
706 		if (unlikely(!skb))
707 			return NULL;
708 	}
709 	skbuff_clear(skb);
710 
711 	/* We do our best to align skb_shared_info on a separate cache
712 	 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
713 	 * aligned memory blocks, unless SLUB/SLAB debug is enabled.
714 	 * Both skb->head and skb_shared_info are cache line aligned.
715 	 */
716 	data = kmalloc_reserve(&size, gfp_mask, node, skb);
717 	if (unlikely(!data))
718 		goto nodata;
719 	/* kmalloc_size_roundup() might give us more room than requested.
720 	 * Put skb_shared_info exactly at the end of allocated zone,
721 	 * to allow max possible filling before reallocation.
722 	 */
723 	__finalize_skb_around(skb, data, size);
724 
725 	if (flags & SKB_ALLOC_FCLONE) {
726 		struct sk_buff_fclones *fclones;
727 
728 		fclones = container_of(skb, struct sk_buff_fclones, skb1);
729 
730 		/* skb->fclone is a 2bits field.
731 		 * Replace expensive RMW (skb->fclone = SKB_FCLONE_ORIG)
732 		 * with a single OR.
733 		 */
734 		BUILD_BUG_ON(SKB_FCLONE_UNAVAILABLE != 0);
735 		DEBUG_NET_WARN_ON_ONCE(skb->fclone != SKB_FCLONE_UNAVAILABLE);
736 		skb->fclone |= SKB_FCLONE_ORIG;
737 
738 		refcount_set(&fclones->fclone_ref, 1);
739 	}
740 
741 	return skb;
742 
743 nodata:
744 	kmem_cache_free(cache, skb);
745 	return NULL;
746 }
747 EXPORT_SYMBOL(__alloc_skb);
748 
749 /**
750  *	__netdev_alloc_skb - allocate an skbuff for rx on a specific device
751  *	@dev: network device to receive on
752  *	@len: length to allocate
753  *	@gfp_mask: get_free_pages mask, passed to alloc_skb
754  *
755  *	Allocate a new &sk_buff and assign it a usage count of one. The
756  *	buffer has NET_SKB_PAD headroom built in. Users should allocate
757  *	the headroom they think they need without accounting for the
758  *	built in space. The built in space is used for optimisations.
759  *
760  *	%NULL is returned if there is no free memory.
761  */
762 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len,
763 				   gfp_t gfp_mask)
764 {
765 	struct page_frag_cache *nc;
766 	struct sk_buff *skb;
767 	bool pfmemalloc;
768 	void *data;
769 
770 	len += NET_SKB_PAD;
771 
772 	/* If requested length is either too small or too big,
773 	 * we use kmalloc() for skb->head allocation.
774 	 */
775 	if (len <= SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE) ||
776 	    len > SKB_WITH_OVERHEAD(PAGE_SIZE) ||
777 	    (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
778 		skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
779 		if (!skb)
780 			goto skb_fail;
781 		goto skb_success;
782 	}
783 
784 	len = SKB_HEAD_ALIGN(len);
785 
786 	if (sk_memalloc_socks())
787 		gfp_mask |= __GFP_MEMALLOC;
788 
789 	if (in_hardirq() || irqs_disabled()) {
790 		nc = this_cpu_ptr(&netdev_alloc_cache);
791 		data = page_frag_alloc(nc, len, gfp_mask);
792 		pfmemalloc = page_frag_cache_is_pfmemalloc(nc);
793 	} else {
794 		local_bh_disable();
795 		local_lock_nested_bh(&napi_alloc_cache.bh_lock);
796 
797 		nc = this_cpu_ptr(&napi_alloc_cache.page);
798 		data = page_frag_alloc(nc, len, gfp_mask);
799 		pfmemalloc = page_frag_cache_is_pfmemalloc(nc);
800 
801 		local_unlock_nested_bh(&napi_alloc_cache.bh_lock);
802 		local_bh_enable();
803 	}
804 
805 	if (unlikely(!data))
806 		return NULL;
807 
808 	skb = __build_skb(data, len);
809 	if (unlikely(!skb)) {
810 		skb_free_frag(data);
811 		return NULL;
812 	}
813 
814 	if (pfmemalloc)
815 		skb->pfmemalloc = 1;
816 	skb->head_frag = 1;
817 
818 skb_success:
819 	skb_reserve(skb, NET_SKB_PAD);
820 	skb->dev = dev;
821 
822 skb_fail:
823 	return skb;
824 }
825 EXPORT_SYMBOL(__netdev_alloc_skb);
826 
827 /**
828  *	napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance
829  *	@napi: napi instance this buffer was allocated for
830  *	@len: length to allocate
831  *
832  *	Allocate a new sk_buff for use in NAPI receive.  This buffer will
833  *	attempt to allocate the head from a special reserved region used
834  *	only for NAPI Rx allocation.  By doing this we can save several
835  *	CPU cycles by avoiding having to disable and re-enable IRQs.
836  *
837  *	%NULL is returned if there is no free memory.
838  */
839 struct sk_buff *napi_alloc_skb(struct napi_struct *napi, unsigned int len)
840 {
841 	gfp_t gfp_mask = GFP_ATOMIC | __GFP_NOWARN;
842 	struct napi_alloc_cache *nc;
843 	struct sk_buff *skb;
844 	bool pfmemalloc;
845 	void *data;
846 
847 	DEBUG_NET_WARN_ON_ONCE(!in_softirq());
848 	len += NET_SKB_PAD + NET_IP_ALIGN;
849 
850 	/* If requested length is either too small or too big,
851 	 * we use kmalloc() for skb->head allocation.
852 	 */
853 	if (len <= SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE) ||
854 	    len > SKB_WITH_OVERHEAD(PAGE_SIZE) ||
855 	    (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
856 		skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX | SKB_ALLOC_NAPI,
857 				  NUMA_NO_NODE);
858 		if (!skb)
859 			goto skb_fail;
860 		goto skb_success;
861 	}
862 
863 	len = SKB_HEAD_ALIGN(len);
864 
865 	if (sk_memalloc_socks())
866 		gfp_mask |= __GFP_MEMALLOC;
867 
868 	local_lock_nested_bh(&napi_alloc_cache.bh_lock);
869 	nc = this_cpu_ptr(&napi_alloc_cache);
870 
871 	data = page_frag_alloc(&nc->page, len, gfp_mask);
872 	pfmemalloc = page_frag_cache_is_pfmemalloc(&nc->page);
873 	local_unlock_nested_bh(&napi_alloc_cache.bh_lock);
874 
875 	if (unlikely(!data))
876 		return NULL;
877 
878 	skb = __napi_build_skb(data, len);
879 	if (unlikely(!skb)) {
880 		skb_free_frag(data);
881 		return NULL;
882 	}
883 
884 	if (pfmemalloc)
885 		skb->pfmemalloc = 1;
886 	skb->head_frag = 1;
887 
888 skb_success:
889 	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
890 	skb->dev = napi->dev;
891 
892 skb_fail:
893 	return skb;
894 }
895 EXPORT_SYMBOL(napi_alloc_skb);
896 
897 
898 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
899 			  unsigned int truesize)
900 {
901 	skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
902 
903 	DEBUG_NET_WARN_ON_ONCE(size > truesize);
904 
905 	skb_frag_size_add(frag, size);
906 	skb->len += size;
907 	skb->data_len += size;
908 	skb->truesize += truesize;
909 }
910 EXPORT_SYMBOL(skb_coalesce_rx_frag);
911 
912 static void skb_drop_list(struct sk_buff **listp)
913 {
914 	kfree_skb_list(*listp);
915 	*listp = NULL;
916 }
917 
918 static inline void skb_drop_fraglist(struct sk_buff *skb)
919 {
920 	skb_drop_list(&skb_shinfo(skb)->frag_list);
921 }
922 
923 static void skb_clone_fraglist(struct sk_buff *skb)
924 {
925 	struct sk_buff *list;
926 
927 	skb_walk_frags(skb, list)
928 		skb_get(list);
929 }
930 
931 /**
932  * skb_pp_cow_data() - copy skb data into page-pool backed storage
933  * @pool: page pool to allocate from
934  * @pskb: pointer to skb pointer, replaced with the copied skb on success
935  * @headroom: headroom to reserve in the copied skb
936  *
937  * skb_copy_bits() handles both frags[] and frag_list input. If the copied
938  * skb remains non-linear, it uses frags[], which is the representation used
939  * by XDP multi-buffer.
940  *
941  * Return: 0 on success or a negative errno on failure.
942  */
943 int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb,
944 		    unsigned int headroom)
945 {
946 #if IS_ENABLED(CONFIG_PAGE_POOL)
947 	u32 size, truesize, len, max_head_size, off;
948 	struct sk_buff *skb = *pskb, *nskb;
949 	int err, i, head_off;
950 	void *data;
951 
952 	max_head_size = SKB_WITH_OVERHEAD(PAGE_SIZE - headroom);
953 	if (skb->len > max_head_size + MAX_SKB_FRAGS * PAGE_SIZE)
954 		return -ENOMEM;
955 
956 	size = min_t(u32, skb->len, max_head_size);
957 	truesize = SKB_HEAD_ALIGN(size) + headroom;
958 	data = page_pool_dev_alloc_va(pool, &truesize);
959 	if (!data)
960 		return -ENOMEM;
961 
962 	nskb = napi_build_skb(data, truesize);
963 	if (!nskb) {
964 		page_pool_free_va(pool, data, true);
965 		return -ENOMEM;
966 	}
967 
968 	skb_reserve(nskb, headroom);
969 	skb_copy_header(nskb, skb);
970 	nskb->mac_len = skb->mac_len;
971 	skb_mark_for_recycle(nskb);
972 
973 	err = skb_copy_bits(skb, 0, nskb->data, size);
974 	if (err) {
975 		consume_skb(nskb);
976 		return err;
977 	}
978 	skb_put(nskb, size);
979 
980 	head_off = skb_headroom(nskb) - skb_headroom(skb);
981 	skb_headers_offset_update(nskb, head_off);
982 
983 	off = size;
984 	len = skb->len - off;
985 	for (i = 0; i < MAX_SKB_FRAGS && off < skb->len; i++) {
986 		struct page *page;
987 		u32 page_off;
988 
989 		size = min_t(u32, len, PAGE_SIZE);
990 		truesize = size;
991 
992 		page = page_pool_dev_alloc(pool, &page_off, &truesize);
993 		if (!page) {
994 			consume_skb(nskb);
995 			return -ENOMEM;
996 		}
997 
998 		skb_add_rx_frag(nskb, i, page, page_off, size, truesize);
999 		err = skb_copy_bits(skb, off, page_address(page) + page_off,
1000 				    size);
1001 		if (err) {
1002 			consume_skb(nskb);
1003 			return err;
1004 		}
1005 
1006 		len -= size;
1007 		off += size;
1008 	}
1009 
1010 	consume_skb(skb);
1011 	*pskb = nskb;
1012 
1013 	return 0;
1014 #else
1015 	return -EOPNOTSUPP;
1016 #endif
1017 }
1018 EXPORT_SYMBOL(skb_pp_cow_data);
1019 
1020 int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb,
1021 			 const struct bpf_prog *prog)
1022 {
1023 	if (!prog->aux->xdp_has_frags)
1024 		return -EINVAL;
1025 
1026 	return skb_pp_cow_data(pool, pskb, XDP_PACKET_HEADROOM);
1027 }
1028 EXPORT_SYMBOL(skb_cow_data_for_xdp);
1029 
1030 #if IS_ENABLED(CONFIG_PAGE_POOL)
1031 /**
1032  * napi_pp_put_page - release a page_pool reference on a netmem
1033  * @netmem: netmem to unreference
1034  *
1035  * Drops a page_pool reference (pp_ref_count) on @netmem if it belongs to a
1036  * page_pool. Counterpart to napi_pp_get_page().
1037  *
1038  * Return: true if @netmem belongs to a page_pool, false otherwise.
1039  */
1040 bool napi_pp_put_page(netmem_ref netmem)
1041 {
1042 	netmem = netmem_compound_head(netmem);
1043 
1044 	if (unlikely(!netmem_is_pp(netmem)))
1045 		return false;
1046 
1047 	page_pool_put_full_netmem(netmem_get_pp(netmem), netmem, false);
1048 
1049 	return true;
1050 }
1051 EXPORT_SYMBOL(napi_pp_put_page);
1052 
1053 /**
1054  * napi_pp_get_page - acquire a page_pool reference on a netmem
1055  * @netmem: netmem to reference
1056  *
1057  * Acquires a page_pool reference (pp_ref_count) on @netmem if it belongs to a
1058  * page_pool. Counterpart to napi_pp_put_page().
1059  *
1060  * Return: true if @netmem belongs to a page_pool, false otherwise.
1061  */
1062 bool napi_pp_get_page(netmem_ref netmem)
1063 {
1064 	netmem = netmem_compound_head(netmem);
1065 
1066 	if (unlikely(!netmem_is_pp(netmem)))
1067 		return false;
1068 
1069 	page_pool_ref_netmem(netmem);
1070 
1071 	return true;
1072 }
1073 EXPORT_SYMBOL(napi_pp_get_page);
1074 #endif
1075 
1076 static bool skb_pp_recycle(struct sk_buff *skb, void *data)
1077 {
1078 	if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle)
1079 		return false;
1080 	return napi_pp_put_page(page_to_netmem(virt_to_page(data)));
1081 }
1082 
1083 static void skb_kfree_head(void *head)
1084 {
1085 	kfree(head);
1086 }
1087 
1088 static void skb_free_head(struct sk_buff *skb)
1089 {
1090 	unsigned char *head = skb->head;
1091 
1092 	if (skb->head_frag) {
1093 		if (skb_pp_recycle(skb, head))
1094 			return;
1095 		skb_free_frag(head);
1096 	} else {
1097 		skb_kfree_head(head);
1098 	}
1099 }
1100 
1101 static void skb_release_data(struct sk_buff *skb, enum skb_drop_reason reason)
1102 {
1103 	struct skb_shared_info *shinfo = skb_shinfo(skb);
1104 	int i;
1105 
1106 	if (!skb_data_unref(skb, shinfo))
1107 		return;
1108 
1109 	if (skb_zcopy(skb)) {
1110 		bool skip_unref = shinfo->flags & SKBFL_MANAGED_FRAG_REFS;
1111 
1112 		skb_zcopy_clear(skb, true);
1113 		if (skip_unref)
1114 			goto free_head;
1115 	}
1116 
1117 	for (i = 0; i < shinfo->nr_frags; i++)
1118 		skb_netmem_unref(skb_frag_netmem(&shinfo->frags[i]),
1119 				 skb->pp_recycle);
1120 
1121 free_head:
1122 	if (shinfo->frag_list)
1123 		kfree_skb_list_reason(shinfo->frag_list, reason);
1124 
1125 	skb_free_head(skb);
1126 }
1127 
1128 /*
1129  *	Free an skbuff by memory without cleaning the state.
1130  */
1131 static void kfree_skbmem(struct sk_buff *skb)
1132 {
1133 	struct sk_buff_fclones *fclones;
1134 
1135 	switch (skb->fclone) {
1136 	case SKB_FCLONE_UNAVAILABLE:
1137 		kmem_cache_free(net_hotdata.skbuff_cache, skb);
1138 		return;
1139 
1140 	case SKB_FCLONE_ORIG:
1141 		fclones = container_of(skb, struct sk_buff_fclones, skb1);
1142 
1143 		/* We usually free the clone (TX completion) before original skb
1144 		 * This test would have no chance to be true for the clone,
1145 		 * while here, branch prediction will be good.
1146 		 */
1147 		if (refcount_read(&fclones->fclone_ref) == 1)
1148 			goto fastpath;
1149 		break;
1150 
1151 	default: /* SKB_FCLONE_CLONE */
1152 		fclones = container_of(skb, struct sk_buff_fclones, skb2);
1153 		break;
1154 	}
1155 	if (!refcount_dec_and_test(&fclones->fclone_ref))
1156 		return;
1157 fastpath:
1158 	kmem_cache_free(net_hotdata.skbuff_fclone_cache, fclones);
1159 }
1160 
1161 void skb_release_head_state(struct sk_buff *skb)
1162 {
1163 	skb_dst_drop(skb);
1164 	if (skb->destructor) {
1165 		DEBUG_NET_WARN_ON_ONCE(in_hardirq());
1166 #ifdef CONFIG_INET
1167 		INDIRECT_CALL_4(skb->destructor,
1168 				tcp_wfree, __sock_wfree, sock_wfree,
1169 				xsk_destruct_skb,
1170 				skb);
1171 #else
1172 		INDIRECT_CALL_2(skb->destructor,
1173 				sock_wfree, xsk_destruct_skb,
1174 				skb);
1175 
1176 #endif
1177 		skb->destructor = NULL;
1178 		skb->sk = NULL;
1179 	}
1180 	nf_reset_ct(skb);
1181 	skb_ext_reset(skb);
1182 }
1183 
1184 /* Free everything but the sk_buff shell. */
1185 static void skb_release_all(struct sk_buff *skb, enum skb_drop_reason reason)
1186 {
1187 	skb_release_head_state(skb);
1188 	if (likely(skb->head))
1189 		skb_release_data(skb, reason);
1190 }
1191 
1192 /**
1193  *	__kfree_skb - private function
1194  *	@skb: buffer
1195  *
1196  *	Free an sk_buff. Release anything attached to the buffer.
1197  *	Clean the state. This is an internal helper function. Users should
1198  *	always call kfree_skb
1199  */
1200 
1201 void __kfree_skb(struct sk_buff *skb)
1202 {
1203 	skb_release_all(skb, SKB_DROP_REASON_NOT_SPECIFIED);
1204 	kfree_skbmem(skb);
1205 }
1206 EXPORT_SYMBOL(__kfree_skb);
1207 
1208 static __always_inline
1209 bool __sk_skb_reason_drop(const struct sock *sk, struct sk_buff *skb,
1210 			  enum skb_drop_reason reason)
1211 {
1212 	if (unlikely(!skb_unref(skb)))
1213 		return false;
1214 
1215 	DEBUG_NET_WARN_ON_ONCE(reason == SKB_NOT_DROPPED_YET ||
1216 			       u32_get_bits(reason,
1217 					    SKB_DROP_REASON_SUBSYS_MASK) >=
1218 				SKB_DROP_REASON_SUBSYS_NUM);
1219 
1220 	if (reason == SKB_CONSUMED)
1221 		trace_consume_skb(skb, __builtin_return_address(0));
1222 	else
1223 		trace_kfree_skb(skb, __builtin_return_address(0), reason, sk);
1224 	return true;
1225 }
1226 
1227 /**
1228  *	sk_skb_reason_drop - free an sk_buff with special reason
1229  *	@sk: the socket to receive @skb, or NULL if not applicable
1230  *	@skb: buffer to free
1231  *	@reason: reason why this skb is dropped
1232  *
1233  *	Drop a reference to the buffer and free it if the usage count has hit
1234  *	zero. Meanwhile, pass the receiving socket and drop reason to
1235  *	'kfree_skb' tracepoint.
1236  */
1237 void __fix_address
1238 sk_skb_reason_drop(const struct sock *sk, struct sk_buff *skb,
1239 		   enum skb_drop_reason reason)
1240 {
1241 	if (__sk_skb_reason_drop(sk, skb, reason))
1242 		__kfree_skb(skb);
1243 }
1244 EXPORT_SYMBOL(sk_skb_reason_drop);
1245 
1246 #define KFREE_SKB_BULK_SIZE	16
1247 
1248 struct skb_free_array {
1249 	unsigned int skb_count;
1250 	void *skb_array[KFREE_SKB_BULK_SIZE];
1251 };
1252 
1253 static void kfree_skb_add_bulk(struct sk_buff *skb,
1254 			       struct skb_free_array *sa,
1255 			       enum skb_drop_reason reason)
1256 {
1257 	/* if SKB is a clone, don't handle this case */
1258 	if (unlikely(skb->fclone != SKB_FCLONE_UNAVAILABLE)) {
1259 		__kfree_skb(skb);
1260 		return;
1261 	}
1262 
1263 	skb_release_all(skb, reason);
1264 	sa->skb_array[sa->skb_count++] = skb;
1265 
1266 	if (unlikely(sa->skb_count == KFREE_SKB_BULK_SIZE)) {
1267 		kmem_cache_free_bulk(net_hotdata.skbuff_cache, KFREE_SKB_BULK_SIZE,
1268 				     sa->skb_array);
1269 		sa->skb_count = 0;
1270 	}
1271 }
1272 
1273 void __fix_address
1274 kfree_skb_list_reason(struct sk_buff *segs, enum skb_drop_reason reason)
1275 {
1276 	struct skb_free_array sa;
1277 
1278 	sa.skb_count = 0;
1279 
1280 	while (segs) {
1281 		struct sk_buff *next = segs->next;
1282 
1283 		if (__sk_skb_reason_drop(NULL, segs, reason)) {
1284 			skb_poison_list(segs);
1285 			kfree_skb_add_bulk(segs, &sa, reason);
1286 		}
1287 
1288 		segs = next;
1289 	}
1290 
1291 	if (sa.skb_count)
1292 		kmem_cache_free_bulk(net_hotdata.skbuff_cache, sa.skb_count, sa.skb_array);
1293 }
1294 EXPORT_SYMBOL(kfree_skb_list_reason);
1295 
1296 /* Dump skb information and contents.
1297  *
1298  * Must only be called from net_ratelimit()-ed paths.
1299  *
1300  * Dumps whole packets if full_pkt, only headers otherwise.
1301  */
1302 void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt)
1303 {
1304 	struct skb_shared_info *sh = skb_shinfo(skb);
1305 	struct net_device *dev = skb->dev;
1306 	struct sock *sk = skb->sk;
1307 	struct sk_buff *list_skb;
1308 	bool has_mac, has_trans;
1309 	int headroom, tailroom;
1310 	int i, len, seg_len;
1311 
1312 	if (full_pkt)
1313 		len = skb->len;
1314 	else
1315 		len = min_t(int, skb->len, MAX_HEADER + 128);
1316 
1317 	headroom = skb_headroom(skb);
1318 	tailroom = skb_tailroom(skb);
1319 
1320 	has_mac = skb_mac_header_was_set(skb);
1321 	has_trans = skb_transport_header_was_set(skb);
1322 
1323 	printk("%sskb len=%u data_len=%u headroom=%u headlen=%u tailroom=%u\n"
1324 	       "end-tail=%u mac=(%d,%d) mac_len=%u net=(%d,%d) trans=%d\n"
1325 	       "shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n"
1326 	       "csum(0x%x start=%u offset=%u ip_summed=%u complete_sw=%u valid=%u level=%u)\n"
1327 	       "hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n"
1328 	       "priority=0x%x mark=0x%x alloc_cpu=%u vlan_all=0x%x\n"
1329 	       "encapsulation=%d inner(proto=0x%04x, mac=%u, net=%u, trans=%u)\n",
1330 	       level, skb->len, skb->data_len, headroom, skb_headlen(skb),
1331 	       tailroom, skb->end - skb->tail,
1332 	       has_mac ? skb->mac_header : -1,
1333 	       has_mac ? skb_mac_header_len(skb) : -1,
1334 	       skb->mac_len,
1335 	       skb->network_header,
1336 	       has_trans ? skb_network_header_len(skb) : -1,
1337 	       has_trans ? skb->transport_header : -1,
1338 	       sh->tx_flags, sh->nr_frags,
1339 	       sh->gso_size, sh->gso_type, sh->gso_segs,
1340 	       skb->csum, skb->csum_start, skb->csum_offset, skb->ip_summed,
1341 	       skb->csum_complete_sw, skb->csum_valid, skb->csum_level,
1342 	       skb->hash, skb->sw_hash, skb->l4_hash,
1343 	       ntohs(skb->protocol), skb->pkt_type, skb->skb_iif,
1344 	       skb->priority, skb->mark, skb->alloc_cpu, skb->vlan_all,
1345 	       skb->encapsulation, skb->inner_protocol, skb->inner_mac_header,
1346 	       skb->inner_network_header, skb->inner_transport_header);
1347 
1348 	if (dev)
1349 		printk("%sdev name=%s feat=%pNF\n",
1350 		       level, dev->name, &dev->features);
1351 	if (sk)
1352 		printk("%ssk family=%hu type=%u proto=%u\n",
1353 		       level, sk->sk_family, sk->sk_type, sk->sk_protocol);
1354 
1355 	if (full_pkt && headroom)
1356 		print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET,
1357 			       16, 1, skb->head, headroom, false);
1358 
1359 	seg_len = min_t(int, skb_headlen(skb), len);
1360 	if (seg_len)
1361 		print_hex_dump(level, "skb linear:   ", DUMP_PREFIX_OFFSET,
1362 			       16, 1, skb->data, seg_len, false);
1363 	len -= seg_len;
1364 
1365 	if (full_pkt && tailroom)
1366 		print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET,
1367 			       16, 1, skb_tail_pointer(skb), tailroom, false);
1368 
1369 	for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) {
1370 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1371 		u32 p_off, p_len, copied;
1372 		struct page *p;
1373 		u8 *vaddr;
1374 
1375 		if (skb_frag_is_net_iov(frag)) {
1376 			printk("%sskb frag %d: not readable\n", level, i);
1377 			len -= skb_frag_size(frag);
1378 			if (!len)
1379 				break;
1380 			continue;
1381 		}
1382 
1383 		skb_frag_foreach_page(frag, skb_frag_off(frag),
1384 				      skb_frag_size(frag), p, p_off, p_len,
1385 				      copied) {
1386 			seg_len = min_t(int, p_len, len);
1387 			vaddr = kmap_atomic(p);
1388 			print_hex_dump(level, "skb frag:     ",
1389 				       DUMP_PREFIX_OFFSET,
1390 				       16, 1, vaddr + p_off, seg_len, false);
1391 			kunmap_atomic(vaddr);
1392 			len -= seg_len;
1393 			if (!len)
1394 				break;
1395 		}
1396 	}
1397 
1398 	if (full_pkt && skb_has_frag_list(skb)) {
1399 		printk("skb fraglist:\n");
1400 		skb_walk_frags(skb, list_skb)
1401 			skb_dump(level, list_skb, true);
1402 	}
1403 }
1404 EXPORT_SYMBOL(skb_dump);
1405 
1406 /**
1407  *	skb_tx_error - report an sk_buff xmit error
1408  *	@skb: buffer that triggered an error
1409  *
1410  *	Report xmit error if a device callback is tracking this skb.
1411  *	skb must be freed afterwards.
1412  *
1413  *	Does nothing for a cloned skb: the zerocopy state lives in
1414  *	skb_shinfo(), which the clones share.
1415  */
1416 void skb_tx_error(struct sk_buff *skb)
1417 {
1418 	if (skb && !skb_cloned(skb)) {
1419 		skb_zcopy_downgrade_managed(skb);
1420 		skb_zcopy_clear(skb, true);
1421 	}
1422 }
1423 EXPORT_SYMBOL(skb_tx_error);
1424 
1425 #ifdef CONFIG_TRACEPOINTS
1426 /**
1427  *	consume_skb - free an skbuff
1428  *	@skb: buffer to free
1429  *
1430  *	Drop a ref to the buffer and free it if the usage count has hit zero
1431  *	Functions identically to kfree_skb, but kfree_skb assumes that the frame
1432  *	is being dropped after a failure and notes that
1433  */
1434 void consume_skb(struct sk_buff *skb)
1435 {
1436 	if (!skb_unref(skb))
1437 		return;
1438 
1439 	trace_consume_skb(skb, __builtin_return_address(0));
1440 	__kfree_skb(skb);
1441 }
1442 EXPORT_SYMBOL(consume_skb);
1443 #endif
1444 
1445 /**
1446  *	__consume_stateless_skb - free an skbuff, assuming it is stateless
1447  *	@skb: buffer to free
1448  *
1449  *	Alike consume_skb(), but this variant assumes that this is the last
1450  *	skb reference and all the head states have been already dropped
1451  */
1452 void __consume_stateless_skb(struct sk_buff *skb)
1453 {
1454 	trace_consume_skb(skb, __builtin_return_address(0));
1455 	skb_release_data(skb, SKB_CONSUMED);
1456 	kfree_skbmem(skb);
1457 }
1458 
1459 static void napi_skb_cache_put(struct sk_buff *skb)
1460 {
1461 	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
1462 
1463 	if (!kasan_mempool_poison_object(skb))
1464 		return;
1465 
1466 	local_lock_nested_bh(&napi_alloc_cache.bh_lock);
1467 	nc->skb_cache[nc->skb_count++] = skb;
1468 
1469 	if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) {
1470 		u32 i, remaining = NAPI_SKB_CACHE_SIZE - NAPI_SKB_CACHE_FREE;
1471 
1472 		for (i = remaining; i < NAPI_SKB_CACHE_SIZE; i++)
1473 			kasan_mempool_unpoison_object(nc->skb_cache[i],
1474 						skbuff_cache_size);
1475 
1476 		kmem_cache_free_bulk(net_hotdata.skbuff_cache,
1477 				     NAPI_SKB_CACHE_FREE,
1478 				     nc->skb_cache + remaining);
1479 		nc->skb_count = remaining;
1480 	}
1481 	local_unlock_nested_bh(&napi_alloc_cache.bh_lock);
1482 }
1483 
1484 void __napi_kfree_skb(struct sk_buff *skb, enum skb_drop_reason reason)
1485 {
1486 	skb_release_all(skb, reason);
1487 	napi_skb_cache_put(skb);
1488 }
1489 
1490 void napi_skb_free_stolen_head(struct sk_buff *skb)
1491 {
1492 	if (unlikely(skb->slow_gro)) {
1493 		nf_reset_ct(skb);
1494 		skb_dst_drop(skb);
1495 		skb_ext_put(skb);
1496 		skb_orphan(skb);
1497 		skb->slow_gro = 0;
1498 	}
1499 	if (skb->fclone != SKB_FCLONE_UNAVAILABLE) {
1500 		kfree_skbmem(skb);
1501 		return;
1502 	}
1503 	napi_skb_cache_put(skb);
1504 }
1505 
1506 /**
1507  * napi_consume_skb() - consume skb in NAPI context, try to feed skb cache
1508  * @skb: buffer to free
1509  * @budget: NAPI budget
1510  *
1511  * Non-zero @budget must come from the @budget argument passed by the core
1512  * to a NAPI poll function. Note that core may pass budget of 0 to NAPI poll
1513  * for example when polling for netpoll / netconsole.
1514  *
1515  * Passing @budget of 0 is safe from any context, it turns this function
1516  * into dev_consume_skb_any().
1517  */
1518 void napi_consume_skb(struct sk_buff *skb, int budget)
1519 {
1520 	if (unlikely(!budget || !skb)) {
1521 		dev_consume_skb_any(skb);
1522 		return;
1523 	}
1524 
1525 	DEBUG_NET_WARN_ON_ONCE(!in_softirq());
1526 
1527 	if (!static_branch_unlikely(&skb_defer_disable_key) &&
1528 	    skb->alloc_cpu != smp_processor_id() && !skb_shared(skb)) {
1529 		skb_release_head_state(skb);
1530 		return skb_attempt_defer_free(skb);
1531 	}
1532 
1533 	if (!skb_unref(skb))
1534 		return;
1535 
1536 	/* if reaching here SKB is ready to free */
1537 	trace_consume_skb(skb, __builtin_return_address(0));
1538 
1539 	/* if SKB is a clone, don't handle this case */
1540 	if (skb->fclone != SKB_FCLONE_UNAVAILABLE) {
1541 		__kfree_skb(skb);
1542 		return;
1543 	}
1544 
1545 	skb_release_all(skb, SKB_CONSUMED);
1546 	napi_skb_cache_put(skb);
1547 }
1548 EXPORT_SYMBOL(napi_consume_skb);
1549 
1550 /* Make sure a field is contained by headers group */
1551 #define CHECK_SKB_FIELD(field) \
1552 	BUILD_BUG_ON(offsetof(struct sk_buff, field) !=		\
1553 		     offsetof(struct sk_buff, headers.field));	\
1554 
1555 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
1556 {
1557 	new->tstamp		= old->tstamp;
1558 	/* We do not copy old->sk */
1559 	new->dev		= old->dev;
1560 	memcpy(new->cb, old->cb, sizeof(old->cb));
1561 	skb_dst_copy(new, old);
1562 	__skb_ext_copy(new, old);
1563 	__nf_copy(new, old, false);
1564 
1565 	/* Note : this field could be in the headers group.
1566 	 * It is not yet because we do not want to have a 16 bit hole
1567 	 */
1568 	new->queue_mapping = old->queue_mapping;
1569 
1570 	memcpy(&new->headers, &old->headers, sizeof(new->headers));
1571 	CHECK_SKB_FIELD(protocol);
1572 	CHECK_SKB_FIELD(csum);
1573 	CHECK_SKB_FIELD(hash);
1574 	CHECK_SKB_FIELD(priority);
1575 	CHECK_SKB_FIELD(skb_iif);
1576 	CHECK_SKB_FIELD(vlan_proto);
1577 	CHECK_SKB_FIELD(vlan_tci);
1578 	CHECK_SKB_FIELD(transport_header);
1579 	CHECK_SKB_FIELD(network_header);
1580 	CHECK_SKB_FIELD(mac_header);
1581 	CHECK_SKB_FIELD(inner_protocol);
1582 	CHECK_SKB_FIELD(inner_transport_header);
1583 	CHECK_SKB_FIELD(inner_network_header);
1584 	CHECK_SKB_FIELD(inner_mac_header);
1585 	CHECK_SKB_FIELD(mark);
1586 #ifdef CONFIG_NETWORK_SECMARK
1587 	CHECK_SKB_FIELD(secmark);
1588 #endif
1589 #ifdef CONFIG_NET_RX_BUSY_POLL
1590 	CHECK_SKB_FIELD(napi_id);
1591 #endif
1592 #ifdef CONFIG_XPS
1593 	CHECK_SKB_FIELD(sender_cpu);
1594 #endif
1595 #ifdef CONFIG_NET_SCHED
1596 	CHECK_SKB_FIELD(tc_index);
1597 #endif
1598 
1599 }
1600 
1601 /*
1602  * You should not add any new code to this function.  Add it to
1603  * __copy_skb_header above instead.
1604  */
1605 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
1606 {
1607 #define C(x) n->x = skb->x
1608 
1609 	n->next = n->prev = NULL;
1610 	n->sk = NULL;
1611 	__copy_skb_header(n, skb);
1612 
1613 	C(len);
1614 	C(data_len);
1615 	C(mac_len);
1616 	n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
1617 	n->cloned = 1;
1618 	n->nohdr = 0;
1619 	n->peeked = 0;
1620 	C(pfmemalloc);
1621 	C(pp_recycle);
1622 	n->destructor = NULL;
1623 	C(alloc_cpu);
1624 	C(tail);
1625 	C(end);
1626 	C(head);
1627 	C(head_frag);
1628 	C(data);
1629 	C(truesize);
1630 	refcount_set(&n->users, 1);
1631 
1632 	atomic_inc(&(skb_shinfo(skb)->dataref));
1633 	skb->cloned = 1;
1634 
1635 	return n;
1636 #undef C
1637 }
1638 
1639 /**
1640  * alloc_skb_for_msg() - allocate sk_buff to wrap frag list forming a msg
1641  * @first: first sk_buff of the msg
1642  */
1643 struct sk_buff *alloc_skb_for_msg(struct sk_buff *first)
1644 {
1645 	struct sk_buff *n;
1646 
1647 	n = alloc_skb(0, GFP_ATOMIC);
1648 	if (!n)
1649 		return NULL;
1650 
1651 	n->len = first->len;
1652 	n->data_len = first->len;
1653 	n->truesize = first->truesize;
1654 
1655 	skb_shinfo(n)->frag_list = first;
1656 
1657 	__copy_skb_header(n, first);
1658 	n->destructor = NULL;
1659 
1660 	return n;
1661 }
1662 EXPORT_SYMBOL_GPL(alloc_skb_for_msg);
1663 
1664 /**
1665  *	skb_morph	-	morph one skb into another
1666  *	@dst: the skb to receive the contents
1667  *	@src: the skb to supply the contents
1668  *
1669  *	This is identical to skb_clone except that the target skb is
1670  *	supplied by the user.
1671  *
1672  *	The target skb is returned upon exit.
1673  */
1674 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
1675 {
1676 	skb_release_all(dst, SKB_CONSUMED);
1677 	return __skb_clone(dst, src);
1678 }
1679 EXPORT_SYMBOL_GPL(skb_morph);
1680 
1681 int mm_account_pinned_pages(struct mmpin *mmp, size_t size)
1682 {
1683 	unsigned long max_pg, num_pg, new_pg, old_pg, rlim;
1684 	struct user_struct *user;
1685 
1686 	if (capable(CAP_IPC_LOCK) || !size)
1687 		return 0;
1688 
1689 	rlim = rlimit(RLIMIT_MEMLOCK);
1690 	if (rlim == RLIM_INFINITY)
1691 		return 0;
1692 
1693 	num_pg = (size >> PAGE_SHIFT) + 2;	/* worst case */
1694 	max_pg = rlim >> PAGE_SHIFT;
1695 	user = mmp->user ? : current_user();
1696 
1697 	old_pg = atomic_long_read(&user->locked_vm);
1698 	do {
1699 		new_pg = old_pg + num_pg;
1700 		if (new_pg > max_pg)
1701 			return -ENOBUFS;
1702 	} while (!atomic_long_try_cmpxchg(&user->locked_vm, &old_pg, new_pg));
1703 
1704 	if (!mmp->user) {
1705 		mmp->user = get_uid(user);
1706 		mmp->num_pg = num_pg;
1707 	} else {
1708 		mmp->num_pg += num_pg;
1709 	}
1710 
1711 	return 0;
1712 }
1713 EXPORT_SYMBOL_GPL(mm_account_pinned_pages);
1714 
1715 void mm_unaccount_pinned_pages(struct mmpin *mmp)
1716 {
1717 	if (mmp->user) {
1718 		atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm);
1719 		free_uid(mmp->user);
1720 	}
1721 }
1722 EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages);
1723 
1724 static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size,
1725 					    bool devmem)
1726 {
1727 	struct ubuf_info_msgzc *uarg;
1728 	struct sk_buff *skb;
1729 
1730 	WARN_ON_ONCE(!in_task());
1731 
1732 	skb = sock_omalloc(sk, 0, GFP_KERNEL);
1733 	if (!skb)
1734 		return NULL;
1735 
1736 	BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb));
1737 	uarg = (void *)skb->cb;
1738 	uarg->mmp.user = NULL;
1739 
1740 	if (likely(!devmem) && mm_account_pinned_pages(&uarg->mmp, size)) {
1741 		kfree_skb(skb);
1742 		return NULL;
1743 	}
1744 
1745 	uarg->ubuf.ops = &msg_zerocopy_ubuf_ops;
1746 	uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
1747 	uarg->len = 1;
1748 	uarg->bytelen = size;
1749 	uarg->zerocopy = 1;
1750 	uarg->ubuf.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN;
1751 	refcount_set(&uarg->ubuf.refcnt, 1);
1752 	sock_hold(sk);
1753 
1754 	return &uarg->ubuf;
1755 }
1756 
1757 static inline struct sk_buff *skb_from_uarg(struct ubuf_info_msgzc *uarg)
1758 {
1759 	return container_of((void *)uarg, struct sk_buff, cb);
1760 }
1761 
1762 struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
1763 				       struct ubuf_info *uarg, bool devmem)
1764 {
1765 	if (uarg) {
1766 		struct ubuf_info_msgzc *uarg_zc;
1767 		const u32 byte_limit = 1 << 19;		/* limit to a few TSO */
1768 		u32 bytelen, next;
1769 
1770 		/* there might be non MSG_ZEROCOPY users */
1771 		if (uarg->ops != &msg_zerocopy_ubuf_ops)
1772 			return NULL;
1773 
1774 		/* realloc only when socket is locked (TCP, UDP cork),
1775 		 * so uarg->len and sk_zckey access is serialized
1776 		 */
1777 		if (!sock_owned_by_user(sk)) {
1778 			WARN_ON_ONCE(1);
1779 			return NULL;
1780 		}
1781 
1782 		uarg_zc = uarg_to_msgzc(uarg);
1783 		bytelen = uarg_zc->bytelen + size;
1784 		if (uarg_zc->len == USHRT_MAX - 1 || bytelen > byte_limit) {
1785 			/* TCP can create new skb to attach new uarg */
1786 			if (sk->sk_type == SOCK_STREAM)
1787 				goto new_alloc;
1788 			return NULL;
1789 		}
1790 
1791 		next = (u32)atomic_read(&sk->sk_zckey);
1792 		if ((u32)(uarg_zc->id + uarg_zc->len) == next) {
1793 			if (likely(!devmem) &&
1794 			    mm_account_pinned_pages(&uarg_zc->mmp, size))
1795 				return NULL;
1796 			uarg_zc->len++;
1797 			uarg_zc->bytelen = bytelen;
1798 			atomic_set(&sk->sk_zckey, ++next);
1799 
1800 			/* no extra ref when appending to datagram (MSG_MORE) */
1801 			if (sk->sk_type == SOCK_STREAM)
1802 				net_zcopy_get(uarg);
1803 
1804 			return uarg;
1805 		}
1806 	}
1807 
1808 new_alloc:
1809 	return msg_zerocopy_alloc(sk, size, devmem);
1810 }
1811 EXPORT_SYMBOL_GPL(msg_zerocopy_realloc);
1812 
1813 static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len)
1814 {
1815 	struct sock_exterr_skb *serr = SKB_EXT_ERR(skb);
1816 	u32 old_lo, old_hi;
1817 	u64 sum_len;
1818 
1819 	old_lo = serr->ee.ee_info;
1820 	old_hi = serr->ee.ee_data;
1821 	sum_len = old_hi - old_lo + 1ULL + len;
1822 
1823 	if (sum_len >= (1ULL << 32))
1824 		return false;
1825 
1826 	if (lo != old_hi + 1)
1827 		return false;
1828 
1829 	serr->ee.ee_data += len;
1830 	return true;
1831 }
1832 
1833 static void __msg_zerocopy_callback(struct ubuf_info_msgzc *uarg)
1834 {
1835 	struct sk_buff *tail, *skb = skb_from_uarg(uarg);
1836 	struct sock_exterr_skb *serr;
1837 	struct sock *sk = skb->sk;
1838 	struct sk_buff_head *q;
1839 	unsigned long flags;
1840 	bool is_zerocopy;
1841 	u32 lo, hi;
1842 	u16 len;
1843 
1844 	mm_unaccount_pinned_pages(&uarg->mmp);
1845 
1846 	/* if !len, there was only 1 call, and it was aborted
1847 	 * so do not queue a completion notification
1848 	 */
1849 	if (!uarg->len || sock_flag(sk, SOCK_DEAD))
1850 		goto release;
1851 
1852 	len = uarg->len;
1853 	lo = uarg->id;
1854 	hi = uarg->id + len - 1;
1855 	is_zerocopy = uarg->zerocopy;
1856 
1857 	serr = SKB_EXT_ERR(skb);
1858 	memset(serr, 0, sizeof(*serr));
1859 	serr->ee.ee_errno = 0;
1860 	serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY;
1861 	serr->ee.ee_data = hi;
1862 	serr->ee.ee_info = lo;
1863 	if (!is_zerocopy)
1864 		serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED;
1865 
1866 	q = &sk->sk_error_queue;
1867 	spin_lock_irqsave(&q->lock, flags);
1868 	tail = skb_peek_tail(q);
1869 	if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY ||
1870 	    !skb_zerocopy_notify_extend(tail, lo, len)) {
1871 		__skb_queue_tail(q, skb);
1872 		skb = NULL;
1873 	}
1874 	spin_unlock_irqrestore(&q->lock, flags);
1875 
1876 	sk_error_report(sk);
1877 
1878 release:
1879 	consume_skb(skb);
1880 	sock_put(sk);
1881 }
1882 
1883 static void msg_zerocopy_complete(struct sk_buff *skb, struct ubuf_info *uarg,
1884 				  bool success)
1885 {
1886 	struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg);
1887 
1888 	uarg_zc->zerocopy = uarg_zc->zerocopy & success;
1889 
1890 	if (refcount_dec_and_test(&uarg->refcnt))
1891 		__msg_zerocopy_callback(uarg_zc);
1892 }
1893 
1894 void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
1895 {
1896 	struct sock *sk = skb_from_uarg(uarg_to_msgzc(uarg))->sk;
1897 
1898 	atomic_dec(&sk->sk_zckey);
1899 	uarg_to_msgzc(uarg)->len--;
1900 
1901 	if (have_uref)
1902 		msg_zerocopy_complete(NULL, uarg, true);
1903 }
1904 EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort);
1905 
1906 const struct ubuf_info_ops msg_zerocopy_ubuf_ops = {
1907 	.complete = msg_zerocopy_complete,
1908 };
1909 EXPORT_SYMBOL_GPL(msg_zerocopy_ubuf_ops);
1910 
1911 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
1912 			     struct msghdr *msg, int len,
1913 			     struct ubuf_info *uarg,
1914 			     struct net_devmem_dmabuf_binding *binding)
1915 {
1916 	unsigned int orig_truesize = skb->truesize;
1917 	int err, orig_len = skb->len;
1918 
1919 	if (uarg->ops->link_skb) {
1920 		err = uarg->ops->link_skb(skb, uarg);
1921 		if (err)
1922 			return err;
1923 	} else {
1924 		struct ubuf_info *orig_uarg = skb_zcopy(skb);
1925 
1926 		/* An skb can only point to one uarg. This edge case happens
1927 		 * when TCP appends to an skb, but zerocopy_realloc triggered
1928 		 * a new alloc.
1929 		 */
1930 		if (orig_uarg && uarg != orig_uarg)
1931 			return -EEXIST;
1932 	}
1933 
1934 	err = __zerocopy_sg_from_iter(msg, sk, skb, &msg->msg_iter, len,
1935 				      binding);
1936 	if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
1937 		u32 delta_truesize = skb->truesize - orig_truesize;
1938 		struct sock *save_sk = skb->sk;
1939 
1940 		/* Streams do not free skb on error. Reset to prev state. */
1941 		iov_iter_revert(&msg->msg_iter, skb->len - orig_len);
1942 		skb->sk = sk;
1943 		if (!___pskb_trim(skb, orig_len)) {
1944 			skb->truesize -= delta_truesize;
1945 			sk_wmem_queued_add(sk, -delta_truesize);
1946 			if (!skb_zcopy_pure(skb))
1947 				sk_mem_uncharge(sk, delta_truesize);
1948 		}
1949 		skb->sk = save_sk;
1950 		return err;
1951 	}
1952 
1953 	skb_zcopy_set(skb, uarg, NULL);
1954 	return skb->len - orig_len;
1955 }
1956 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);
1957 
1958 void __skb_zcopy_downgrade_managed(struct sk_buff *skb)
1959 {
1960 	int i;
1961 
1962 	skb_shinfo(skb)->flags &= ~SKBFL_MANAGED_FRAG_REFS;
1963 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1964 		skb_frag_ref(skb, i);
1965 }
1966 EXPORT_SYMBOL_GPL(__skb_zcopy_downgrade_managed);
1967 
1968 static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig,
1969 			      gfp_t gfp_mask)
1970 {
1971 	if (skb_zcopy(orig)) {
1972 		if (skb_zcopy(nskb)) {
1973 			/* !gfp_mask callers are verified to !skb_zcopy(nskb) */
1974 			if (!gfp_mask) {
1975 				WARN_ON_ONCE(1);
1976 				return -ENOMEM;
1977 			}
1978 			if (skb_uarg(nskb) == skb_uarg(orig))
1979 				return 0;
1980 			if (skb_copy_ubufs(nskb, GFP_ATOMIC))
1981 				return -EIO;
1982 		}
1983 		skb_zcopy_set(nskb, skb_uarg(orig), NULL);
1984 	}
1985 	return 0;
1986 }
1987 
1988 /**
1989  *	skb_copy_ubufs	-	copy userspace skb frags buffers to kernel
1990  *	@skb: the skb to modify
1991  *	@gfp_mask: allocation priority
1992  *
1993  *	This must be called on skb with SKBFL_ZEROCOPY_ENABLE.
1994  *	It will copy all frags into kernel and drop the reference
1995  *	to userspace pages.
1996  *
1997  *	If this function is called from an interrupt gfp_mask() must be
1998  *	%GFP_ATOMIC.
1999  *
2000  *	Returns 0 on success or a negative error code on failure
2001  *	to allocate kernel memory to copy to.
2002  */
2003 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
2004 {
2005 	int num_frags = skb_shinfo(skb)->nr_frags;
2006 	struct page *page, *head = NULL;
2007 	int i, order, psize, new_frags;
2008 	u32 d_off;
2009 
2010 	if (!skb_frags_readable(skb))
2011 		return -EFAULT;
2012 
2013 	if (skb_shared(skb) || skb_unclone(skb, gfp_mask))
2014 		return -EINVAL;
2015 
2016 	if (!num_frags)
2017 		goto release;
2018 
2019 	/* We might have to allocate high order pages, so compute what minimum
2020 	 * page order is needed.
2021 	 */
2022 	order = 0;
2023 	while ((PAGE_SIZE << order) * MAX_SKB_FRAGS < __skb_pagelen(skb))
2024 		order++;
2025 	psize = (PAGE_SIZE << order);
2026 
2027 	new_frags = (__skb_pagelen(skb) + psize - 1) >> (PAGE_SHIFT + order);
2028 	for (i = 0; i < new_frags; i++) {
2029 		page = alloc_pages(gfp_mask | __GFP_COMP, order);
2030 		if (!page) {
2031 			while (head) {
2032 				struct page *next = (struct page *)page_private(head);
2033 				put_page(head);
2034 				head = next;
2035 			}
2036 			return -ENOMEM;
2037 		}
2038 		set_page_private(page, (unsigned long)head);
2039 		head = page;
2040 	}
2041 
2042 	page = head;
2043 	d_off = 0;
2044 	for (i = 0; i < num_frags; i++) {
2045 		skb_frag_t *f = &skb_shinfo(skb)->frags[i];
2046 		u32 p_off, p_len, copied;
2047 		struct page *p;
2048 		u8 *vaddr;
2049 
2050 		skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f),
2051 				      p, p_off, p_len, copied) {
2052 			u32 copy, done = 0;
2053 			vaddr = kmap_atomic(p);
2054 
2055 			while (done < p_len) {
2056 				if (d_off == psize) {
2057 					d_off = 0;
2058 					page = (struct page *)page_private(page);
2059 				}
2060 				copy = min_t(u32, psize - d_off, p_len - done);
2061 				memcpy(page_address(page) + d_off,
2062 				       vaddr + p_off + done, copy);
2063 				done += copy;
2064 				d_off += copy;
2065 			}
2066 			kunmap_atomic(vaddr);
2067 		}
2068 	}
2069 
2070 	/* skb frags release userspace buffers */
2071 	for (i = 0; i < num_frags; i++)
2072 		skb_frag_unref(skb, i);
2073 
2074 	/* skb frags point to kernel buffers */
2075 	for (i = 0; i < new_frags - 1; i++) {
2076 		__skb_fill_netmem_desc(skb, i, page_to_netmem(head), 0, psize);
2077 		head = (struct page *)page_private(head);
2078 	}
2079 	__skb_fill_netmem_desc(skb, new_frags - 1, page_to_netmem(head), 0,
2080 			       d_off);
2081 	skb_shinfo(skb)->nr_frags = new_frags;
2082 
2083 release:
2084 	skb_zcopy_clear(skb, false);
2085 	return 0;
2086 }
2087 EXPORT_SYMBOL_GPL(skb_copy_ubufs);
2088 
2089 /**
2090  *	skb_clone	-	duplicate an sk_buff
2091  *	@skb: buffer to clone
2092  *	@gfp_mask: allocation priority
2093  *
2094  *	Duplicate an &sk_buff. The new one is not owned by a socket. Both
2095  *	copies share the same packet data but not structure. The new
2096  *	buffer has a reference count of 1. If the allocation fails the
2097  *	function returns %NULL otherwise the new buffer is returned.
2098  *
2099  *	If this function is called from an interrupt gfp_mask() must be
2100  *	%GFP_ATOMIC.
2101  */
2102 
2103 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
2104 {
2105 	struct sk_buff_fclones *fclones = container_of(skb,
2106 						       struct sk_buff_fclones,
2107 						       skb1);
2108 	struct sk_buff *n;
2109 
2110 	if (skb_orphan_frags(skb, gfp_mask))
2111 		return NULL;
2112 
2113 	if (skb->fclone == SKB_FCLONE_ORIG &&
2114 	    refcount_read(&fclones->fclone_ref) == 1) {
2115 		n = &fclones->skb2;
2116 		refcount_set(&fclones->fclone_ref, 2);
2117 		n->fclone = SKB_FCLONE_CLONE;
2118 	} else {
2119 		if (skb_pfmemalloc(skb))
2120 			gfp_mask |= __GFP_MEMALLOC;
2121 
2122 		n = kmem_cache_alloc(net_hotdata.skbuff_cache, gfp_mask);
2123 		if (!n)
2124 			return NULL;
2125 
2126 		n->fclone = SKB_FCLONE_UNAVAILABLE;
2127 	}
2128 
2129 	return __skb_clone(n, skb);
2130 }
2131 EXPORT_SYMBOL(skb_clone);
2132 
2133 void skb_headers_offset_update(struct sk_buff *skb, int off)
2134 {
2135 	/* Only adjust this if it actually is csum_start rather than csum */
2136 	if (skb->ip_summed == CHECKSUM_PARTIAL)
2137 		skb->csum_start += off;
2138 	/* {transport,network,mac}_header and tail are relative to skb->head */
2139 	skb->transport_header += off;
2140 	skb->network_header   += off;
2141 	if (skb_mac_header_was_set(skb))
2142 		skb->mac_header += off;
2143 	skb->inner_transport_header += off;
2144 	skb->inner_network_header += off;
2145 	skb->inner_mac_header += off;
2146 }
2147 EXPORT_SYMBOL(skb_headers_offset_update);
2148 
2149 void skb_copy_header(struct sk_buff *new, const struct sk_buff *old)
2150 {
2151 	__copy_skb_header(new, old);
2152 
2153 	skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
2154 	skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
2155 	skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
2156 }
2157 EXPORT_SYMBOL(skb_copy_header);
2158 
2159 static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
2160 {
2161 	if (skb_pfmemalloc(skb))
2162 		return SKB_ALLOC_RX;
2163 	return 0;
2164 }
2165 
2166 /**
2167  *	skb_copy	-	create private copy of an sk_buff
2168  *	@skb: buffer to copy
2169  *	@gfp_mask: allocation priority
2170  *
2171  *	Make a copy of both an &sk_buff and its data. This is used when the
2172  *	caller wishes to modify the data and needs a private copy of the
2173  *	data to alter. Returns %NULL on failure or the pointer to the buffer
2174  *	on success. The returned buffer has a reference count of 1.
2175  *
2176  *	As by-product this function converts non-linear &sk_buff to linear
2177  *	one, so that &sk_buff becomes completely private and caller is allowed
2178  *	to modify all the data of returned buffer. This means that this
2179  *	function is not recommended for use in circumstances when only
2180  *	header is going to be modified. Use pskb_copy() instead.
2181  */
2182 
2183 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
2184 {
2185 	struct sk_buff *n;
2186 	unsigned int size;
2187 	int headerlen;
2188 
2189 	if (!skb_frags_readable(skb))
2190 		return NULL;
2191 
2192 	if (WARN_ON_ONCE(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST))
2193 		return NULL;
2194 
2195 	headerlen = skb_headroom(skb);
2196 	size = skb_end_offset(skb) + skb->data_len;
2197 	n = __alloc_skb(size, gfp_mask,
2198 			skb_alloc_rx_flag(skb), NUMA_NO_NODE);
2199 	if (!n)
2200 		return NULL;
2201 
2202 	/* Set the data pointer */
2203 	skb_reserve(n, headerlen);
2204 	/* Set the tail pointer and length */
2205 	skb_put(n, skb->len);
2206 
2207 	if (unlikely(skb_copy_bits(skb, -headerlen, n->head,
2208 				   headerlen + skb->len))) {
2209 		DEBUG_NET_WARN_ON_ONCE(1);
2210 		kfree_skb(n);
2211 		return NULL;
2212 	}
2213 
2214 	skb_copy_header(n, skb);
2215 	return n;
2216 }
2217 EXPORT_SYMBOL(skb_copy);
2218 
2219 /**
2220  *	__pskb_copy_fclone	-  create copy of an sk_buff with private head.
2221  *	@skb: buffer to copy
2222  *	@headroom: headroom of new skb
2223  *	@gfp_mask: allocation priority
2224  *	@fclone: if true allocate the copy of the skb from the fclone
2225  *	cache instead of the head cache; it is recommended to set this
2226  *	to true for the cases where the copy will likely be cloned
2227  *
2228  *	Make a copy of both an &sk_buff and part of its data, located
2229  *	in header. Fragmented data remain shared. This is used when
2230  *	the caller wishes to modify only header of &sk_buff and needs
2231  *	private copy of the header to alter. Returns %NULL on failure
2232  *	or the pointer to the buffer on success.
2233  *	The returned buffer has a reference count of 1.
2234  */
2235 
2236 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
2237 				   gfp_t gfp_mask, bool fclone)
2238 {
2239 	unsigned int size = skb_headlen(skb) + headroom;
2240 	int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0);
2241 	struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE);
2242 
2243 	if (!n)
2244 		goto out;
2245 
2246 	/* Set the data pointer */
2247 	skb_reserve(n, headroom);
2248 	/* Set the tail pointer and length */
2249 	skb_put(n, skb_headlen(skb));
2250 	/* Copy the bytes */
2251 	skb_copy_from_linear_data(skb, n->data, n->len);
2252 
2253 	n->truesize += skb->data_len;
2254 	n->data_len  = skb->data_len;
2255 	n->len	     = skb->len;
2256 
2257 	if (skb_shinfo(skb)->nr_frags) {
2258 		int i;
2259 
2260 		if (skb_orphan_frags(skb, gfp_mask) ||
2261 		    skb_zerocopy_clone(n, skb, gfp_mask)) {
2262 			kfree_skb(n);
2263 			n = NULL;
2264 			goto out;
2265 		}
2266 		n->pp_recycle = skb->pp_recycle;
2267 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2268 			skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
2269 			skb_frag_ref(n, i);
2270 		}
2271 		skb_shinfo(n)->nr_frags = i;
2272 		skb_shinfo(n)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG;
2273 	}
2274 
2275 	if (skb_has_frag_list(skb)) {
2276 		skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
2277 		skb_clone_fraglist(n);
2278 	}
2279 
2280 	skb_copy_header(n, skb);
2281 out:
2282 	return n;
2283 }
2284 EXPORT_SYMBOL(__pskb_copy_fclone);
2285 
2286 /**
2287  *	pskb_expand_head - reallocate header of &sk_buff
2288  *	@skb: buffer to reallocate
2289  *	@nhead: room to add at head
2290  *	@ntail: room to add at tail
2291  *	@gfp_mask: allocation priority
2292  *
2293  *	Expands (or creates identical copy, if @nhead and @ntail are zero)
2294  *	header of @skb. &sk_buff itself is not changed. &sk_buff MUST have
2295  *	reference count of 1. Returns zero in the case of success or error,
2296  *	if expansion failed. In the last case, &sk_buff is not changed.
2297  *
2298  *	All the pointers pointing into skb header may change and must be
2299  *	reloaded after call to this function.
2300  *
2301  *	Note: If you skb_push() the start of the buffer after reallocating the
2302  *	header, call skb_postpush_data_move() first to move the metadata out of
2303  *	the way before writing to &sk_buff->data.
2304  */
2305 
2306 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
2307 		     gfp_t gfp_mask)
2308 {
2309 	unsigned int osize = skb_end_offset(skb);
2310 	unsigned int size = osize + nhead + ntail;
2311 	long off;
2312 	u8 *data;
2313 	int i;
2314 
2315 	if (unlikely(nhead < 0)) {
2316 		DEBUG_NET_WARN_ON_ONCE(1);
2317 		return -EINVAL;
2318 	}
2319 
2320 	if (unlikely(skb_shared(skb))) {
2321 		DEBUG_NET_WARN_ON_ONCE(1);
2322 		return -EINVAL;
2323 	}
2324 
2325 	skb_zcopy_downgrade_managed(skb);
2326 
2327 	if (skb_pfmemalloc(skb))
2328 		gfp_mask |= __GFP_MEMALLOC;
2329 
2330 	data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL);
2331 	if (!data)
2332 		goto nodata;
2333 	size = SKB_WITH_OVERHEAD(size);
2334 
2335 	/* Copy only real data... and, alas, header. This should be
2336 	 * optimized for the cases when header is void.
2337 	 */
2338 	memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
2339 
2340 	memcpy((struct skb_shared_info *)(data + size),
2341 	       skb_shinfo(skb),
2342 	       offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
2343 
2344 	/*
2345 	 * if shinfo is shared we must drop the old head gracefully, but if it
2346 	 * is not we can just drop the old head and let the existing refcount
2347 	 * be since all we did is relocate the values
2348 	 */
2349 	if (skb_cloned(skb)) {
2350 		if (skb_orphan_frags(skb, gfp_mask))
2351 			goto nofrags;
2352 		if (skb_zcopy(skb))
2353 			net_zcopy_get(skb_uarg(skb));
2354 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
2355 			skb_frag_ref(skb, i);
2356 
2357 		if (skb_has_frag_list(skb))
2358 			skb_clone_fraglist(skb);
2359 
2360 		skb_release_data(skb, SKB_CONSUMED);
2361 	} else {
2362 		skb_free_head(skb);
2363 	}
2364 	off = (data + nhead) - skb->head;
2365 
2366 	skb->head     = data;
2367 	skb->head_frag = 0;
2368 	skb->data    += off;
2369 
2370 	skb_set_end_offset(skb, size);
2371 #ifdef NET_SKBUFF_DATA_USES_OFFSET
2372 	off           = nhead;
2373 #endif
2374 	skb->tail	      += off;
2375 	skb_headers_offset_update(skb, nhead);
2376 	skb->cloned   = 0;
2377 	skb->hdr_len  = 0;
2378 	skb->nohdr    = 0;
2379 	atomic_set(&skb_shinfo(skb)->dataref, 1);
2380 
2381 	/* It is not generally safe to change skb->truesize.
2382 	 * For the moment, we really care of rx path, or
2383 	 * when skb is orphaned (not attached to a socket).
2384 	 */
2385 	if (!skb->sk || skb->destructor == sock_edemux)
2386 		skb->truesize += size - osize;
2387 
2388 	return 0;
2389 
2390 nofrags:
2391 	skb_kfree_head(data);
2392 nodata:
2393 	return -ENOMEM;
2394 }
2395 EXPORT_SYMBOL(pskb_expand_head);
2396 
2397 /* Make private copy of skb with writable head and some headroom */
2398 
2399 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
2400 {
2401 	struct sk_buff *skb2;
2402 	int delta = headroom - skb_headroom(skb);
2403 
2404 	if (delta <= 0)
2405 		skb2 = pskb_copy(skb, GFP_ATOMIC);
2406 	else {
2407 		skb2 = skb_clone(skb, GFP_ATOMIC);
2408 		if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
2409 					     GFP_ATOMIC)) {
2410 			kfree_skb(skb2);
2411 			skb2 = NULL;
2412 		}
2413 	}
2414 	return skb2;
2415 }
2416 EXPORT_SYMBOL(skb_realloc_headroom);
2417 
2418 /* Note: We plan to rework this in linux-6.4 */
2419 int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri)
2420 {
2421 	unsigned int saved_end_offset, saved_truesize;
2422 	struct skb_shared_info *shinfo;
2423 	int res;
2424 
2425 	saved_end_offset = skb_end_offset(skb);
2426 	saved_truesize = skb->truesize;
2427 
2428 	res = pskb_expand_head(skb, 0, 0, pri);
2429 	if (res)
2430 		return res;
2431 
2432 	skb->truesize = saved_truesize;
2433 
2434 	if (likely(skb_end_offset(skb) == saved_end_offset))
2435 		return 0;
2436 
2437 	shinfo = skb_shinfo(skb);
2438 
2439 	/* We are about to change back skb->end,
2440 	 * we need to move skb_shinfo() to its new location.
2441 	 */
2442 	memmove(skb->head + saved_end_offset,
2443 		shinfo,
2444 		offsetof(struct skb_shared_info, frags[shinfo->nr_frags]));
2445 
2446 	skb_set_end_offset(skb, saved_end_offset);
2447 
2448 	return 0;
2449 }
2450 
2451 /**
2452  *	skb_expand_head - reallocate header of &sk_buff
2453  *	@skb: buffer to reallocate
2454  *	@headroom: needed headroom
2455  *
2456  *	Unlike skb_realloc_headroom, this one does not allocate a new skb
2457  *	if possible; copies skb->sk to new skb as needed
2458  *	and frees original skb in case of failures.
2459  *
2460  *	It expect increased headroom and generates warning otherwise.
2461  */
2462 
2463 struct sk_buff *skb_expand_head(struct sk_buff *skb, unsigned int headroom)
2464 {
2465 	int delta = headroom - skb_headroom(skb);
2466 	int osize = skb_end_offset(skb);
2467 	struct sock *sk = skb->sk;
2468 
2469 	if (WARN_ONCE(delta <= 0,
2470 		      "%s is expecting an increase in the headroom", __func__))
2471 		return skb;
2472 
2473 	delta = SKB_DATA_ALIGN(delta);
2474 	/* pskb_expand_head() might crash, if skb is shared. */
2475 	if (skb_shared(skb) || !is_skb_wmem(skb)) {
2476 		struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
2477 
2478 		if (unlikely(!nskb))
2479 			goto fail;
2480 
2481 		if (sk)
2482 			skb_set_owner_w(nskb, sk);
2483 		consume_skb(skb);
2484 		skb = nskb;
2485 	}
2486 	if (pskb_expand_head(skb, delta, 0, GFP_ATOMIC))
2487 		goto fail;
2488 
2489 	if (sk && is_skb_wmem(skb)) {
2490 		delta = skb_end_offset(skb) - osize;
2491 		refcount_add(delta, &sk->sk_wmem_alloc);
2492 		skb->truesize += delta;
2493 	}
2494 	return skb;
2495 
2496 fail:
2497 	kfree_skb(skb);
2498 	return NULL;
2499 }
2500 EXPORT_SYMBOL(skb_expand_head);
2501 
2502 /**
2503  *	skb_copy_expand	-	copy and expand sk_buff
2504  *	@skb: buffer to copy
2505  *	@newheadroom: new free bytes at head
2506  *	@newtailroom: new free bytes at tail
2507  *	@gfp_mask: allocation priority
2508  *
2509  *	Make a copy of both an &sk_buff and its data and while doing so
2510  *	allocate additional space.
2511  *
2512  *	This is used when the caller wishes to modify the data and needs a
2513  *	private copy of the data to alter as well as more space for new fields.
2514  *	Returns %NULL on failure or the pointer to the buffer
2515  *	on success. The returned buffer has a reference count of 1.
2516  *
2517  *	You must pass %GFP_ATOMIC as the allocation priority if this function
2518  *	is called from an interrupt.
2519  */
2520 struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
2521 				int newheadroom, int newtailroom,
2522 				gfp_t gfp_mask)
2523 {
2524 	/*
2525 	 *	Allocate the copy buffer
2526 	 */
2527 	int head_copy_len, head_copy_off;
2528 	struct sk_buff *n;
2529 	int oldheadroom;
2530 
2531 	if (!skb_frags_readable(skb))
2532 		return NULL;
2533 
2534 	if (WARN_ON_ONCE(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST))
2535 		return NULL;
2536 
2537 	oldheadroom = skb_headroom(skb);
2538 	n = __alloc_skb(newheadroom + skb->len + newtailroom,
2539 			gfp_mask, skb_alloc_rx_flag(skb),
2540 			NUMA_NO_NODE);
2541 	if (!n)
2542 		return NULL;
2543 
2544 	skb_reserve(n, newheadroom);
2545 
2546 	/* Set the tail pointer and length */
2547 	skb_put(n, skb->len);
2548 
2549 	head_copy_len = oldheadroom;
2550 	head_copy_off = 0;
2551 	if (newheadroom <= head_copy_len)
2552 		head_copy_len = newheadroom;
2553 	else
2554 		head_copy_off = newheadroom - head_copy_len;
2555 
2556 	/* Copy the linear header and data. */
2557 	if (unlikely(skb_copy_bits(skb, -head_copy_len,
2558 				   n->head + head_copy_off,
2559 				   skb->len + head_copy_len))) {
2560 		DEBUG_NET_WARN_ON_ONCE(1);
2561 		kfree_skb(n);
2562 		return NULL;
2563 	}
2564 
2565 	skb_copy_header(n, skb);
2566 
2567 	skb_headers_offset_update(n, newheadroom - oldheadroom);
2568 
2569 	return n;
2570 }
2571 EXPORT_SYMBOL(skb_copy_expand);
2572 
2573 /**
2574  *	__skb_pad		-	zero pad the tail of an skb
2575  *	@skb: buffer to pad
2576  *	@pad: space to pad
2577  *	@free_on_error: free buffer on error
2578  *
2579  *	Ensure that a buffer is followed by a padding area that is zero
2580  *	filled. Used by network drivers which may DMA or transfer data
2581  *	beyond the buffer end onto the wire.
2582  *
2583  *	May return error in out of memory cases. The skb is freed on error
2584  *	if @free_on_error is true.
2585  */
2586 
2587 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error)
2588 {
2589 	int err;
2590 	int ntail;
2591 
2592 	/* If the skbuff is non linear tailroom is always zero.. */
2593 	if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
2594 		memset(skb->data+skb->len, 0, pad);
2595 		return 0;
2596 	}
2597 
2598 	ntail = skb->data_len + pad - (skb->end - skb->tail);
2599 	if (likely(skb_cloned(skb) || ntail > 0)) {
2600 		err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
2601 		if (unlikely(err))
2602 			goto free_skb;
2603 	}
2604 
2605 	/* FIXME: The use of this function with non-linear skb's really needs
2606 	 * to be audited.
2607 	 */
2608 	err = skb_linearize(skb);
2609 	if (unlikely(err))
2610 		goto free_skb;
2611 
2612 	memset(skb->data + skb->len, 0, pad);
2613 	return 0;
2614 
2615 free_skb:
2616 	if (free_on_error)
2617 		kfree_skb(skb);
2618 	return err;
2619 }
2620 EXPORT_SYMBOL(__skb_pad);
2621 
2622 /**
2623  *	pskb_put - add data to the tail of a potentially fragmented buffer
2624  *	@skb: start of the buffer to use
2625  *	@tail: tail fragment of the buffer to use
2626  *	@len: amount of data to add
2627  *
2628  *	This function extends the used data area of the potentially
2629  *	fragmented buffer. @tail must be the last fragment of @skb -- or
2630  *	@skb itself. If this would exceed the total buffer size the kernel
2631  *	will panic. A pointer to the first byte of the extra data is
2632  *	returned.
2633  */
2634 
2635 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len)
2636 {
2637 	if (tail != skb) {
2638 		skb->data_len += len;
2639 		skb->len += len;
2640 	}
2641 	return skb_put(tail, len);
2642 }
2643 EXPORT_SYMBOL_GPL(pskb_put);
2644 
2645 /**
2646  *	skb_put - add data to a buffer
2647  *	@skb: buffer to use
2648  *	@len: amount of data to add
2649  *
2650  *	This function extends the used data area of the buffer. If this would
2651  *	exceed the total buffer size the kernel will panic. A pointer to the
2652  *	first byte of the extra data is returned.
2653  */
2654 void *skb_put(struct sk_buff *skb, unsigned int len)
2655 {
2656 	void *tmp = skb_tail_pointer(skb);
2657 	SKB_LINEAR_ASSERT(skb);
2658 	skb->tail += len;
2659 	skb->len  += len;
2660 	if (unlikely(skb->tail > skb->end))
2661 		skb_over_panic(skb, len, __builtin_return_address(0));
2662 	return tmp;
2663 }
2664 EXPORT_SYMBOL(skb_put);
2665 
2666 /**
2667  *	skb_push - add data to the start of a buffer
2668  *	@skb: buffer to use
2669  *	@len: amount of data to add
2670  *
2671  *	This function extends the used data area of the buffer at the buffer
2672  *	start. If this would exceed the total buffer headroom the kernel will
2673  *	panic. A pointer to the first byte of the extra data is returned.
2674  */
2675 void *skb_push(struct sk_buff *skb, unsigned int len)
2676 {
2677 	skb->data -= len;
2678 	skb->len  += len;
2679 	if (unlikely(skb->data < skb->head))
2680 		skb_under_panic(skb, len, __builtin_return_address(0));
2681 	return skb->data;
2682 }
2683 EXPORT_SYMBOL(skb_push);
2684 
2685 /**
2686  *	skb_pull - remove data from the start of a buffer
2687  *	@skb: buffer to use
2688  *	@len: amount of data to remove
2689  *
2690  *	This function removes data from the start of a buffer, returning
2691  *	the memory to the headroom. A pointer to the next data in the buffer
2692  *	is returned. Once the data has been pulled future pushes will overwrite
2693  *	the old data.
2694  */
2695 void *skb_pull(struct sk_buff *skb, unsigned int len)
2696 {
2697 	return skb_pull_inline(skb, len);
2698 }
2699 EXPORT_SYMBOL(skb_pull);
2700 
2701 /**
2702  *	skb_pull_data - remove data from the start of a buffer returning its
2703  *	original position.
2704  *	@skb: buffer to use
2705  *	@len: amount of data to remove
2706  *
2707  *	This function removes data from the start of a buffer, returning
2708  *	the memory to the headroom. A pointer to the original data in the buffer
2709  *	is returned after checking if there is enough data to pull. Once the
2710  *	data has been pulled future pushes will overwrite the old data.
2711  */
2712 void *skb_pull_data(struct sk_buff *skb, size_t len)
2713 {
2714 	void *data = skb->data;
2715 
2716 	if (skb->len < len)
2717 		return NULL;
2718 
2719 	skb_pull(skb, len);
2720 
2721 	return data;
2722 }
2723 EXPORT_SYMBOL(skb_pull_data);
2724 
2725 /**
2726  *	skb_trim - remove end from a buffer
2727  *	@skb: buffer to alter
2728  *	@len: new length
2729  *
2730  *	Cut the length of a buffer down by removing data from the tail. If
2731  *	the buffer is already under the length specified it is not modified.
2732  *	The skb must be linear.
2733  */
2734 void skb_trim(struct sk_buff *skb, unsigned int len)
2735 {
2736 	if (skb->len > len)
2737 		__skb_trim(skb, len);
2738 }
2739 EXPORT_SYMBOL(skb_trim);
2740 
2741 /* Trims skb to length len. It can change skb pointers.
2742  */
2743 
2744 int ___pskb_trim(struct sk_buff *skb, unsigned int len)
2745 {
2746 	struct sk_buff **fragp;
2747 	struct sk_buff *frag;
2748 	int offset = skb_headlen(skb);
2749 	int nfrags = skb_shinfo(skb)->nr_frags;
2750 	int i;
2751 	int err;
2752 
2753 	if (skb_cloned(skb) &&
2754 	    unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
2755 		return err;
2756 
2757 	i = 0;
2758 	if (offset >= len)
2759 		goto drop_pages;
2760 
2761 	for (; i < nfrags; i++) {
2762 		int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
2763 
2764 		if (end < len) {
2765 			offset = end;
2766 			continue;
2767 		}
2768 
2769 		skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
2770 
2771 drop_pages:
2772 		skb_shinfo(skb)->nr_frags = i;
2773 
2774 		for (; i < nfrags; i++)
2775 			skb_frag_unref(skb, i);
2776 
2777 		if (skb_has_frag_list(skb))
2778 			skb_drop_fraglist(skb);
2779 		goto done;
2780 	}
2781 
2782 	for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
2783 	     fragp = &frag->next) {
2784 		int end = offset + frag->len;
2785 
2786 		if (skb_shared(frag)) {
2787 			struct sk_buff *nfrag;
2788 
2789 			nfrag = skb_clone(frag, GFP_ATOMIC);
2790 			if (unlikely(!nfrag))
2791 				return -ENOMEM;
2792 
2793 			nfrag->next = frag->next;
2794 			consume_skb(frag);
2795 			frag = nfrag;
2796 			*fragp = frag;
2797 		}
2798 
2799 		if (end < len) {
2800 			offset = end;
2801 			continue;
2802 		}
2803 
2804 		if (end > len &&
2805 		    unlikely((err = pskb_trim(frag, len - offset))))
2806 			return err;
2807 
2808 		if (frag->next)
2809 			skb_drop_list(&frag->next);
2810 		break;
2811 	}
2812 
2813 done:
2814 	if (len > skb_headlen(skb)) {
2815 		skb->data_len -= skb->len - len;
2816 		skb->len       = len;
2817 	} else {
2818 		skb->len       = len;
2819 		skb->data_len  = 0;
2820 		skb_set_tail_pointer(skb, len);
2821 	}
2822 	if (!skb_shinfo(skb)->nr_frags && !skb_has_frag_list(skb))
2823 		skb->unreadable = 0;
2824 
2825 	if (!skb->sk || skb->destructor == sock_edemux)
2826 		skb_condense(skb);
2827 	return 0;
2828 }
2829 EXPORT_SYMBOL(___pskb_trim);
2830 
2831 static int pskb_trim_rcsum_complete(struct sk_buff *skb, unsigned int len)
2832 {
2833 	int delta = skb->len - len;
2834 
2835 	if (skb_frags_readable(skb)) {
2836 		skb->csum = csum_block_sub(skb->csum,
2837 					   skb_checksum(skb, len, delta, 0),
2838 					   len);
2839 		return 0;
2840 	}
2841 
2842 	if (len > skb_headlen(skb))
2843 		return -EFAULT;
2844 
2845 	/* The trimmed bytes are unreadable, but the remaining packet can be
2846 	 * checksummed by software after trimming.
2847 	 */
2848 	skb->ip_summed = CHECKSUM_NONE;
2849 	return 0;
2850 }
2851 
2852 /* Note : use pskb_trim_rcsum() instead of calling this directly
2853  */
2854 int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len)
2855 {
2856 	if (skb->ip_summed == CHECKSUM_COMPLETE) {
2857 		int err;
2858 
2859 		err = pskb_trim_rcsum_complete(skb, len);
2860 		if (err)
2861 			return err;
2862 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
2863 		int hdlen = (len > skb_headlen(skb)) ? skb_headlen(skb) : len;
2864 		int offset = skb_checksum_start_offset(skb) + skb->csum_offset;
2865 
2866 		if (offset + sizeof(__sum16) > hdlen)
2867 			return -EINVAL;
2868 	}
2869 	return __pskb_trim(skb, len);
2870 }
2871 EXPORT_SYMBOL(pskb_trim_rcsum_slow);
2872 
2873 /**
2874  *	__pskb_pull_tail - advance tail of skb header
2875  *	@skb: buffer to reallocate
2876  *	@delta: number of bytes to advance tail
2877  *
2878  *	The function makes a sense only on a fragmented &sk_buff,
2879  *	it expands header moving its tail forward and copying necessary
2880  *	data from fragmented part.
2881  *
2882  *	&sk_buff MUST have reference count of 1.
2883  *
2884  *	Returns %NULL (and &sk_buff does not change) if pull failed
2885  *	or value of new tail of skb in the case of success.
2886  *
2887  *	All the pointers pointing into skb header may change and must be
2888  *	reloaded after call to this function.
2889  */
2890 
2891 /* Moves tail of skb head forward, copying data from fragmented part,
2892  * when it is necessary.
2893  * 1. It may fail due to malloc failure.
2894  * 2. It may change skb pointers.
2895  *
2896  * It is pretty complicated. Luckily, it is called only in exceptional cases.
2897  */
2898 void *__pskb_pull_tail(struct sk_buff *skb, int delta)
2899 {
2900 	/* If skb has not enough free space at tail, get new one
2901 	 * plus 128 bytes for future expansions. If we have enough
2902 	 * room at tail, reallocate without expansion only if skb is cloned.
2903 	 */
2904 	int i, k, eat = (skb->tail + delta) - skb->end;
2905 
2906 	if (!skb_frags_readable(skb))
2907 		return NULL;
2908 
2909 	if (eat > 0 || skb_cloned(skb)) {
2910 		if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
2911 				     GFP_ATOMIC))
2912 			return NULL;
2913 	}
2914 
2915 	BUG_ON(skb_copy_bits(skb, skb_headlen(skb),
2916 			     skb_tail_pointer(skb), delta));
2917 
2918 	/* Optimization: no fragments, no reasons to preestimate
2919 	 * size of pulled pages. Superb.
2920 	 */
2921 	if (!skb_has_frag_list(skb))
2922 		goto pull_pages;
2923 
2924 	/* Estimate size of pulled pages. */
2925 	eat = delta;
2926 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2927 		int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
2928 
2929 		if (size >= eat)
2930 			goto pull_pages;
2931 		eat -= size;
2932 	}
2933 
2934 	/* If we need update frag list, we are in troubles.
2935 	 * Certainly, it is possible to add an offset to skb data,
2936 	 * but taking into account that pulling is expected to
2937 	 * be very rare operation, it is worth to fight against
2938 	 * further bloating skb head and crucify ourselves here instead.
2939 	 * Pure masohism, indeed. 8)8)
2940 	 */
2941 	if (eat) {
2942 		struct sk_buff *list = skb_shinfo(skb)->frag_list;
2943 		struct sk_buff *clone = NULL;
2944 		struct sk_buff *insp = NULL;
2945 
2946 		do {
2947 			if (list->len <= eat) {
2948 				/* Eaten as whole. */
2949 				eat -= list->len;
2950 				list = list->next;
2951 				insp = list;
2952 			} else {
2953 				/* Eaten partially. */
2954 				if (skb_is_gso(skb) && !list->head_frag &&
2955 				    skb_headlen(list))
2956 					skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2957 
2958 				if (skb_shared(list)) {
2959 					/* Sucks! We need to fork list. :-( */
2960 					clone = skb_clone(list, GFP_ATOMIC);
2961 					if (!clone)
2962 						return NULL;
2963 					insp = list->next;
2964 					list = clone;
2965 				} else {
2966 					/* This may be pulled without
2967 					 * problems. */
2968 					insp = list;
2969 				}
2970 				if (!pskb_pull(list, eat)) {
2971 					kfree_skb(clone);
2972 					return NULL;
2973 				}
2974 				break;
2975 			}
2976 		} while (eat);
2977 
2978 		/* Free pulled out fragments. */
2979 		while ((list = skb_shinfo(skb)->frag_list) != insp) {
2980 			skb_shinfo(skb)->frag_list = list->next;
2981 			consume_skb(list);
2982 		}
2983 		/* And insert new clone at head. */
2984 		if (clone) {
2985 			clone->next = list;
2986 			skb_shinfo(skb)->frag_list = clone;
2987 		}
2988 	}
2989 	/* Success! Now we may commit changes to skb data. */
2990 
2991 pull_pages:
2992 	eat = delta;
2993 	k = 0;
2994 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2995 		int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
2996 
2997 		if (size <= eat) {
2998 			skb_frag_unref(skb, i);
2999 			eat -= size;
3000 		} else {
3001 			skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
3002 
3003 			*frag = skb_shinfo(skb)->frags[i];
3004 			if (eat) {
3005 				skb_frag_off_add(frag, eat);
3006 				skb_frag_size_sub(frag, eat);
3007 				if (!i)
3008 					goto end;
3009 				eat = 0;
3010 			}
3011 			k++;
3012 		}
3013 	}
3014 	skb_shinfo(skb)->nr_frags = k;
3015 
3016 end:
3017 	skb->tail     += delta;
3018 	skb->data_len -= delta;
3019 
3020 	if (!skb->data_len)
3021 		skb_zcopy_clear(skb, false);
3022 
3023 	return skb_tail_pointer(skb);
3024 }
3025 EXPORT_SYMBOL(__pskb_pull_tail);
3026 
3027 /**
3028  *	skb_copy_bits - copy bits from skb to kernel buffer
3029  *	@skb: source skb
3030  *	@offset: offset in source
3031  *	@to: destination buffer
3032  *	@len: number of bytes to copy
3033  *
3034  *	Copy the specified number of bytes from the source skb to the
3035  *	destination buffer.
3036  *
3037  *	CAUTION ! :
3038  *		If its prototype is ever changed,
3039  *		check arch/{*}/net/{*}.S files,
3040  *		since it is called from BPF assembly code.
3041  */
3042 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
3043 {
3044 	int start = skb_headlen(skb);
3045 	struct sk_buff *frag_iter;
3046 	int i, copy;
3047 
3048 	if (offset > (int)skb->len - len)
3049 		goto fault;
3050 
3051 	/* Copy header. */
3052 	if ((copy = start - offset) > 0) {
3053 		if (copy > len)
3054 			copy = len;
3055 		skb_copy_from_linear_data_offset(skb, offset, to, copy);
3056 		if ((len -= copy) == 0)
3057 			return 0;
3058 		offset += copy;
3059 		to     += copy;
3060 	}
3061 
3062 	if (!skb_frags_readable(skb))
3063 		goto fault;
3064 
3065 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3066 		int end;
3067 		skb_frag_t *f = &skb_shinfo(skb)->frags[i];
3068 
3069 		WARN_ON(start > offset + len);
3070 
3071 		end = start + skb_frag_size(f);
3072 		if ((copy = end - offset) > 0) {
3073 			u32 p_off, p_len, copied;
3074 			struct page *p;
3075 			u8 *vaddr;
3076 
3077 			if (copy > len)
3078 				copy = len;
3079 
3080 			skb_frag_foreach_page(f,
3081 					      skb_frag_off(f) + offset - start,
3082 					      copy, p, p_off, p_len, copied) {
3083 				vaddr = kmap_atomic(p);
3084 				memcpy(to + copied, vaddr + p_off, p_len);
3085 				kunmap_atomic(vaddr);
3086 			}
3087 
3088 			if ((len -= copy) == 0)
3089 				return 0;
3090 			offset += copy;
3091 			to     += copy;
3092 		}
3093 		start = end;
3094 	}
3095 
3096 	skb_walk_frags(skb, frag_iter) {
3097 		int end;
3098 
3099 		WARN_ON(start > offset + len);
3100 
3101 		end = start + frag_iter->len;
3102 		if ((copy = end - offset) > 0) {
3103 			if (copy > len)
3104 				copy = len;
3105 			if (skb_copy_bits(frag_iter, offset - start, to, copy))
3106 				goto fault;
3107 			if ((len -= copy) == 0)
3108 				return 0;
3109 			offset += copy;
3110 			to     += copy;
3111 		}
3112 		start = end;
3113 	}
3114 
3115 	if (!len)
3116 		return 0;
3117 
3118 fault:
3119 	return -EFAULT;
3120 }
3121 EXPORT_SYMBOL(skb_copy_bits);
3122 
3123 /*
3124  * Callback from splice_to_pipe(), if we need to release some pages
3125  * at the end of the spd in case we error'ed out in filling the pipe.
3126  */
3127 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
3128 {
3129 	put_page(spd->pages[i]);
3130 }
3131 
3132 static struct page *linear_to_page(struct page *page, unsigned int *len,
3133 				   unsigned int *offset,
3134 				   struct sock *sk)
3135 {
3136 	struct page_frag *pfrag = sk_page_frag(sk);
3137 
3138 	if (!sk_page_frag_refill(sk, pfrag))
3139 		return NULL;
3140 
3141 	*len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);
3142 
3143 	memcpy(page_address(pfrag->page) + pfrag->offset,
3144 	       page_address(page) + *offset, *len);
3145 	*offset = pfrag->offset;
3146 	pfrag->offset += *len;
3147 
3148 	return pfrag->page;
3149 }
3150 
3151 static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
3152 			     struct page *page,
3153 			     unsigned int offset)
3154 {
3155 	return	spd->nr_pages &&
3156 		spd->pages[spd->nr_pages - 1] == page &&
3157 		(spd->partial[spd->nr_pages - 1].offset +
3158 		 spd->partial[spd->nr_pages - 1].len == offset);
3159 }
3160 
3161 /*
3162  * Fill page/offset/length into spd, if it can hold more pages.
3163  */
3164 static bool spd_fill_page(struct splice_pipe_desc *spd, struct page *page,
3165 			  unsigned int *len, unsigned int offset, bool linear,
3166 			  struct sock *sk)
3167 {
3168 	if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
3169 		return true;
3170 
3171 	if (linear) {
3172 		page = linear_to_page(page, len, &offset, sk);
3173 		if (!page)
3174 			return true;
3175 	}
3176 	if (spd_can_coalesce(spd, page, offset)) {
3177 		spd->partial[spd->nr_pages - 1].len += *len;
3178 		return false;
3179 	}
3180 	get_page(page);
3181 	spd->pages[spd->nr_pages] = page;
3182 	spd->partial[spd->nr_pages].len = *len;
3183 	spd->partial[spd->nr_pages].offset = offset;
3184 	spd->nr_pages++;
3185 
3186 	return false;
3187 }
3188 
3189 static bool __splice_segment(struct page *page, unsigned int poff,
3190 			     unsigned int plen, unsigned int *off,
3191 			     unsigned int *len,
3192 			     struct splice_pipe_desc *spd, bool linear,
3193 			     struct sock *sk)
3194 {
3195 	if (!*len)
3196 		return true;
3197 
3198 	/* skip this segment if already processed */
3199 	if (*off >= plen) {
3200 		*off -= plen;
3201 		return false;
3202 	}
3203 
3204 	/* ignore any bits we already processed */
3205 	poff += *off;
3206 	plen -= *off;
3207 	*off = 0;
3208 
3209 	do {
3210 		unsigned int flen = min(*len, plen);
3211 
3212 		if (spd_fill_page(spd, page, &flen, poff, linear, sk))
3213 			return true;
3214 		poff += flen;
3215 		plen -= flen;
3216 		*len -= flen;
3217 		if (!*len)
3218 			return true;
3219 	} while (plen);
3220 
3221 	return false;
3222 }
3223 
3224 /*
3225  * Map linear and fragment data from the skb to spd. It reports true if the
3226  * pipe is full or if we already spliced the requested length.
3227  */
3228 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
3229 			      unsigned int *offset, unsigned int *len,
3230 			      struct splice_pipe_desc *spd, struct sock *sk)
3231 {
3232 	struct sk_buff *iter;
3233 	int seg;
3234 
3235 	/* map the linear part :
3236 	 * If skb->head_frag is set, this 'linear' part is backed by a
3237 	 * fragment, and if the head is not shared with any clones then
3238 	 * we can avoid a copy since we own the head portion of this page.
3239 	 */
3240 	if (__splice_segment(virt_to_page(skb->data),
3241 			     (unsigned long) skb->data & (PAGE_SIZE - 1),
3242 			     skb_headlen(skb),
3243 			     offset, len, spd,
3244 			     skb_head_is_locked(skb),
3245 			     sk))
3246 		return true;
3247 
3248 	/*
3249 	 * then map the fragments
3250 	 */
3251 	if (!skb_frags_readable(skb))
3252 		return false;
3253 
3254 	for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
3255 		const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
3256 
3257 		if (WARN_ON_ONCE(!skb_frag_page(f)))
3258 			return false;
3259 
3260 		if (__splice_segment(skb_frag_page(f),
3261 				     skb_frag_off(f), skb_frag_size(f),
3262 				     offset, len, spd, false, sk))
3263 			return true;
3264 	}
3265 
3266 	skb_walk_frags(skb, iter) {
3267 		if (*offset >= iter->len) {
3268 			*offset -= iter->len;
3269 			continue;
3270 		}
3271 		/* __skb_splice_bits() only fails if the output has no room
3272 		 * left, so no point in going over the frag_list for the error
3273 		 * case.
3274 		 */
3275 		if (__skb_splice_bits(iter, pipe, offset, len, spd, sk))
3276 			return true;
3277 	}
3278 
3279 	return false;
3280 }
3281 
3282 /*
3283  * Map data from the skb to a pipe. Should handle both the linear part,
3284  * the fragments, and the frag list.
3285  */
3286 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
3287 		    struct pipe_inode_info *pipe, unsigned int tlen,
3288 		    unsigned int flags)
3289 {
3290 	struct partial_page partial[MAX_SKB_FRAGS];
3291 	struct page *pages[MAX_SKB_FRAGS];
3292 	struct splice_pipe_desc spd = {
3293 		.pages = pages,
3294 		.partial = partial,
3295 		.nr_pages_max = MAX_SKB_FRAGS,
3296 		.ops = &default_pipe_buf_ops,
3297 		.spd_release = sock_spd_release,
3298 	};
3299 	int ret = 0;
3300 
3301 	__skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk);
3302 
3303 	if (spd.nr_pages)
3304 		ret = splice_to_pipe(pipe, &spd);
3305 
3306 	return ret;
3307 }
3308 EXPORT_SYMBOL_GPL(skb_splice_bits);
3309 
3310 static int sendmsg_locked(struct sock *sk, struct msghdr *msg)
3311 {
3312 	struct socket *sock = sk->sk_socket;
3313 	size_t size = msg_data_left(msg);
3314 
3315 	if (!sock)
3316 		return -EINVAL;
3317 
3318 	if (!sock->ops->sendmsg_locked)
3319 		return sock_no_sendmsg_locked(sk, msg, size);
3320 
3321 	return sock->ops->sendmsg_locked(sk, msg, size);
3322 }
3323 
3324 static int sendmsg_unlocked(struct sock *sk, struct msghdr *msg)
3325 {
3326 	struct socket *sock = sk->sk_socket;
3327 
3328 	if (!sock)
3329 		return -EINVAL;
3330 	return sock_sendmsg(sock, msg);
3331 }
3332 
3333 typedef int (*sendmsg_func)(struct sock *sk, struct msghdr *msg);
3334 static int __skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset,
3335 			   int len, sendmsg_func sendmsg, int flags)
3336 {
3337 	int more_hint = sk_is_tcp(sk) ? MSG_MORE : 0;
3338 	unsigned int orig_len = len;
3339 	struct sk_buff *head = skb;
3340 	unsigned short fragidx;
3341 	int slen, ret;
3342 
3343 do_frag_list:
3344 
3345 	/* Deal with head data */
3346 	while (offset < skb_headlen(skb) && len) {
3347 		struct kvec kv;
3348 		struct msghdr msg;
3349 
3350 		slen = min_t(int, len, skb_headlen(skb) - offset);
3351 		kv.iov_base = skb->data + offset;
3352 		kv.iov_len = slen;
3353 		memset(&msg, 0, sizeof(msg));
3354 		msg.msg_flags = MSG_DONTWAIT | flags;
3355 		if (slen < len)
3356 			msg.msg_flags |= more_hint;
3357 
3358 		iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, &kv, 1, slen);
3359 		ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked,
3360 				      sendmsg_unlocked, sk, &msg);
3361 		if (ret <= 0)
3362 			goto error;
3363 
3364 		offset += ret;
3365 		len -= ret;
3366 	}
3367 
3368 	/* All the data was skb head? */
3369 	if (!len)
3370 		goto out;
3371 
3372 	/* Make offset relative to start of frags */
3373 	offset -= skb_headlen(skb);
3374 
3375 	/* Find where we are in frag list */
3376 	for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
3377 		skb_frag_t *frag  = &skb_shinfo(skb)->frags[fragidx];
3378 
3379 		if (offset < skb_frag_size(frag))
3380 			break;
3381 
3382 		offset -= skb_frag_size(frag);
3383 	}
3384 
3385 	for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
3386 		skb_frag_t *frag  = &skb_shinfo(skb)->frags[fragidx];
3387 
3388 		slen = min_t(size_t, len, skb_frag_size(frag) - offset);
3389 
3390 		while (slen) {
3391 			struct bio_vec bvec;
3392 			struct msghdr msg = {
3393 				.msg_flags = MSG_SPLICE_PAGES | MSG_DONTWAIT |
3394 					     flags,
3395 			};
3396 
3397 			if (slen < len)
3398 				msg.msg_flags |= more_hint;
3399 			bvec_set_page(&bvec, skb_frag_page(frag), slen,
3400 				      skb_frag_off(frag) + offset);
3401 			iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1,
3402 				      slen);
3403 
3404 			ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked,
3405 					      sendmsg_unlocked, sk, &msg);
3406 			if (ret <= 0)
3407 				goto error;
3408 
3409 			len -= ret;
3410 			offset += ret;
3411 			slen -= ret;
3412 		}
3413 
3414 		offset = 0;
3415 	}
3416 
3417 	if (len) {
3418 		/* Process any frag lists */
3419 
3420 		if (skb == head) {
3421 			if (skb_has_frag_list(skb)) {
3422 				skb = skb_shinfo(skb)->frag_list;
3423 				goto do_frag_list;
3424 			}
3425 		} else if (skb->next) {
3426 			skb = skb->next;
3427 			goto do_frag_list;
3428 		}
3429 	}
3430 
3431 out:
3432 	return orig_len - len;
3433 
3434 error:
3435 	return orig_len == len ? ret : orig_len - len;
3436 }
3437 
3438 /* Send skb data on a socket. Socket must be locked. */
3439 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
3440 			 int len)
3441 {
3442 	return __skb_send_sock(sk, skb, offset, len, sendmsg_locked, 0);
3443 }
3444 EXPORT_SYMBOL_GPL(skb_send_sock_locked);
3445 
3446 int skb_send_sock_locked_with_flags(struct sock *sk, struct sk_buff *skb,
3447 				    int offset, int len, int flags)
3448 {
3449 	return __skb_send_sock(sk, skb, offset, len, sendmsg_locked, flags);
3450 }
3451 EXPORT_SYMBOL_GPL(skb_send_sock_locked_with_flags);
3452 
3453 /* Send skb data on a socket. Socket must be unlocked. */
3454 int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len)
3455 {
3456 	return __skb_send_sock(sk, skb, offset, len, sendmsg_unlocked, 0);
3457 }
3458 
3459 /**
3460  *	skb_store_bits - store bits from kernel buffer to skb
3461  *	@skb: destination buffer
3462  *	@offset: offset in destination
3463  *	@from: source buffer
3464  *	@len: number of bytes to copy
3465  *
3466  *	Copy the specified number of bytes from the source buffer to the
3467  *	destination skb.  This function handles all the messy bits of
3468  *	traversing fragment lists and such.
3469  */
3470 
3471 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
3472 {
3473 	int start = skb_headlen(skb);
3474 	struct sk_buff *frag_iter;
3475 	int i, copy;
3476 
3477 	if (offset > (int)skb->len - len)
3478 		goto fault;
3479 
3480 	if ((copy = start - offset) > 0) {
3481 		if (copy > len)
3482 			copy = len;
3483 		skb_copy_to_linear_data_offset(skb, offset, from, copy);
3484 		if ((len -= copy) == 0)
3485 			return 0;
3486 		offset += copy;
3487 		from += copy;
3488 	}
3489 
3490 	if (!skb_frags_readable(skb))
3491 		goto fault;
3492 
3493 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3494 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3495 		int end;
3496 
3497 		WARN_ON(start > offset + len);
3498 
3499 		end = start + skb_frag_size(frag);
3500 		if ((copy = end - offset) > 0) {
3501 			u32 p_off, p_len, copied;
3502 			struct page *p;
3503 			u8 *vaddr;
3504 
3505 			if (copy > len)
3506 				copy = len;
3507 
3508 			skb_frag_foreach_page(frag,
3509 					      skb_frag_off(frag) + offset - start,
3510 					      copy, p, p_off, p_len, copied) {
3511 				vaddr = kmap_atomic(p);
3512 				memcpy(vaddr + p_off, from + copied, p_len);
3513 				kunmap_atomic(vaddr);
3514 			}
3515 
3516 			if ((len -= copy) == 0)
3517 				return 0;
3518 			offset += copy;
3519 			from += copy;
3520 		}
3521 		start = end;
3522 	}
3523 
3524 	skb_walk_frags(skb, frag_iter) {
3525 		int end;
3526 
3527 		WARN_ON(start > offset + len);
3528 
3529 		end = start + frag_iter->len;
3530 		if ((copy = end - offset) > 0) {
3531 			if (copy > len)
3532 				copy = len;
3533 			if (skb_store_bits(frag_iter, offset - start,
3534 					   from, copy))
3535 				goto fault;
3536 			if ((len -= copy) == 0)
3537 				return 0;
3538 			offset += copy;
3539 			from += copy;
3540 		}
3541 		start = end;
3542 	}
3543 	if (!len)
3544 		return 0;
3545 
3546 fault:
3547 	return -EFAULT;
3548 }
3549 EXPORT_SYMBOL(skb_store_bits);
3550 
3551 /* Checksum skb data. */
3552 __wsum skb_checksum(const struct sk_buff *skb, int offset, int len, __wsum csum)
3553 {
3554 	int start = skb_headlen(skb);
3555 	int i, copy = start - offset;
3556 	struct sk_buff *frag_iter;
3557 	int pos = 0;
3558 
3559 	/* Checksum header. */
3560 	if (copy > 0) {
3561 		if (copy > len)
3562 			copy = len;
3563 		csum = csum_partial(skb->data + offset, copy, csum);
3564 		if ((len -= copy) == 0)
3565 			return csum;
3566 		offset += copy;
3567 		pos	= copy;
3568 	}
3569 
3570 	if (WARN_ON_ONCE(!skb_frags_readable(skb)))
3571 		return 0;
3572 
3573 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3574 		int end;
3575 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3576 
3577 		WARN_ON(start > offset + len);
3578 
3579 		end = start + skb_frag_size(frag);
3580 		if ((copy = end - offset) > 0) {
3581 			u32 p_off, p_len, copied;
3582 			struct page *p;
3583 			__wsum csum2;
3584 			u8 *vaddr;
3585 
3586 			if (copy > len)
3587 				copy = len;
3588 
3589 			skb_frag_foreach_page(frag,
3590 					      skb_frag_off(frag) + offset - start,
3591 					      copy, p, p_off, p_len, copied) {
3592 				vaddr = kmap_atomic(p);
3593 				csum2 = csum_partial(vaddr + p_off, p_len, 0);
3594 				kunmap_atomic(vaddr);
3595 				csum = csum_block_add(csum, csum2, pos);
3596 				pos += p_len;
3597 			}
3598 
3599 			if (!(len -= copy))
3600 				return csum;
3601 			offset += copy;
3602 		}
3603 		start = end;
3604 	}
3605 
3606 	skb_walk_frags(skb, frag_iter) {
3607 		int end;
3608 
3609 		WARN_ON(start > offset + len);
3610 
3611 		end = start + frag_iter->len;
3612 		if ((copy = end - offset) > 0) {
3613 			__wsum csum2;
3614 			if (copy > len)
3615 				copy = len;
3616 			csum2 = skb_checksum(frag_iter, offset - start, copy,
3617 					     0);
3618 			csum = csum_block_add(csum, csum2, pos);
3619 			if ((len -= copy) == 0)
3620 				return csum;
3621 			offset += copy;
3622 			pos    += copy;
3623 		}
3624 		start = end;
3625 	}
3626 	if (unlikely(len)) {
3627 		DEBUG_NET_WARN_ON_ONCE(1);
3628 		return 0;
3629 	}
3630 
3631 	return csum;
3632 }
3633 EXPORT_SYMBOL(skb_checksum);
3634 
3635 /* Both of above in one bottle. */
3636 
3637 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
3638 				    u8 *to, int len)
3639 {
3640 	int start = skb_headlen(skb);
3641 	int i, copy = start - offset;
3642 	struct sk_buff *frag_iter;
3643 	int pos = 0;
3644 	__wsum csum = 0;
3645 
3646 	/* Copy header. */
3647 	if (copy > 0) {
3648 		if (copy > len)
3649 			copy = len;
3650 		csum = csum_partial_copy_nocheck(skb->data + offset, to,
3651 						 copy);
3652 		if ((len -= copy) == 0)
3653 			return csum;
3654 		offset += copy;
3655 		to     += copy;
3656 		pos	= copy;
3657 	}
3658 
3659 	if (!skb_frags_readable(skb)) {
3660 		/* Don't hand the caller a buffer with stale bytes in it. */
3661 		if (len > 0)
3662 			memset(to, 0, len);
3663 		return 0;
3664 	}
3665 
3666 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3667 		int end;
3668 
3669 		WARN_ON(start > offset + len);
3670 
3671 		end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
3672 		if ((copy = end - offset) > 0) {
3673 			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3674 			u32 p_off, p_len, copied;
3675 			struct page *p;
3676 			__wsum csum2;
3677 			u8 *vaddr;
3678 
3679 			if (copy > len)
3680 				copy = len;
3681 
3682 			skb_frag_foreach_page(frag,
3683 					      skb_frag_off(frag) + offset - start,
3684 					      copy, p, p_off, p_len, copied) {
3685 				vaddr = kmap_atomic(p);
3686 				csum2 = csum_partial_copy_nocheck(vaddr + p_off,
3687 								  to + copied,
3688 								  p_len);
3689 				kunmap_atomic(vaddr);
3690 				csum = csum_block_add(csum, csum2, pos);
3691 				pos += p_len;
3692 			}
3693 
3694 			if (!(len -= copy))
3695 				return csum;
3696 			offset += copy;
3697 			to     += copy;
3698 		}
3699 		start = end;
3700 	}
3701 
3702 	skb_walk_frags(skb, frag_iter) {
3703 		__wsum csum2;
3704 		int end;
3705 
3706 		WARN_ON(start > offset + len);
3707 
3708 		end = start + frag_iter->len;
3709 		if ((copy = end - offset) > 0) {
3710 			if (copy > len)
3711 				copy = len;
3712 			csum2 = skb_copy_and_csum_bits(frag_iter,
3713 						       offset - start,
3714 						       to, copy);
3715 			csum = csum_block_add(csum, csum2, pos);
3716 			if ((len -= copy) == 0)
3717 				return csum;
3718 			offset += copy;
3719 			to     += copy;
3720 			pos    += copy;
3721 		}
3722 		start = end;
3723 	}
3724 	if (unlikely(len)) {
3725 		DEBUG_NET_WARN_ON_ONCE(1);
3726 		/* Don't hand the caller a buffer with stale bytes in it. */
3727 		if (len > 0)
3728 			memset(to, 0, len);
3729 		return 0;
3730 	}
3731 	return csum;
3732 }
3733 EXPORT_SYMBOL(skb_copy_and_csum_bits);
3734 
3735 #ifdef CONFIG_NET_CRC32C
3736 u32 skb_crc32c(const struct sk_buff *skb, int offset, int len, u32 crc)
3737 {
3738 	int start = skb_headlen(skb);
3739 	int i, copy = start - offset;
3740 	struct sk_buff *frag_iter;
3741 
3742 	if (copy > 0) {
3743 		copy = min(copy, len);
3744 		crc = crc32c(crc, skb->data + offset, copy);
3745 		len -= copy;
3746 		if (len == 0)
3747 			return crc;
3748 		offset += copy;
3749 	}
3750 
3751 	if (WARN_ON_ONCE(!skb_frags_readable(skb)))
3752 		return 0;
3753 
3754 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3755 		int end;
3756 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3757 
3758 		WARN_ON(start > offset + len);
3759 
3760 		end = start + skb_frag_size(frag);
3761 		copy = end - offset;
3762 		if (copy > 0) {
3763 			u32 p_off, p_len, copied;
3764 			struct page *p;
3765 			u8 *vaddr;
3766 
3767 			copy = min(copy, len);
3768 			skb_frag_foreach_page(frag,
3769 					      skb_frag_off(frag) + offset - start,
3770 					      copy, p, p_off, p_len, copied) {
3771 				vaddr = kmap_atomic(p);
3772 				crc = crc32c(crc, vaddr + p_off, p_len);
3773 				kunmap_atomic(vaddr);
3774 			}
3775 			len -= copy;
3776 			if (len == 0)
3777 				return crc;
3778 			offset += copy;
3779 		}
3780 		start = end;
3781 	}
3782 
3783 	skb_walk_frags(skb, frag_iter) {
3784 		int end;
3785 
3786 		WARN_ON(start > offset + len);
3787 
3788 		end = start + frag_iter->len;
3789 		copy = end - offset;
3790 		if (copy > 0) {
3791 			copy = min(copy, len);
3792 			crc = skb_crc32c(frag_iter, offset - start, copy, crc);
3793 			len -= copy;
3794 			if (len == 0)
3795 				return crc;
3796 			offset += copy;
3797 		}
3798 		start = end;
3799 	}
3800 	if (unlikely(len)) {
3801 		DEBUG_NET_WARN_ON_ONCE(1);
3802 		return 0;
3803 	}
3804 
3805 	return crc;
3806 }
3807 EXPORT_SYMBOL(skb_crc32c);
3808 #endif /* CONFIG_NET_CRC32C */
3809 
3810 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len)
3811 {
3812 	__sum16 sum;
3813 
3814 	sum = csum_fold(skb_checksum(skb, 0, len, skb->csum));
3815 	/* See comments in __skb_checksum_complete(). */
3816 	if (likely(!sum)) {
3817 		if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
3818 		    !skb->csum_complete_sw)
3819 			netdev_rx_csum_fault(skb->dev, skb);
3820 	}
3821 	if (!skb_shared(skb))
3822 		skb->csum_valid = !sum;
3823 	return sum;
3824 }
3825 EXPORT_SYMBOL(__skb_checksum_complete_head);
3826 
3827 /* This function assumes skb->csum already holds pseudo header's checksum,
3828  * which has been changed from the hardware checksum, for example, by
3829  * __skb_checksum_validate_complete(). And, the original skb->csum must
3830  * have been validated unsuccessfully for CHECKSUM_COMPLETE case.
3831  *
3832  * It returns non-zero if the recomputed checksum is still invalid, otherwise
3833  * zero. The new checksum is stored back into skb->csum unless the skb is
3834  * shared.
3835  */
3836 __sum16 __skb_checksum_complete(struct sk_buff *skb)
3837 {
3838 	__wsum csum;
3839 	__sum16 sum;
3840 
3841 	csum = skb_checksum(skb, 0, skb->len, 0);
3842 
3843 	sum = csum_fold(csum_add(skb->csum, csum));
3844 	/* This check is inverted, because we already knew the hardware
3845 	 * checksum is invalid before calling this function. So, if the
3846 	 * re-computed checksum is valid instead, then we have a mismatch
3847 	 * between the original skb->csum and skb_checksum(). This means either
3848 	 * the original hardware checksum is incorrect or we screw up skb->csum
3849 	 * when moving skb->data around.
3850 	 */
3851 	if (likely(!sum)) {
3852 		if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
3853 		    !skb->csum_complete_sw)
3854 			netdev_rx_csum_fault(skb->dev, skb);
3855 	}
3856 
3857 	if (!skb_shared(skb)) {
3858 		/* Save full packet checksum */
3859 		skb->csum = csum;
3860 		skb->ip_summed = CHECKSUM_COMPLETE;
3861 		skb->csum_complete_sw = 1;
3862 		skb->csum_valid = !sum;
3863 	}
3864 
3865 	return sum;
3866 }
3867 EXPORT_SYMBOL(__skb_checksum_complete);
3868 
3869  /**
3870  *	skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy()
3871  *	@from: source buffer
3872  *
3873  *	Calculates the amount of linear headroom needed in the 'to' skb passed
3874  *	into skb_zerocopy().
3875  */
3876 unsigned int
3877 skb_zerocopy_headlen(const struct sk_buff *from)
3878 {
3879 	unsigned int hlen = 0;
3880 
3881 	if (!from->head_frag ||
3882 	    skb_headlen(from) < L1_CACHE_BYTES ||
3883 	    skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) {
3884 		hlen = skb_headlen(from);
3885 		if (!hlen)
3886 			hlen = from->len;
3887 	}
3888 
3889 	if (skb_has_frag_list(from))
3890 		hlen = from->len;
3891 
3892 	return hlen;
3893 }
3894 EXPORT_SYMBOL_GPL(skb_zerocopy_headlen);
3895 
3896 /**
3897  *	skb_zerocopy - Zero copy skb to skb
3898  *	@to: destination buffer
3899  *	@from: source buffer
3900  *	@len: number of bytes to copy from source buffer
3901  *	@hlen: size of linear headroom in destination buffer
3902  *
3903  *	Copies up to `len` bytes from `from` to `to` by creating references
3904  *	to the frags in the source buffer.
3905  *
3906  *	The `hlen` as calculated by skb_zerocopy_headlen() specifies the
3907  *	headroom in the `to` buffer.
3908  *
3909  *	Return value:
3910  *	0: everything is OK
3911  *	-ENOMEM: couldn't orphan frags of @from due to lack of memory
3912  *	-EFAULT: skb_copy_bits() found some problem with skb geometry, or readable head
3913  *      payload would be mixed with unreadable frags.
3914  */
3915 int
3916 skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen)
3917 {
3918 	int i, j = 0;
3919 	int plen = 0; /* length of skb->head fragment */
3920 	int ret;
3921 	struct page *page;
3922 	unsigned int offset;
3923 
3924 	if (unlikely(!from->head_frag && !hlen)) {
3925 		DEBUG_NET_WARN_ON_ONCE(1);
3926 		return -EFAULT;
3927 	}
3928 
3929 	/* dont bother with small payloads */
3930 	if (len <= skb_tailroom(to))
3931 		return skb_copy_bits(from, 0, skb_put(to, len), len);
3932 
3933 	to->pp_recycle = from->pp_recycle;
3934 	if (hlen) {
3935 		ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen);
3936 		if (unlikely(ret))
3937 			return ret;
3938 		len -= hlen;
3939 	} else {
3940 		plen = min_t(int, skb_headlen(from), len);
3941 		if (plen) {
3942 			page = virt_to_head_page(from->head);
3943 			offset = from->data - (unsigned char *)page_address(page);
3944 			__skb_fill_netmem_desc(to, 0, page_to_netmem(page),
3945 					       offset, plen);
3946 			skb_frag_ref(to, 0);
3947 			j = 1;
3948 			len -= plen;
3949 		}
3950 	}
3951 
3952 	if (!skb_frags_readable(from) && j > 0 && len) {
3953 		skb_frag_unref(to, 0);
3954 		return -EFAULT;
3955 	}
3956 
3957 	skb_len_add(to, len + plen);
3958 
3959 	if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) {
3960 		if (j > 0)
3961 			skb_frag_unref(to, 0);
3962 		return -ENOMEM;
3963 	}
3964 	skb_zerocopy_clone(to, from, GFP_ATOMIC);
3965 
3966 	for (i = 0; i < skb_shinfo(from)->nr_frags; i++) {
3967 		int size;
3968 
3969 		if (!len)
3970 			break;
3971 		skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i];
3972 		size = min_t(int, skb_frag_size(&skb_shinfo(to)->frags[j]),
3973 					len);
3974 		skb_frag_size_set(&skb_shinfo(to)->frags[j], size);
3975 		len -= size;
3976 		skb_frag_ref(to, j);
3977 		j++;
3978 	}
3979 	skb_shinfo(to)->nr_frags = j;
3980 
3981 	skb_shinfo(to)->flags |= skb_shinfo(from)->flags & SKBFL_SHARED_FRAG;
3982 
3983 	if (i > 0 && from->unreadable)
3984 		to->unreadable = 1;
3985 
3986 	return 0;
3987 }
3988 EXPORT_SYMBOL_GPL(skb_zerocopy);
3989 
3990 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
3991 {
3992 	__wsum csum;
3993 	long csstart;
3994 
3995 	if (skb->ip_summed == CHECKSUM_PARTIAL)
3996 		csstart = skb_checksum_start_offset(skb);
3997 	else
3998 		csstart = skb_headlen(skb);
3999 
4000 	if (unlikely(csstart < 0 || csstart > skb_headlen(skb) ||
4001 		     (skb->ip_summed == CHECKSUM_PARTIAL &&
4002 		      csstart + skb->csum_offset + sizeof(__sum16) >
4003 		      skb->len))) {
4004 		DEBUG_NET_WARN_ON_ONCE(1);
4005 		/* Send the frame without the checksum filled in, or send
4006 		 * zeroes if we can't even copy it.
4007 		 */
4008 		if (skb_copy_bits(skb, 0, to, skb->len))
4009 			memset(to, 0, skb->len);
4010 		return;
4011 	}
4012 
4013 	skb_copy_from_linear_data(skb, to, csstart);
4014 
4015 	csum = 0;
4016 	if (csstart != skb->len)
4017 		csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
4018 					      skb->len - csstart);
4019 
4020 	if (skb->ip_summed == CHECKSUM_PARTIAL) {
4021 		long csstuff = csstart + skb->csum_offset;
4022 
4023 		*((__sum16 *)(to + csstuff)) = csum_fold(csum);
4024 	}
4025 }
4026 EXPORT_SYMBOL(skb_copy_and_csum_dev);
4027 
4028 /**
4029  *	skb_dequeue - remove from the head of the queue
4030  *	@list: list to dequeue from
4031  *
4032  *	Remove the head of the list. The list lock is taken so the function
4033  *	may be used safely with other locking list functions. The head item is
4034  *	returned or %NULL if the list is empty.
4035  */
4036 
4037 struct sk_buff *skb_dequeue(struct sk_buff_head *list)
4038 {
4039 	unsigned long flags;
4040 	struct sk_buff *result;
4041 
4042 	spin_lock_irqsave(&list->lock, flags);
4043 	result = __skb_dequeue(list);
4044 	spin_unlock_irqrestore(&list->lock, flags);
4045 	return result;
4046 }
4047 EXPORT_SYMBOL(skb_dequeue);
4048 
4049 /**
4050  *	skb_dequeue_tail - remove from the tail of the queue
4051  *	@list: list to dequeue from
4052  *
4053  *	Remove the tail of the list. The list lock is taken so the function
4054  *	may be used safely with other locking list functions. The tail item is
4055  *	returned or %NULL if the list is empty.
4056  */
4057 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
4058 {
4059 	unsigned long flags;
4060 	struct sk_buff *result;
4061 
4062 	spin_lock_irqsave(&list->lock, flags);
4063 	result = __skb_dequeue_tail(list);
4064 	spin_unlock_irqrestore(&list->lock, flags);
4065 	return result;
4066 }
4067 EXPORT_SYMBOL(skb_dequeue_tail);
4068 
4069 /**
4070  *	skb_queue_purge_reason - empty a list
4071  *	@list: list to empty
4072  *	@reason: drop reason
4073  *
4074  *	Delete all buffers on an &sk_buff list. Each buffer is removed from
4075  *	the list and one reference dropped. This function takes the list
4076  *	lock and is atomic with respect to other list locking functions.
4077  */
4078 void skb_queue_purge_reason(struct sk_buff_head *list,
4079 			    enum skb_drop_reason reason)
4080 {
4081 	struct sk_buff_head tmp;
4082 	unsigned long flags;
4083 
4084 	if (skb_queue_empty_lockless(list))
4085 		return;
4086 
4087 	__skb_queue_head_init(&tmp);
4088 
4089 	spin_lock_irqsave(&list->lock, flags);
4090 	skb_queue_splice_init(list, &tmp);
4091 	spin_unlock_irqrestore(&list->lock, flags);
4092 
4093 	__skb_queue_purge_reason(&tmp, reason);
4094 }
4095 EXPORT_SYMBOL(skb_queue_purge_reason);
4096 
4097 /**
4098  *	skb_rbtree_purge - empty a skb rbtree
4099  *	@root: root of the rbtree to empty
4100  *	Return value: the sum of truesizes of all purged skbs.
4101  *
4102  *	Delete all buffers on an &sk_buff rbtree. Each buffer is removed from
4103  *	the list and one reference dropped. This function does not take
4104  *	any lock. Synchronization should be handled by the caller (e.g., TCP
4105  *	out-of-order queue is protected by the socket lock).
4106  */
4107 unsigned int skb_rbtree_purge(struct rb_root *root)
4108 {
4109 	struct rb_node *p = rb_first(root);
4110 	unsigned int sum = 0;
4111 
4112 	while (p) {
4113 		struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode);
4114 
4115 		p = rb_next(p);
4116 		rb_erase(&skb->rbnode, root);
4117 		sum += skb->truesize;
4118 		kfree_skb(skb);
4119 	}
4120 	return sum;
4121 }
4122 
4123 void skb_errqueue_purge(struct sk_buff_head *list)
4124 {
4125 	struct sk_buff *skb, *next;
4126 	struct sk_buff_head kill;
4127 	unsigned long flags;
4128 
4129 	__skb_queue_head_init(&kill);
4130 
4131 	spin_lock_irqsave(&list->lock, flags);
4132 	skb_queue_walk_safe(list, skb, next) {
4133 		if (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ZEROCOPY ||
4134 		    SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_TIMESTAMPING)
4135 			continue;
4136 		__skb_unlink(skb, list);
4137 		__skb_queue_tail(&kill, skb);
4138 	}
4139 	spin_unlock_irqrestore(&list->lock, flags);
4140 	__skb_queue_purge(&kill);
4141 }
4142 EXPORT_SYMBOL(skb_errqueue_purge);
4143 
4144 /**
4145  *	skb_queue_head - queue a buffer at the list head
4146  *	@list: list to use
4147  *	@newsk: buffer to queue
4148  *
4149  *	Queue a buffer at the start of the list. This function takes the
4150  *	list lock and can be used safely with other locking &sk_buff functions
4151  *	safely.
4152  *
4153  *	A buffer cannot be placed on two lists at the same time.
4154  */
4155 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
4156 {
4157 	unsigned long flags;
4158 
4159 	spin_lock_irqsave(&list->lock, flags);
4160 	__skb_queue_head(list, newsk);
4161 	spin_unlock_irqrestore(&list->lock, flags);
4162 }
4163 EXPORT_SYMBOL(skb_queue_head);
4164 
4165 /**
4166  *	skb_queue_tail - queue a buffer at the list tail
4167  *	@list: list to use
4168  *	@newsk: buffer to queue
4169  *
4170  *	Queue a buffer at the tail of the list. This function takes the
4171  *	list lock and can be used safely with other locking &sk_buff functions
4172  *	safely.
4173  *
4174  *	A buffer cannot be placed on two lists at the same time.
4175  */
4176 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
4177 {
4178 	unsigned long flags;
4179 
4180 	spin_lock_irqsave(&list->lock, flags);
4181 	__skb_queue_tail(list, newsk);
4182 	spin_unlock_irqrestore(&list->lock, flags);
4183 }
4184 EXPORT_SYMBOL(skb_queue_tail);
4185 
4186 /**
4187  *	skb_unlink	-	remove a buffer from a list
4188  *	@skb: buffer to remove
4189  *	@list: list to use
4190  *
4191  *	Remove a packet from a list. The list locks are taken and this
4192  *	function is atomic with respect to other list locked calls
4193  *
4194  *	You must know what list the SKB is on.
4195  */
4196 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
4197 {
4198 	unsigned long flags;
4199 
4200 	spin_lock_irqsave(&list->lock, flags);
4201 	__skb_unlink(skb, list);
4202 	spin_unlock_irqrestore(&list->lock, flags);
4203 }
4204 EXPORT_SYMBOL(skb_unlink);
4205 
4206 /**
4207  *	skb_append	-	append a buffer
4208  *	@old: buffer to insert after
4209  *	@newsk: buffer to insert
4210  *	@list: list to use
4211  *
4212  *	Place a packet after a given packet in a list. The list locks are taken
4213  *	and this function is atomic with respect to other list locked calls.
4214  *	A buffer cannot be placed on two lists at the same time.
4215  */
4216 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
4217 {
4218 	unsigned long flags;
4219 
4220 	spin_lock_irqsave(&list->lock, flags);
4221 	__skb_queue_after(list, old, newsk);
4222 	spin_unlock_irqrestore(&list->lock, flags);
4223 }
4224 EXPORT_SYMBOL(skb_append);
4225 
4226 static inline void skb_split_inside_header(struct sk_buff *skb,
4227 					   struct sk_buff* skb1,
4228 					   const u32 len, const int pos)
4229 {
4230 	int i;
4231 
4232 	skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
4233 					 pos - len);
4234 	/* And move data appendix as is. */
4235 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
4236 		skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
4237 
4238 	skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
4239 	skb1->unreadable	   = skb->unreadable;
4240 	skb_shinfo(skb)->nr_frags  = 0;
4241 	skb1->data_len		   = skb->data_len;
4242 	skb1->len		   += skb1->data_len;
4243 	skb->data_len		   = 0;
4244 	skb->len		   = len;
4245 	skb_set_tail_pointer(skb, len);
4246 }
4247 
4248 static inline void skb_split_no_header(struct sk_buff *skb,
4249 				       struct sk_buff* skb1,
4250 				       const u32 len, int pos)
4251 {
4252 	int i, k = 0;
4253 	const int nfrags = skb_shinfo(skb)->nr_frags;
4254 
4255 	skb_shinfo(skb)->nr_frags = 0;
4256 	skb1->len		  = skb1->data_len = skb->len - len;
4257 	skb->len		  = len;
4258 	skb->data_len		  = len - pos;
4259 
4260 	for (i = 0; i < nfrags; i++) {
4261 		int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
4262 
4263 		if (pos + size > len) {
4264 			skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
4265 
4266 			if (pos < len) {
4267 				/* Split frag.
4268 				 * We have two variants in this case:
4269 				 * 1. Move all the frag to the second
4270 				 *    part, if it is possible. F.e.
4271 				 *    this approach is mandatory for TUX,
4272 				 *    where splitting is expensive.
4273 				 * 2. Split is accurately. We make this.
4274 				 */
4275 				skb_frag_ref(skb1, k);
4276 				skb_frag_off_add(&skb_shinfo(skb1)->frags[0], len - pos);
4277 				skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
4278 				skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
4279 				skb_shinfo(skb)->nr_frags++;
4280 			}
4281 			k++;
4282 		} else
4283 			skb_shinfo(skb)->nr_frags++;
4284 		pos += size;
4285 	}
4286 	skb_shinfo(skb1)->nr_frags = k;
4287 
4288 	skb1->unreadable = skb->unreadable;
4289 }
4290 
4291 /**
4292  * skb_split - Split fragmented skb to two parts at length len.
4293  * @skb: the buffer to split
4294  * @skb1: the buffer to receive the second part
4295  * @len: new length for skb
4296  */
4297 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
4298 {
4299 	int pos = skb_headlen(skb);
4300 	const int zc_flags = SKBFL_SHARED_FRAG | SKBFL_PURE_ZEROCOPY;
4301 
4302 	skb_zcopy_downgrade_managed(skb);
4303 
4304 	skb_shinfo(skb1)->flags |= skb_shinfo(skb)->flags & zc_flags;
4305 	skb_zerocopy_clone(skb1, skb, 0);
4306 	skb1->pp_recycle = skb->pp_recycle;
4307 	if (len < pos)	/* Split line is inside header. */
4308 		skb_split_inside_header(skb, skb1, len, pos);
4309 	else		/* Second chunk has no header, nothing to copy. */
4310 		skb_split_no_header(skb, skb1, len, pos);
4311 }
4312 EXPORT_SYMBOL(skb_split);
4313 
4314 /* Shifting from/to a cloned skb is a no-go.
4315  *
4316  * Caller cannot keep skb_shinfo related pointers past calling here!
4317  */
4318 static int skb_prepare_for_shift(struct sk_buff *skb)
4319 {
4320 	return skb_unclone_keeptruesize(skb, GFP_ATOMIC);
4321 }
4322 
4323 /**
4324  * skb_shift - Shifts paged data partially from skb to another
4325  * @tgt: buffer into which tail data gets added
4326  * @skb: buffer from which the paged data comes from
4327  * @shiftlen: shift up to this many bytes
4328  *
4329  * Attempts to shift up to shiftlen worth of bytes, which may be less than
4330  * the length of the skb, from skb to tgt. Returns number bytes shifted.
4331  * It's up to caller to free skb if everything was shifted.
4332  *
4333  * If @tgt runs out of frags, the whole operation is aborted.
4334  *
4335  * Skb cannot include anything else but paged data while tgt is allowed
4336  * to have non-paged data as well.
4337  *
4338  * TODO: full sized shift could be optimized but that would need
4339  * specialized skb free'er to handle frags without up-to-date nr_frags.
4340  */
4341 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
4342 {
4343 	int from, to, merge, todo;
4344 	skb_frag_t *fragfrom, *fragto;
4345 
4346 	if (unlikely(shiftlen > skb->len)) {
4347 		DEBUG_NET_WARN_ON_ONCE(1);
4348 		return 0;
4349 	}
4350 
4351 	if (skb_headlen(skb))
4352 		return 0;
4353 	if (skb_zcopy(tgt) || skb_zcopy(skb))
4354 		return 0;
4355 
4356 	DEBUG_NET_WARN_ON_ONCE(tgt->pp_recycle != skb->pp_recycle);
4357 	DEBUG_NET_WARN_ON_ONCE(skb_cmp_decrypted(tgt, skb));
4358 
4359 	todo = shiftlen;
4360 	from = 0;
4361 	to = skb_shinfo(tgt)->nr_frags;
4362 	fragfrom = &skb_shinfo(skb)->frags[from];
4363 
4364 	/* Actual merge is delayed until the point when we know we can
4365 	 * commit all, so that we don't have to undo partial changes
4366 	 */
4367 	if (!skb_can_coalesce_netmem(tgt, to, skb_frag_netmem(fragfrom),
4368 				     skb_frag_off(fragfrom))) {
4369 		merge = -1;
4370 	} else {
4371 		merge = to - 1;
4372 
4373 		todo -= skb_frag_size(fragfrom);
4374 		if (todo < 0) {
4375 			if (skb_prepare_for_shift(skb) ||
4376 			    skb_prepare_for_shift(tgt))
4377 				return 0;
4378 
4379 			/* All previous frag pointers might be stale! */
4380 			fragfrom = &skb_shinfo(skb)->frags[from];
4381 			fragto = &skb_shinfo(tgt)->frags[merge];
4382 
4383 			skb_frag_size_add(fragto, shiftlen);
4384 			skb_frag_size_sub(fragfrom, shiftlen);
4385 			skb_frag_off_add(fragfrom, shiftlen);
4386 
4387 			goto onlymerged;
4388 		}
4389 
4390 		from++;
4391 	}
4392 
4393 	/* Skip full, not-fitting skb to avoid expensive operations */
4394 	if ((shiftlen == skb->len) &&
4395 	    (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
4396 		return 0;
4397 
4398 	if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
4399 		return 0;
4400 
4401 	while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
4402 		if (to == MAX_SKB_FRAGS)
4403 			return 0;
4404 
4405 		fragfrom = &skb_shinfo(skb)->frags[from];
4406 		fragto = &skb_shinfo(tgt)->frags[to];
4407 
4408 		if (todo >= skb_frag_size(fragfrom)) {
4409 			*fragto = *fragfrom;
4410 			todo -= skb_frag_size(fragfrom);
4411 			from++;
4412 			to++;
4413 
4414 		} else {
4415 			skb_frag_page_copy(fragto, fragfrom);
4416 			skb_frag_ref(tgt, to);
4417 			skb_frag_off_copy(fragto, fragfrom);
4418 			skb_frag_size_set(fragto, todo);
4419 
4420 			skb_frag_off_add(fragfrom, todo);
4421 			skb_frag_size_sub(fragfrom, todo);
4422 			todo = 0;
4423 
4424 			to++;
4425 			break;
4426 		}
4427 	}
4428 
4429 	/* The frags ran out before shiftlen did.  Nothing has been committed
4430 	 * yet, so back out.
4431 	 */
4432 	if (unlikely(todo > 0)) {
4433 		DEBUG_NET_WARN_ON_ONCE(1);
4434 		return 0;
4435 	}
4436 
4437 	/* Ready to "commit" this state change to tgt */
4438 	skb_shinfo(tgt)->nr_frags = to;
4439 
4440 	if (merge >= 0) {
4441 		fragfrom = &skb_shinfo(skb)->frags[0];
4442 		fragto = &skb_shinfo(tgt)->frags[merge];
4443 
4444 		skb_frag_size_add(fragto, skb_frag_size(fragfrom));
4445 		skb_frag_unref(skb, 0);
4446 	}
4447 
4448 	/* Reposition in the original skb */
4449 	to = 0;
4450 	while (from < skb_shinfo(skb)->nr_frags)
4451 		skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
4452 	skb_shinfo(skb)->nr_frags = to;
4453 
4454 onlymerged:
4455 	/* Most likely the tgt won't ever need its checksum anymore, skb on
4456 	 * the other hand might need it if it needs to be resent
4457 	 */
4458 	tgt->ip_summed = CHECKSUM_PARTIAL;
4459 	skb->ip_summed = CHECKSUM_PARTIAL;
4460 
4461 	skb_shinfo(tgt)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG;
4462 
4463 	skb_len_add(skb, -shiftlen);
4464 	skb_len_add(tgt, shiftlen);
4465 
4466 	return shiftlen;
4467 }
4468 EXPORT_SYMBOL_IF_KUNIT(skb_shift);
4469 
4470 /**
4471  * skb_prepare_seq_read - Prepare a sequential read of skb data
4472  * @skb: the buffer to read
4473  * @from: lower offset of data to be read
4474  * @to: upper offset of data to be read
4475  * @st: state variable
4476  *
4477  * Initializes the specified state variable. Must be called before
4478  * invoking skb_seq_read() for the first time.
4479  */
4480 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
4481 			  unsigned int to, struct skb_seq_state *st)
4482 {
4483 	st->lower_offset = from;
4484 	st->upper_offset = to;
4485 	st->root_skb = st->cur_skb = skb;
4486 	st->frag_idx = st->stepped_offset = 0;
4487 	st->frag_data = NULL;
4488 	st->frag_off = 0;
4489 }
4490 EXPORT_SYMBOL(skb_prepare_seq_read);
4491 
4492 /**
4493  * skb_seq_read - Sequentially read skb data
4494  * @consumed: number of bytes consumed by the caller so far
4495  * @data: destination pointer for data to be returned
4496  * @st: state variable
4497  *
4498  * Reads a block of skb data at @consumed relative to the
4499  * lower offset specified to skb_prepare_seq_read(). Assigns
4500  * the head of the data block to @data and returns the length
4501  * of the block or 0 if the end of the skb data or the upper
4502  * offset has been reached.
4503  *
4504  * The caller is not required to consume all of the data
4505  * returned, i.e. @consumed is typically set to the number
4506  * of bytes already consumed and the next call to
4507  * skb_seq_read() will return the remaining part of the block.
4508  *
4509  * Note 1: The size of each block of data returned can be arbitrary,
4510  *       this limitation is the cost for zerocopy sequential
4511  *       reads of potentially non linear data.
4512  *
4513  * Note 2: Fragment lists within fragments are not implemented
4514  *       at the moment, state->root_skb could be replaced with
4515  *       a stack for this purpose.
4516  */
4517 unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
4518 			  struct skb_seq_state *st)
4519 {
4520 	unsigned int block_limit, abs_offset = consumed + st->lower_offset;
4521 	skb_frag_t *frag;
4522 
4523 	if (unlikely(abs_offset >= st->upper_offset)) {
4524 		if (st->frag_data) {
4525 			kunmap_atomic(st->frag_data);
4526 			st->frag_data = NULL;
4527 		}
4528 		return 0;
4529 	}
4530 
4531 next_skb:
4532 	block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
4533 
4534 	if (abs_offset < block_limit && !st->frag_data) {
4535 		*data = st->cur_skb->data + (abs_offset - st->stepped_offset);
4536 		return block_limit - abs_offset;
4537 	}
4538 
4539 	if (!skb_frags_readable(st->cur_skb))
4540 		return 0;
4541 
4542 	if (st->frag_idx == 0 && !st->frag_data)
4543 		st->stepped_offset += skb_headlen(st->cur_skb);
4544 
4545 	while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
4546 		unsigned int pg_idx, pg_off, pg_sz;
4547 
4548 		frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
4549 
4550 		pg_idx = 0;
4551 		pg_off = skb_frag_off(frag);
4552 		pg_sz = skb_frag_size(frag);
4553 
4554 		if (skb_frag_must_loop(skb_frag_page(frag))) {
4555 			pg_idx = (pg_off + st->frag_off) >> PAGE_SHIFT;
4556 			pg_off = offset_in_page(pg_off + st->frag_off);
4557 			pg_sz = min_t(unsigned int, pg_sz - st->frag_off,
4558 						    PAGE_SIZE - pg_off);
4559 		}
4560 
4561 		block_limit = pg_sz + st->stepped_offset;
4562 		if (abs_offset < block_limit) {
4563 			if (!st->frag_data)
4564 				st->frag_data = kmap_atomic(skb_frag_page(frag) + pg_idx);
4565 
4566 			*data = (u8 *)st->frag_data + pg_off +
4567 				(abs_offset - st->stepped_offset);
4568 
4569 			return block_limit - abs_offset;
4570 		}
4571 
4572 		if (st->frag_data) {
4573 			kunmap_atomic(st->frag_data);
4574 			st->frag_data = NULL;
4575 		}
4576 
4577 		st->stepped_offset += pg_sz;
4578 		st->frag_off += pg_sz;
4579 		if (st->frag_off == skb_frag_size(frag)) {
4580 			st->frag_off = 0;
4581 			st->frag_idx++;
4582 		}
4583 	}
4584 
4585 	if (st->frag_data) {
4586 		kunmap_atomic(st->frag_data);
4587 		st->frag_data = NULL;
4588 	}
4589 
4590 	if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
4591 		st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
4592 		st->frag_idx = 0;
4593 		goto next_skb;
4594 	} else if (st->cur_skb->next) {
4595 		st->cur_skb = st->cur_skb->next;
4596 		st->frag_idx = 0;
4597 		goto next_skb;
4598 	}
4599 
4600 	return 0;
4601 }
4602 EXPORT_SYMBOL(skb_seq_read);
4603 
4604 /**
4605  * skb_abort_seq_read - Abort a sequential read of skb data
4606  * @st: state variable
4607  *
4608  * Must be called if skb_seq_read() was not called until it
4609  * returned 0.
4610  */
4611 void skb_abort_seq_read(struct skb_seq_state *st)
4612 {
4613 	if (st->frag_data)
4614 		kunmap_atomic(st->frag_data);
4615 }
4616 EXPORT_SYMBOL(skb_abort_seq_read);
4617 
4618 /**
4619  * skb_copy_seq_read() - copy from a skb_seq_state to a buffer
4620  * @st: source skb_seq_state
4621  * @offset: offset in source
4622  * @to: destination buffer
4623  * @len: number of bytes to copy
4624  *
4625  * Copy @len bytes from @offset bytes into the source @st to the destination
4626  * buffer @to. `offset` should increase (or be unchanged) with each subsequent
4627  * call to this function. If offset needs to decrease from the previous use `st`
4628  * should be reset first.
4629  *
4630  * Return: 0 on success or -EINVAL if the copy ended early
4631  */
4632 int skb_copy_seq_read(struct skb_seq_state *st, int offset, void *to, int len)
4633 {
4634 	const u8 *data;
4635 	u32 sqlen;
4636 
4637 	for (;;) {
4638 		sqlen = skb_seq_read(offset, &data, st);
4639 		if (sqlen == 0)
4640 			return -EINVAL;
4641 		if (sqlen >= len) {
4642 			memcpy(to, data, len);
4643 			return 0;
4644 		}
4645 		memcpy(to, data, sqlen);
4646 		to += sqlen;
4647 		offset += sqlen;
4648 		len -= sqlen;
4649 	}
4650 }
4651 EXPORT_SYMBOL(skb_copy_seq_read);
4652 
4653 #define TS_SKB_CB(state)	((struct skb_seq_state *) &((state)->cb))
4654 
4655 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
4656 					  struct ts_config *conf,
4657 					  struct ts_state *state)
4658 {
4659 	return skb_seq_read(offset, text, TS_SKB_CB(state));
4660 }
4661 
4662 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
4663 {
4664 	skb_abort_seq_read(TS_SKB_CB(state));
4665 }
4666 
4667 /**
4668  * skb_find_text - Find a text pattern in skb data
4669  * @skb: the buffer to look in
4670  * @from: search offset
4671  * @to: search limit
4672  * @config: textsearch configuration
4673  *
4674  * Finds a pattern in the skb data according to the specified
4675  * textsearch configuration. Use textsearch_next() to retrieve
4676  * subsequent occurrences of the pattern. Returns the offset
4677  * to the first occurrence or UINT_MAX if no match was found.
4678  */
4679 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
4680 			   unsigned int to, struct ts_config *config)
4681 {
4682 	unsigned int patlen = config->ops->get_pattern_len(config);
4683 	struct ts_state state;
4684 	unsigned int ret;
4685 
4686 	BUILD_BUG_ON(sizeof(struct skb_seq_state) > sizeof(state.cb));
4687 
4688 	config->get_next_block = skb_ts_get_next_block;
4689 	config->finish = skb_ts_finish;
4690 
4691 	skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state));
4692 
4693 	ret = textsearch_find(config, &state);
4694 	return (ret + patlen <= to - from ? ret : UINT_MAX);
4695 }
4696 EXPORT_SYMBOL(skb_find_text);
4697 
4698 int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
4699 			 int offset, size_t size, size_t max_frags)
4700 {
4701 	int i = skb_shinfo(skb)->nr_frags;
4702 
4703 	if (skb_can_coalesce(skb, i, page, offset)) {
4704 		skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size);
4705 	} else if (i < max_frags) {
4706 		skb_zcopy_downgrade_managed(skb);
4707 		get_page(page);
4708 		skb_fill_page_desc_noacc(skb, i, page, offset, size);
4709 	} else {
4710 		return -EMSGSIZE;
4711 	}
4712 
4713 	return 0;
4714 }
4715 EXPORT_SYMBOL_GPL(skb_append_pagefrags);
4716 
4717 /**
4718  *	skb_pull_rcsum - pull skb and update receive checksum
4719  *	@skb: buffer to update
4720  *	@len: length of data pulled
4721  *
4722  *	This function performs an skb_pull on the packet and updates
4723  *	the CHECKSUM_COMPLETE checksum.  It should be used on
4724  *	receive path processing instead of skb_pull unless you know
4725  *	that the checksum difference is zero (e.g., a valid IP header)
4726  *	or you are setting ip_summed to CHECKSUM_NONE.
4727  */
4728 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
4729 {
4730 	unsigned char *data = skb->data;
4731 
4732 	BUG_ON(len > skb->len);
4733 	__skb_pull(skb, len);
4734 	skb_postpull_rcsum(skb, data, len);
4735 	return skb->data;
4736 }
4737 EXPORT_SYMBOL_GPL(skb_pull_rcsum);
4738 
4739 static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb)
4740 {
4741 	skb_frag_t head_frag;
4742 	struct page *page;
4743 
4744 	page = virt_to_head_page(frag_skb->head);
4745 	skb_frag_fill_page_desc(&head_frag, page, frag_skb->data -
4746 				(unsigned char *)page_address(page),
4747 				skb_headlen(frag_skb));
4748 	return head_frag;
4749 }
4750 
4751 struct sk_buff *skb_segment_list(struct sk_buff *skb,
4752 				 netdev_features_t features,
4753 				 unsigned int offset)
4754 {
4755 	struct sk_buff *list_skb = skb_shinfo(skb)->frag_list;
4756 	unsigned int tnl_hlen = skb_tnl_header_len(skb);
4757 	unsigned int delta_len = 0;
4758 	struct sk_buff *tail = NULL;
4759 	struct sk_buff *nskb, *tmp;
4760 	int len_diff, err;
4761 
4762 	/* Only skb_gro_receive_list generated skbs arrive here */
4763 	DEBUG_NET_WARN_ON_ONCE(!(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST));
4764 
4765 	skb_push(skb, -skb_network_offset(skb) + offset);
4766 
4767 	/* Ensure the head is writeable before touching the shared info */
4768 	err = skb_unclone(skb, GFP_ATOMIC);
4769 	if (err)
4770 		goto err_linearize;
4771 
4772 	skb_shinfo(skb)->frag_list = NULL;
4773 
4774 	while (list_skb) {
4775 		nskb = list_skb;
4776 		list_skb = list_skb->next;
4777 
4778 		DEBUG_NET_WARN_ON_ONCE(nskb->sk);
4779 
4780 		err = 0;
4781 		if (skb_shared(nskb)) {
4782 			tmp = skb_clone(nskb, GFP_ATOMIC);
4783 			if (tmp) {
4784 				consume_skb(nskb);
4785 				nskb = tmp;
4786 				err = skb_unclone(nskb, GFP_ATOMIC);
4787 			} else {
4788 				err = -ENOMEM;
4789 			}
4790 		}
4791 
4792 		if (!tail)
4793 			skb->next = nskb;
4794 		else
4795 			tail->next = nskb;
4796 
4797 		/* nskb may be a clone with a NULL ->next: keep the remaining
4798 		 * members linked so that err_linearize frees them.
4799 		 */
4800 		nskb->next = list_skb;
4801 
4802 		if (unlikely(err))
4803 			goto err_linearize;
4804 
4805 		tail = nskb;
4806 
4807 		delta_len += nskb->len;
4808 
4809 		skb_push(nskb, -skb_network_offset(nskb) + offset);
4810 
4811 		skb_release_head_state(nskb);
4812 		len_diff = skb_network_header_len(nskb) - skb_network_header_len(skb);
4813 		__copy_skb_header(nskb, skb);
4814 
4815 		skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb));
4816 		nskb->transport_header += len_diff;
4817 		skb_copy_from_linear_data_offset(skb, -tnl_hlen,
4818 						 nskb->data - tnl_hlen,
4819 						 offset + tnl_hlen);
4820 
4821 		if (skb_needs_linearize(nskb, features) &&
4822 		    __skb_linearize(nskb))
4823 			goto err_linearize;
4824 	}
4825 
4826 	skb->data_len = skb->data_len - delta_len;
4827 	skb->len = skb->len - delta_len;
4828 
4829 	skb_gso_reset(skb);
4830 
4831 	skb->prev = tail;
4832 
4833 	if (skb_needs_linearize(skb, features) &&
4834 	    __skb_linearize(skb))
4835 		goto err_linearize;
4836 
4837 	skb_get(skb);
4838 
4839 	return skb;
4840 
4841 err_linearize:
4842 	kfree_skb_list(skb->next);
4843 	skb->next = NULL;
4844 	return ERR_PTR(-ENOMEM);
4845 }
4846 EXPORT_SYMBOL_GPL(skb_segment_list);
4847 
4848 /**
4849  *	skb_segment - Perform protocol segmentation on skb.
4850  *	@head_skb: buffer to segment
4851  *	@features: features for the output path (see dev->features)
4852  *
4853  *	This function performs segmentation on the given skb.  It returns
4854  *	a pointer to the first in a list of new skbs for the segments.
4855  *	In case of error it returns ERR_PTR(err).
4856  */
4857 struct sk_buff *skb_segment(struct sk_buff *head_skb,
4858 			    netdev_features_t features)
4859 {
4860 	struct sk_buff *segs = NULL;
4861 	struct sk_buff *tail = NULL;
4862 	struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list;
4863 	unsigned int max_segs = SKB_GSO_CB(head_skb)->max_segs;
4864 	unsigned int mss = skb_shinfo(head_skb)->gso_size;
4865 	bool gso_by_frags = mss == GSO_BY_FRAGS;
4866 	unsigned int doffset = head_skb->data - skb_mac_header(head_skb);
4867 	unsigned int offset = doffset;
4868 	unsigned int tnl_hlen = skb_tnl_header_len(head_skb);
4869 	unsigned int partial_segs = 0;
4870 	unsigned int headroom;
4871 	unsigned int len = head_skb->len;
4872 	struct sk_buff *frag_skb;
4873 	skb_frag_t *frag;
4874 	__be16 proto;
4875 	bool csum, sg;
4876 	int err = -ENOMEM;
4877 	int i = 0;
4878 	int nfrags, pos;
4879 
4880 	if ((skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY) &&
4881 	    !gso_by_frags && mss != skb_headlen(head_skb)) {
4882 		struct sk_buff *check_skb;
4883 
4884 		for (check_skb = list_skb; check_skb; check_skb = check_skb->next) {
4885 			if (skb_headlen(check_skb) && !check_skb->head_frag) {
4886 				/* gso_size is untrusted, and we have a frag_list with
4887 				 * a linear non head_frag item.
4888 				 *
4889 				 * If head_skb's headlen does not fit requested gso_size,
4890 				 * it means that the frag_list members do NOT terminate
4891 				 * on exact gso_size boundaries. Hence we cannot perform
4892 				 * skb_frag_t page sharing. Therefore we must fallback to
4893 				 * copying the frag_list skbs; we do so by disabling SG.
4894 				 */
4895 				features &= ~NETIF_F_SG;
4896 				break;
4897 			}
4898 		}
4899 	}
4900 
4901 	__skb_push(head_skb, doffset);
4902 	proto = skb_network_protocol(head_skb, NULL);
4903 	if (unlikely(!proto))
4904 		return ERR_PTR(-EINVAL);
4905 
4906 	sg = !!(features & NETIF_F_SG);
4907 	csum = !!can_checksum_protocol(features, proto);
4908 
4909 	if (sg && csum && !gso_by_frags)  {
4910 		if (!max_segs && !(features & NETIF_F_GSO_PARTIAL)) {
4911 			struct sk_buff *iter;
4912 			unsigned int frag_len;
4913 
4914 			if (!list_skb ||
4915 			    !net_gso_ok(features, skb_shinfo(head_skb)->gso_type))
4916 				goto normal;
4917 
4918 			/* If we get here then all the required
4919 			 * GSO features except frag_list are supported.
4920 			 * Try to split the SKB to multiple GSO SKBs
4921 			 * with no frag_list.
4922 			 * Currently we can do that only when the buffers don't
4923 			 * have a linear part and all the buffers except
4924 			 * the last are of the same length.
4925 			 */
4926 			frag_len = list_skb->len;
4927 			skb_walk_frags(head_skb, iter) {
4928 				if (frag_len != iter->len && iter->next)
4929 					goto normal;
4930 				if (skb_headlen(iter) && !iter->head_frag)
4931 					goto normal;
4932 
4933 				len -= iter->len;
4934 			}
4935 
4936 			if (len != frag_len)
4937 				goto normal;
4938 		}
4939 
4940 		/* GSO partial only requires that we trim off any excess that
4941 		 * doesn't fit into an MSS sized block, so take care of that
4942 		 * now.
4943 		 */
4944 		DEBUG_NET_WARN_ON_ONCE(!max_segs && len / mss > GSO_MAX_SEGS);
4945 		if (max_segs)
4946 			partial_segs = min(len / mss, max_segs);
4947 		else
4948 			partial_segs = min(len / mss, GSO_MAX_SEGS);
4949 		if (partial_segs > 1)
4950 			mss *= partial_segs;
4951 		else
4952 			partial_segs = 0;
4953 	}
4954 
4955 normal:
4956 	headroom = skb_headroom(head_skb);
4957 	pos = skb_headlen(head_skb);
4958 
4959 	if (skb_orphan_frags(head_skb, GFP_ATOMIC))
4960 		return ERR_PTR(-ENOMEM);
4961 
4962 	nfrags = skb_shinfo(head_skb)->nr_frags;
4963 	frag = skb_shinfo(head_skb)->frags;
4964 	frag_skb = head_skb;
4965 
4966 	do {
4967 		struct sk_buff *nskb;
4968 		skb_frag_t *nskb_frag;
4969 		int hsize;
4970 		int size;
4971 
4972 		if (unlikely(gso_by_frags)) {
4973 			len = list_skb->len;
4974 		} else {
4975 			len = head_skb->len - offset;
4976 			if (len > mss)
4977 				len = mss;
4978 		}
4979 
4980 		hsize = skb_headlen(head_skb) - offset;
4981 
4982 		if (hsize <= 0 && i >= nfrags && skb_headlen(list_skb) &&
4983 		    (skb_headlen(list_skb) == len || sg)) {
4984 			if (unlikely(skb_headlen(list_skb) > len)) {
4985 				DEBUG_NET_WARN_ON_ONCE(1);
4986 				err = -EINVAL;
4987 				goto err;
4988 			}
4989 
4990 			nskb = skb_clone(list_skb, GFP_ATOMIC);
4991 			if (unlikely(!nskb))
4992 				goto err;
4993 
4994 			i = 0;
4995 			nfrags = skb_shinfo(list_skb)->nr_frags;
4996 			frag = skb_shinfo(list_skb)->frags;
4997 			frag_skb = list_skb;
4998 			pos += skb_headlen(list_skb);
4999 
5000 			while (pos < offset + len) {
5001 				if (unlikely(i >= nfrags)) {
5002 					DEBUG_NET_WARN_ON_ONCE(1);
5003 					kfree_skb(nskb);
5004 					err = -EINVAL;
5005 					goto err;
5006 				}
5007 
5008 				size = skb_frag_size(frag);
5009 				if (pos + size > offset + len)
5010 					break;
5011 
5012 				i++;
5013 				pos += size;
5014 				frag++;
5015 			}
5016 
5017 			list_skb = list_skb->next;
5018 
5019 			if (unlikely(pskb_trim(nskb, len))) {
5020 				kfree_skb(nskb);
5021 				goto err;
5022 			}
5023 
5024 			hsize = skb_end_offset(nskb);
5025 			if (skb_cow_head(nskb, doffset + headroom)) {
5026 				kfree_skb(nskb);
5027 				goto err;
5028 			}
5029 
5030 			nskb->truesize += skb_end_offset(nskb) - hsize;
5031 			skb_release_head_state(nskb);
5032 			__skb_push(nskb, doffset);
5033 		} else {
5034 			if (hsize < 0)
5035 				hsize = 0;
5036 			if (hsize > len || !sg)
5037 				hsize = len;
5038 
5039 			nskb = __alloc_skb(hsize + doffset + headroom,
5040 					   GFP_ATOMIC, skb_alloc_rx_flag(head_skb),
5041 					   NUMA_NO_NODE);
5042 
5043 			if (unlikely(!nskb))
5044 				goto err;
5045 
5046 			skb_reserve(nskb, headroom);
5047 			__skb_put(nskb, doffset);
5048 		}
5049 
5050 		if (segs)
5051 			tail->next = nskb;
5052 		else
5053 			segs = nskb;
5054 		tail = nskb;
5055 
5056 		__copy_skb_header(nskb, head_skb);
5057 
5058 		skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom);
5059 		skb_reset_mac_len(nskb);
5060 
5061 		skb_copy_from_linear_data_offset(head_skb, -tnl_hlen,
5062 						 nskb->data - tnl_hlen,
5063 						 doffset + tnl_hlen);
5064 
5065 		if (nskb->len == len + doffset)
5066 			goto perform_csum_check;
5067 
5068 		if (!sg) {
5069 			if (!csum) {
5070 				if (!nskb->remcsum_offload)
5071 					nskb->ip_summed = CHECKSUM_NONE;
5072 				SKB_GSO_CB(nskb)->csum =
5073 					skb_copy_and_csum_bits(head_skb, offset,
5074 							       skb_put(nskb,
5075 								       len),
5076 							       len);
5077 				SKB_GSO_CB(nskb)->csum_start =
5078 					skb_headroom(nskb) + doffset;
5079 			} else {
5080 				if (skb_copy_bits(head_skb, offset, skb_put(nskb, len), len))
5081 					goto err;
5082 			}
5083 			continue;
5084 		}
5085 
5086 		nskb_frag = skb_shinfo(nskb)->frags;
5087 
5088 		skb_copy_from_linear_data_offset(head_skb, offset,
5089 						 skb_put(nskb, hsize), hsize);
5090 
5091 		skb_shinfo(nskb)->flags |= (skb_shinfo(head_skb)->flags |
5092 					    skb_shinfo(frag_skb)->flags) &
5093 					   SKBFL_SHARED_FRAG;
5094 
5095 		if (skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC))
5096 			goto err;
5097 
5098 		while (pos < offset + len) {
5099 			if (i >= nfrags) {
5100 				if (skb_shinfo(nskb)->nr_frags &&
5101 				    nskb->pp_recycle != list_skb->pp_recycle) {
5102 					nskb->data_len = pos - offset - hsize;
5103 					nskb->len += nskb->data_len;
5104 					nskb->truesize += nskb->data_len;
5105 					if (__skb_linearize(nskb))
5106 						goto err;
5107 					hsize = pos - offset;
5108 					nskb_frag = skb_shinfo(nskb)->frags;
5109 				}
5110 				if (skb_orphan_frags(list_skb, GFP_ATOMIC) ||
5111 				    skb_zerocopy_clone(nskb, list_skb,
5112 						       GFP_ATOMIC))
5113 					goto err;
5114 
5115 				i = 0;
5116 				nfrags = skb_shinfo(list_skb)->nr_frags;
5117 				frag = skb_shinfo(list_skb)->frags;
5118 				frag_skb = list_skb;
5119 
5120 				skb_shinfo(nskb)->flags |= skb_shinfo(frag_skb)->flags & SKBFL_SHARED_FRAG;
5121 
5122 				if (!skb_headlen(list_skb)) {
5123 					if (unlikely(!nfrags)) {
5124 						DEBUG_NET_WARN_ON_ONCE(1);
5125 						err = -EINVAL;
5126 						goto err;
5127 					}
5128 				} else {
5129 					if (unlikely(!list_skb->head_frag)) {
5130 						DEBUG_NET_WARN_ON_ONCE(1);
5131 						err = -EINVAL;
5132 						goto err;
5133 					}
5134 
5135 					/* to make room for head_frag. */
5136 					i--;
5137 					frag--;
5138 				}
5139 
5140 				list_skb = list_skb->next;
5141 			}
5142 
5143 			if (unlikely(skb_shinfo(nskb)->nr_frags >=
5144 				     MAX_SKB_FRAGS)) {
5145 				net_warn_ratelimited(
5146 					"skb_segment: too many frags: %u %u\n",
5147 					pos, mss);
5148 				err = -EINVAL;
5149 				goto err;
5150 			}
5151 
5152 			if (!skb_shinfo(nskb)->nr_frags)
5153 				nskb->pp_recycle = frag_skb->pp_recycle;
5154 			*nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag;
5155 			skb_frag_ref(nskb, skb_shinfo(nskb)->nr_frags);
5156 			size = skb_frag_size(nskb_frag);
5157 
5158 			if (pos < offset) {
5159 				skb_frag_off_add(nskb_frag, offset - pos);
5160 				skb_frag_size_sub(nskb_frag, offset - pos);
5161 			}
5162 
5163 			skb_shinfo(nskb)->nr_frags++;
5164 
5165 			if (pos + size <= offset + len) {
5166 				i++;
5167 				frag++;
5168 				pos += size;
5169 			} else {
5170 				skb_frag_size_sub(nskb_frag, pos + size - (offset + len));
5171 				goto skip_fraglist;
5172 			}
5173 
5174 			nskb_frag++;
5175 		}
5176 
5177 skip_fraglist:
5178 		nskb->data_len = len - hsize;
5179 		nskb->len += nskb->data_len;
5180 		nskb->truesize += nskb->data_len;
5181 
5182 perform_csum_check:
5183 		if (!csum) {
5184 			if (skb_has_shared_frag(nskb) &&
5185 			    __skb_linearize(nskb))
5186 				goto err;
5187 
5188 			if (!nskb->remcsum_offload)
5189 				nskb->ip_summed = CHECKSUM_NONE;
5190 			SKB_GSO_CB(nskb)->csum =
5191 				skb_checksum(nskb, doffset,
5192 					     nskb->len - doffset, 0);
5193 			SKB_GSO_CB(nskb)->csum_start =
5194 				skb_headroom(nskb) + doffset;
5195 		}
5196 	} while ((offset += len) < head_skb->len);
5197 
5198 	/* Some callers want to get the end of the list.
5199 	 * Put it in segs->prev to avoid walking the list.
5200 	 * (see validate_xmit_skb_list() for example)
5201 	 */
5202 	segs->prev = tail;
5203 
5204 	if (partial_segs) {
5205 		struct sk_buff *iter;
5206 		int type = skb_shinfo(head_skb)->gso_type;
5207 		unsigned short gso_size = skb_shinfo(head_skb)->gso_size;
5208 
5209 		/* Update type to add partial and then remove dodgy if set */
5210 		type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL;
5211 		type &= ~SKB_GSO_DODGY;
5212 
5213 		/* Update GSO info and prepare to start updating headers on
5214 		 * our way back down the stack of protocols.
5215 		 */
5216 		for (iter = segs; iter; iter = iter->next) {
5217 			skb_shinfo(iter)->gso_size = gso_size;
5218 			skb_shinfo(iter)->gso_segs = partial_segs;
5219 			skb_shinfo(iter)->gso_type = type;
5220 			SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset;
5221 		}
5222 
5223 		if (tail->len - doffset <= gso_size)
5224 			skb_gso_reset(tail);
5225 		else if (tail != segs)
5226 			skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size);
5227 	}
5228 
5229 	/* Following permits correct backpressure, for protocols
5230 	 * using skb_set_owner_w().
5231 	 * Idea is to tranfert ownership from head_skb to last segment.
5232 	 */
5233 	if (head_skb->destructor == sock_wfree) {
5234 		swap(tail->truesize, head_skb->truesize);
5235 		swap(tail->destructor, head_skb->destructor);
5236 		swap(tail->sk, head_skb->sk);
5237 	}
5238 	return segs;
5239 
5240 err:
5241 	kfree_skb_list(segs);
5242 	return ERR_PTR(err);
5243 }
5244 EXPORT_SYMBOL_GPL(skb_segment);
5245 
5246 #ifdef CONFIG_SKB_EXTENSIONS
5247 #define SKB_EXT_ALIGN_VALUE	8
5248 #define SKB_EXT_CHUNKSIZEOF(x)	(ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE)
5249 
5250 static const u8 skb_ext_type_len[] = {
5251 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
5252 	[SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info),
5253 #endif
5254 #ifdef CONFIG_XFRM
5255 	[SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path),
5256 #endif
5257 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
5258 	[TC_SKB_EXT] = SKB_EXT_CHUNKSIZEOF(struct tc_skb_ext),
5259 #endif
5260 #if IS_ENABLED(CONFIG_MPTCP)
5261 	[SKB_EXT_MPTCP] = SKB_EXT_CHUNKSIZEOF(struct mptcp_ext),
5262 #endif
5263 #if IS_ENABLED(CONFIG_MCTP_FLOWS)
5264 	[SKB_EXT_MCTP] = SKB_EXT_CHUNKSIZEOF(struct mctp_flow),
5265 #endif
5266 #if IS_ENABLED(CONFIG_INET_PSP)
5267 	[SKB_EXT_PSP] = SKB_EXT_CHUNKSIZEOF(struct psp_skb_ext),
5268 #endif
5269 #if IS_ENABLED(CONFIG_CAN)
5270 	[SKB_EXT_CAN] = SKB_EXT_CHUNKSIZEOF(struct can_skb_ext),
5271 #endif
5272 };
5273 
5274 static __always_inline __no_profile unsigned int skb_ext_total_length(void)
5275 {
5276 	unsigned int l = SKB_EXT_CHUNKSIZEOF(struct skb_ext);
5277 	int i;
5278 
5279 	for (i = 0; i < ARRAY_SIZE(skb_ext_type_len); i++)
5280 		l += skb_ext_type_len[i];
5281 
5282 	return l;
5283 }
5284 
5285 static noinline void __init __no_profile skb_extensions_init(void)
5286 {
5287 	BUILD_BUG_ON(SKB_EXT_NUM > 8);
5288 	BUILD_BUG_ON(skb_ext_total_length() > 255);
5289 
5290 	skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache",
5291 					     SKB_EXT_ALIGN_VALUE * skb_ext_total_length(),
5292 					     0,
5293 					     SLAB_HWCACHE_ALIGN|SLAB_PANIC,
5294 					     NULL);
5295 }
5296 #else
5297 static void skb_extensions_init(void) {}
5298 #endif
5299 
5300 /* The SKB kmem_cache slab is critical for network performance.  Never
5301  * merge/alias the slab with similar sized objects.  This avoids fragmentation
5302  * that hurts performance of kmem_cache_{alloc,free}_bulk APIs.
5303  */
5304 #ifndef CONFIG_SLUB_TINY
5305 #define FLAG_SKB_NO_MERGE	SLAB_NO_MERGE
5306 #else /* CONFIG_SLUB_TINY - simple loop in kmem_cache_alloc_bulk */
5307 #define FLAG_SKB_NO_MERGE	0
5308 #endif
5309 
5310 void __init skb_init(void)
5311 {
5312 	net_hotdata.skbuff_cache = kmem_cache_create_usercopy("skbuff_head_cache",
5313 					      sizeof(struct sk_buff),
5314 					      0,
5315 					      SLAB_HWCACHE_ALIGN|SLAB_PANIC|
5316 						FLAG_SKB_NO_MERGE,
5317 					      offsetof(struct sk_buff, cb),
5318 					      sizeof_field(struct sk_buff, cb),
5319 					      NULL);
5320 	skbuff_cache_size = kmem_cache_size(net_hotdata.skbuff_cache);
5321 
5322 	net_hotdata.skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
5323 						sizeof(struct sk_buff_fclones),
5324 						0,
5325 						SLAB_HWCACHE_ALIGN|SLAB_PANIC,
5326 						NULL);
5327 	/* usercopy should only access first SKB_SMALL_HEAD_HEADROOM bytes.
5328 	 * struct skb_shared_info is located at the end of skb->head,
5329 	 * and should not be copied to/from user.
5330 	 */
5331 	net_hotdata.skb_small_head_cache = kmem_cache_create_usercopy("skbuff_small_head",
5332 						SKB_SMALL_HEAD_CACHE_SIZE,
5333 						0,
5334 						SLAB_HWCACHE_ALIGN | SLAB_PANIC,
5335 						0,
5336 						SKB_SMALL_HEAD_HEADROOM,
5337 						NULL);
5338 	skb_extensions_init();
5339 }
5340 
5341 static int
5342 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len,
5343 	       unsigned int recursion_level)
5344 {
5345 	int start = skb_headlen(skb);
5346 	int i, copy = start - offset;
5347 	struct sk_buff *frag_iter;
5348 	int elt = 0;
5349 
5350 	if (unlikely(recursion_level >= 24))
5351 		return -EMSGSIZE;
5352 
5353 	if (copy > 0) {
5354 		if (copy > len)
5355 			copy = len;
5356 		sg_set_buf(sg, skb->data + offset, copy);
5357 		elt++;
5358 		if ((len -= copy) == 0)
5359 			return elt;
5360 		offset += copy;
5361 	}
5362 
5363 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
5364 		int end;
5365 
5366 		WARN_ON(start > offset + len);
5367 
5368 		end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
5369 		if ((copy = end - offset) > 0) {
5370 			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
5371 			if (unlikely(elt && sg_is_last(&sg[elt - 1])))
5372 				return -EMSGSIZE;
5373 
5374 			if (copy > len)
5375 				copy = len;
5376 			sg_set_page(&sg[elt], skb_frag_page(frag), copy,
5377 				    skb_frag_off(frag) + offset - start);
5378 			elt++;
5379 			if (!(len -= copy))
5380 				return elt;
5381 			offset += copy;
5382 		}
5383 		start = end;
5384 	}
5385 
5386 	skb_walk_frags(skb, frag_iter) {
5387 		int end, ret;
5388 
5389 		WARN_ON(start > offset + len);
5390 
5391 		end = start + frag_iter->len;
5392 		if ((copy = end - offset) > 0) {
5393 			if (unlikely(elt && sg_is_last(&sg[elt - 1])))
5394 				return -EMSGSIZE;
5395 
5396 			if (copy > len)
5397 				copy = len;
5398 			ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start,
5399 					      copy, recursion_level + 1);
5400 			if (unlikely(ret < 0))
5401 				return ret;
5402 			elt += ret;
5403 			if ((len -= copy) == 0)
5404 				return elt;
5405 			offset += copy;
5406 		}
5407 		start = end;
5408 	}
5409 	if (unlikely(len)) {
5410 		DEBUG_NET_WARN_ON_ONCE(1);
5411 		return -EINVAL;
5412 	}
5413 	return elt;
5414 }
5415 
5416 /**
5417  *	skb_to_sgvec - Fill a scatter-gather list from a socket buffer
5418  *	@skb: Socket buffer containing the buffers to be mapped
5419  *	@sg: The scatter-gather list to map into
5420  *	@offset: The offset into the buffer's contents to start mapping
5421  *	@len: Length of buffer space to be mapped
5422  *
5423  *	Fill the specified scatter-gather list with mappings/pointers into a
5424  *	region of the buffer space attached to a socket buffer. Returns either
5425  *	the number of scatterlist items used, or -EMSGSIZE if the contents
5426  *	could not fit.
5427  */
5428 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
5429 {
5430 	int nsg = __skb_to_sgvec(skb, sg, offset, len, 0);
5431 
5432 	if (nsg <= 0)
5433 		return nsg;
5434 
5435 	sg_mark_end(&sg[nsg - 1]);
5436 
5437 	return nsg;
5438 }
5439 EXPORT_SYMBOL_GPL(skb_to_sgvec);
5440 
5441 /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given
5442  * sglist without mark the sg which contain last skb data as the end.
5443  * So the caller can mannipulate sg list as will when padding new data after
5444  * the first call without calling sg_unmark_end to expend sg list.
5445  *
5446  * Scenario to use skb_to_sgvec_nomark:
5447  * 1. sg_init_table
5448  * 2. skb_to_sgvec_nomark(payload1)
5449  * 3. skb_to_sgvec_nomark(payload2)
5450  *
5451  * This is equivalent to:
5452  * 1. sg_init_table
5453  * 2. skb_to_sgvec(payload1)
5454  * 3. sg_unmark_end
5455  * 4. skb_to_sgvec(payload2)
5456  *
5457  * When mapping multiple payload conditionally, skb_to_sgvec_nomark
5458  * is more preferable.
5459  */
5460 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
5461 			int offset, int len)
5462 {
5463 	return __skb_to_sgvec(skb, sg, offset, len, 0);
5464 }
5465 EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark);
5466 
5467 
5468 
5469 /**
5470  *	skb_cow_data - Check that a socket buffer's data buffers are writable
5471  *	@skb: The socket buffer to check.
5472  *	@tailbits: Amount of trailing space to be added
5473  *	@trailer: Returned pointer to the skb where the @tailbits space begins
5474  *
5475  *	Make sure that the data buffers attached to a socket buffer are
5476  *	writable. If they are not, private copies are made of the data buffers
5477  *	and the socket buffer is set to use these instead.
5478  *
5479  *	If @tailbits is given, make sure that there is space to write @tailbits
5480  *	bytes of data beyond current end of socket buffer.  @trailer will be
5481  *	set to point to the skb in which this space begins.
5482  *
5483  *	The number of scatterlist elements required to completely map the
5484  *	COW'd and extended socket buffer will be returned.
5485  */
5486 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
5487 {
5488 	int copyflag;
5489 	int elt;
5490 	struct sk_buff *skb1, **skb_p;
5491 
5492 	/* If skb is cloned or its head is paged, reallocate
5493 	 * head pulling out all the pages (pages are considered not writable
5494 	 * at the moment even if they are anonymous).
5495 	 */
5496 	if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
5497 	    !__pskb_pull_tail(skb, __skb_pagelen(skb)))
5498 		return -ENOMEM;
5499 
5500 	/* Easy case. Most of packets will go this way. */
5501 	if (!skb_has_frag_list(skb)) {
5502 		/* A little of trouble, not enough of space for trailer.
5503 		 * This should not happen, when stack is tuned to generate
5504 		 * good frames. OK, on miss we reallocate and reserve even more
5505 		 * space, 128 bytes is fair. */
5506 
5507 		if (skb_tailroom(skb) < tailbits &&
5508 		    pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
5509 			return -ENOMEM;
5510 
5511 		/* Voila! */
5512 		*trailer = skb;
5513 		return 1;
5514 	}
5515 
5516 	/* Misery. We are in troubles, going to mincer fragments... */
5517 
5518 	elt = 1;
5519 	skb_p = &skb_shinfo(skb)->frag_list;
5520 	copyflag = 0;
5521 
5522 	while ((skb1 = *skb_p) != NULL) {
5523 		int ntail = 0;
5524 
5525 		/* The fragment is partially pulled by someone,
5526 		 * this can happen on input. Copy it and everything
5527 		 * after it. */
5528 
5529 		if (skb_shared(skb1))
5530 			copyflag = 1;
5531 
5532 		/* If the skb is the last, worry about trailer. */
5533 
5534 		if (skb1->next == NULL && tailbits) {
5535 			if (skb_shinfo(skb1)->nr_frags ||
5536 			    skb_has_frag_list(skb1) ||
5537 			    skb_tailroom(skb1) < tailbits)
5538 				ntail = tailbits + 128;
5539 		}
5540 
5541 		if (copyflag ||
5542 		    skb_cloned(skb1) ||
5543 		    ntail ||
5544 		    skb_shinfo(skb1)->nr_frags ||
5545 		    skb_has_frag_list(skb1)) {
5546 			struct sk_buff *skb2;
5547 
5548 			/* Fuck, we are miserable poor guys... */
5549 			if (ntail == 0)
5550 				skb2 = skb_copy(skb1, GFP_ATOMIC);
5551 			else
5552 				skb2 = skb_copy_expand(skb1,
5553 						       skb_headroom(skb1),
5554 						       ntail,
5555 						       GFP_ATOMIC);
5556 			if (unlikely(skb2 == NULL))
5557 				return -ENOMEM;
5558 
5559 			if (skb1->sk)
5560 				skb_set_owner_w(skb2, skb1->sk);
5561 
5562 			/* Looking around. Are we still alive?
5563 			 * OK, link new skb, drop old one */
5564 
5565 			skb2->next = skb1->next;
5566 			*skb_p = skb2;
5567 			kfree_skb(skb1);
5568 			skb1 = skb2;
5569 		}
5570 		elt++;
5571 		*trailer = skb1;
5572 		skb_p = &skb1->next;
5573 	}
5574 
5575 	return elt;
5576 }
5577 EXPORT_SYMBOL_GPL(skb_cow_data);
5578 
5579 void sock_rmem_free(struct sk_buff *skb)
5580 {
5581 	struct sock *sk = skb->sk;
5582 
5583 	atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
5584 }
5585 
5586 static void skb_set_err_queue(struct sk_buff *skb)
5587 {
5588 	/* The error-queue test in skb_is_err_queue() matches this marker
5589 	 * with the sock_rmem_free destructor installed by sock_queue_err_skb().
5590 	 */
5591 	skb->pkt_type = PACKET_OUTGOING;
5592 	BUILD_BUG_ON(PACKET_OUTGOING == 0);
5593 }
5594 
5595 /*
5596  * Note: We dont mem charge error packets (no sk_forward_alloc changes)
5597  */
5598 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
5599 {
5600 	if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
5601 	    (unsigned int)READ_ONCE(sk->sk_rcvbuf))
5602 		return -ENOMEM;
5603 
5604 	skb_orphan(skb);
5605 	skb->sk = sk;
5606 	skb->destructor = sock_rmem_free;
5607 	atomic_add(skb->truesize, &sk->sk_rmem_alloc);
5608 	skb_set_err_queue(skb);
5609 
5610 	/* before exiting rcu section, make sure dst is refcounted */
5611 	skb_dst_force(skb);
5612 
5613 	skb_queue_tail(&sk->sk_error_queue, skb);
5614 	if (!sock_flag(sk, SOCK_DEAD))
5615 		sk_error_report(sk);
5616 	return 0;
5617 }
5618 EXPORT_SYMBOL(sock_queue_err_skb);
5619 
5620 static bool is_icmp_err_skb(const struct sk_buff *skb)
5621 {
5622 	return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP ||
5623 		       SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6);
5624 }
5625 
5626 struct sk_buff *sock_dequeue_err_skb(struct sock *sk)
5627 {
5628 	struct sk_buff_head *q = &sk->sk_error_queue;
5629 	struct sk_buff *skb, *skb_next = NULL;
5630 	bool icmp_next = false;
5631 	unsigned long flags;
5632 
5633 	if (skb_queue_empty_lockless(q))
5634 		return NULL;
5635 
5636 	spin_lock_irqsave(&q->lock, flags);
5637 	skb = __skb_dequeue(q);
5638 	if (skb && (skb_next = skb_peek(q))) {
5639 		icmp_next = is_icmp_err_skb(skb_next);
5640 		if (icmp_next)
5641 			sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_errno;
5642 	}
5643 	spin_unlock_irqrestore(&q->lock, flags);
5644 
5645 	if (is_icmp_err_skb(skb) && !icmp_next)
5646 		sk->sk_err = 0;
5647 
5648 	if (skb_next)
5649 		sk_error_report(sk);
5650 
5651 	return skb;
5652 }
5653 EXPORT_SYMBOL(sock_dequeue_err_skb);
5654 
5655 /**
5656  * skb_clone_sk - create clone of skb, and take reference to socket
5657  * @skb: the skb to clone
5658  *
5659  * This function creates a clone of a buffer that holds a reference on
5660  * sk_refcnt.  Buffers created via this function are meant to be
5661  * returned using sock_queue_err_skb, or free via kfree_skb.
5662  *
5663  * When passing buffers allocated with this function to sock_queue_err_skb
5664  * it is necessary to wrap the call with sock_hold/sock_put in order to
5665  * prevent the socket from being released prior to being enqueued on
5666  * the sk_error_queue.
5667  */
5668 struct sk_buff *skb_clone_sk(struct sk_buff *skb)
5669 {
5670 	struct sock *sk = skb->sk;
5671 	struct sk_buff *clone;
5672 
5673 	if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt))
5674 		return NULL;
5675 
5676 	clone = skb_clone(skb, GFP_ATOMIC);
5677 	if (!clone) {
5678 		sock_put(sk);
5679 		return NULL;
5680 	}
5681 
5682 	clone->sk = sk;
5683 	clone->destructor = sock_efree;
5684 
5685 	return clone;
5686 }
5687 EXPORT_SYMBOL(skb_clone_sk);
5688 
5689 static void __skb_complete_tx_timestamp(struct sk_buff *skb,
5690 					struct sock *sk,
5691 					int tstype,
5692 					bool opt_stats)
5693 {
5694 	struct sock_exterr_skb *serr;
5695 	int err;
5696 
5697 	BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb));
5698 
5699 	serr = SKB_EXT_ERR(skb);
5700 	memset(serr, 0, sizeof(*serr));
5701 	serr->ee.ee_errno = ENOMSG;
5702 	serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
5703 	serr->ee.ee_info = tstype;
5704 	serr->opt_stats = opt_stats;
5705 	serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0;
5706 	if (READ_ONCE(sk->sk_tsflags) & SOF_TIMESTAMPING_OPT_ID) {
5707 		serr->ee.ee_data = skb_shinfo(skb)->tskey;
5708 		if (sk_is_tcp(sk))
5709 			serr->ee.ee_data -= atomic_read(&sk->sk_tskey);
5710 	}
5711 
5712 	err = sock_queue_err_skb(sk, skb);
5713 
5714 	if (err)
5715 		kfree_skb(skb);
5716 }
5717 
5718 static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly)
5719 {
5720 	struct socket *sock;
5721 	struct file *file;
5722 	bool ret = false;
5723 
5724 	if (likely(tsonly || READ_ONCE(sock_net(sk)->core.sysctl_tstamp_allow_data)))
5725 		return true;
5726 
5727 	/* The sk pointer remains valid as long as the skb is. The sk_socket and
5728 	 * file pointer may become NULL if the socket is closed. Both structures
5729 	 * (including file->cred) are RCU freed which means they can be accessed
5730 	 * within a RCU read section.
5731 	 */
5732 	rcu_read_lock();
5733 	sock = READ_ONCE(sk->sk_socket);
5734 	if (!sock)
5735 		goto out;
5736 	file = READ_ONCE(sock->file);
5737 	if (!file)
5738 		goto out;
5739 	ret = file_ns_capable(file, &init_user_ns, CAP_NET_RAW);
5740 out:
5741 	rcu_read_unlock();
5742 	return ret;
5743 }
5744 
5745 void skb_complete_tx_timestamp(struct sk_buff *skb,
5746 			       struct skb_shared_hwtstamps *hwtstamps)
5747 {
5748 	struct sock *sk = skb->sk;
5749 
5750 	if (!skb_may_tx_timestamp(sk, false))
5751 		goto err;
5752 
5753 	/* Take a reference to prevent skb_orphan() from freeing the socket,
5754 	 * but only if the socket refcount is not zero.
5755 	 */
5756 	if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
5757 		*skb_hwtstamps(skb) = *hwtstamps;
5758 		__skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false);
5759 		sock_put(sk);
5760 		return;
5761 	}
5762 
5763 err:
5764 	kfree_skb(skb);
5765 }
5766 EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp);
5767 
5768 static bool skb_tstamp_tx_report_so_timestamping(struct sk_buff *skb,
5769 						 struct skb_shared_hwtstamps *hwtstamps,
5770 						 int tstype)
5771 {
5772 	switch (tstype) {
5773 	case SCM_TSTAMP_SCHED:
5774 		return skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP;
5775 	case SCM_TSTAMP_SND:
5776 		return skb_shinfo(skb)->tx_flags & (hwtstamps ? SKBTX_HW_TSTAMP_NOBPF :
5777 						    SKBTX_SW_TSTAMP);
5778 	case SCM_TSTAMP_ACK:
5779 		return TCP_SKB_CB(skb)->txstamp_ack & TSTAMP_ACK_SK;
5780 	case SCM_TSTAMP_COMPLETION:
5781 		return skb_shinfo(skb)->tx_flags & SKBTX_COMPLETION_TSTAMP;
5782 	}
5783 
5784 	return false;
5785 }
5786 
5787 static void skb_tstamp_tx_report_bpf_timestamping(struct sk_buff *skb,
5788 						  struct skb_shared_hwtstamps *hwtstamps,
5789 						  struct sock *sk,
5790 						  int tstype)
5791 {
5792 	int op;
5793 
5794 	switch (tstype) {
5795 	case SCM_TSTAMP_SCHED:
5796 		op = BPF_SOCK_OPS_TSTAMP_SCHED_CB;
5797 		break;
5798 	case SCM_TSTAMP_SND:
5799 		if (hwtstamps) {
5800 			op = BPF_SOCK_OPS_TSTAMP_SND_HW_CB;
5801 			*skb_hwtstamps(skb) = *hwtstamps;
5802 		} else {
5803 			op = BPF_SOCK_OPS_TSTAMP_SND_SW_CB;
5804 		}
5805 		break;
5806 	case SCM_TSTAMP_ACK:
5807 		op = BPF_SOCK_OPS_TSTAMP_ACK_CB;
5808 		break;
5809 	default:
5810 		return;
5811 	}
5812 
5813 	bpf_skops_tx_timestamping(sk, skb, op);
5814 }
5815 
5816 void __skb_tstamp_tx(struct sk_buff *orig_skb,
5817 		     const struct sk_buff *ack_skb,
5818 		     struct skb_shared_hwtstamps *hwtstamps,
5819 		     struct sock *sk, int tstype)
5820 {
5821 	struct sk_buff *skb;
5822 	bool tsonly, opt_stats = false;
5823 	u32 tsflags;
5824 
5825 	if (!sk)
5826 		return;
5827 
5828 	if (skb_shinfo(orig_skb)->tx_flags & SKBTX_BPF)
5829 		skb_tstamp_tx_report_bpf_timestamping(orig_skb, hwtstamps,
5830 						      sk, tstype);
5831 
5832 	if (!skb_tstamp_tx_report_so_timestamping(orig_skb, hwtstamps, tstype))
5833 		return;
5834 
5835 	tsflags = READ_ONCE(sk->sk_tsflags);
5836 	if (!hwtstamps && !(tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) &&
5837 	    skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS)
5838 		return;
5839 
5840 	tsonly = tsflags & SOF_TIMESTAMPING_OPT_TSONLY;
5841 	if (!skb_may_tx_timestamp(sk, tsonly))
5842 		return;
5843 
5844 	if (tsonly) {
5845 #ifdef CONFIG_INET
5846 		if ((tsflags & SOF_TIMESTAMPING_OPT_STATS) &&
5847 		    sk_is_tcp(sk)) {
5848 			skb = tcp_get_timestamping_opt_stats(sk, orig_skb,
5849 							     ack_skb);
5850 			opt_stats = true;
5851 		} else
5852 #endif
5853 			skb = alloc_skb(0, GFP_ATOMIC);
5854 	} else {
5855 		skb = skb_clone(orig_skb, GFP_ATOMIC);
5856 
5857 		if (skb_orphan_frags_rx(skb, GFP_ATOMIC)) {
5858 			kfree_skb(skb);
5859 			return;
5860 		}
5861 	}
5862 	if (!skb)
5863 		return;
5864 
5865 	if (tsonly) {
5866 		skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags &
5867 					     SKBTX_ANY_TSTAMP;
5868 		skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey;
5869 	}
5870 
5871 	if (hwtstamps)
5872 		*skb_hwtstamps(skb) = *hwtstamps;
5873 	else
5874 		__net_timestamp(skb);
5875 
5876 	__skb_complete_tx_timestamp(skb, sk, tstype, opt_stats);
5877 }
5878 EXPORT_SYMBOL_GPL(__skb_tstamp_tx);
5879 
5880 void skb_tstamp_tx(struct sk_buff *orig_skb,
5881 		   struct skb_shared_hwtstamps *hwtstamps)
5882 {
5883 	return __skb_tstamp_tx(orig_skb, NULL, hwtstamps, orig_skb->sk,
5884 			       SCM_TSTAMP_SND);
5885 }
5886 EXPORT_SYMBOL_GPL(skb_tstamp_tx);
5887 
5888 #ifdef CONFIG_WIRELESS
5889 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
5890 {
5891 	struct sock *sk = skb->sk;
5892 	struct sock_exterr_skb *serr;
5893 	int err = 1;
5894 
5895 	skb->wifi_acked_valid = 1;
5896 	skb->wifi_acked = acked;
5897 
5898 	serr = SKB_EXT_ERR(skb);
5899 	memset(serr, 0, sizeof(*serr));
5900 	serr->ee.ee_errno = ENOMSG;
5901 	serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;
5902 
5903 	/* Take a reference to prevent skb_orphan() from freeing the socket,
5904 	 * but only if the socket refcount is not zero.
5905 	 */
5906 	if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
5907 		err = sock_queue_err_skb(sk, skb);
5908 		sock_put(sk);
5909 	}
5910 	if (err)
5911 		kfree_skb(skb);
5912 }
5913 EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);
5914 #endif /* CONFIG_WIRELESS */
5915 
5916 /**
5917  * skb_partial_csum_set - set up and verify partial csum values for packet
5918  * @skb: the skb to set
5919  * @start: the number of bytes after skb->data to start checksumming.
5920  * @off: the offset from start to place the checksum.
5921  *
5922  * For untrusted partially-checksummed packets, we need to make sure the values
5923  * for skb->csum_start and skb->csum_offset are valid so we don't oops.
5924  *
5925  * This function checks and sets those values and skb->ip_summed: if this
5926  * returns false you should drop the packet.
5927  */
5928 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
5929 {
5930 	u32 csum_end = (u32)start + (u32)off + sizeof(__sum16);
5931 	u32 csum_start = skb_headroom(skb) + (u32)start;
5932 
5933 	if (unlikely(csum_start >= U16_MAX || csum_end > skb_headlen(skb))) {
5934 		net_warn_ratelimited("bad partial csum: csum=%u/%u headroom=%u headlen=%u\n",
5935 				     start, off, skb_headroom(skb), skb_headlen(skb));
5936 		return false;
5937 	}
5938 	skb->ip_summed = CHECKSUM_PARTIAL;
5939 	skb->csum_start = csum_start;
5940 	skb->csum_offset = off;
5941 	skb->transport_header = csum_start;
5942 	return true;
5943 }
5944 EXPORT_SYMBOL_GPL(skb_partial_csum_set);
5945 
5946 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len,
5947 			       unsigned int max)
5948 {
5949 	if (skb_headlen(skb) >= len)
5950 		return 0;
5951 
5952 	/* If we need to pullup then pullup to the max, so we
5953 	 * won't need to do it again.
5954 	 */
5955 	if (max > skb->len)
5956 		max = skb->len;
5957 
5958 	if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL)
5959 		return -ENOMEM;
5960 
5961 	if (skb_headlen(skb) < len)
5962 		return -EPROTO;
5963 
5964 	return 0;
5965 }
5966 
5967 #define MAX_TCP_HDR_LEN (15 * 4)
5968 
5969 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb,
5970 				      typeof(IPPROTO_IP) proto,
5971 				      unsigned int off)
5972 {
5973 	int err;
5974 
5975 	switch (proto) {
5976 	case IPPROTO_TCP:
5977 		err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr),
5978 					  off + MAX_TCP_HDR_LEN);
5979 		if (!err && !skb_partial_csum_set(skb, off,
5980 						  offsetof(struct tcphdr,
5981 							   check)))
5982 			err = -EPROTO;
5983 		return err ? ERR_PTR(err) : &tcp_hdr(skb)->check;
5984 
5985 	case IPPROTO_UDP:
5986 		err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr),
5987 					  off + sizeof(struct udphdr));
5988 		if (!err && !skb_partial_csum_set(skb, off,
5989 						  offsetof(struct udphdr,
5990 							   check)))
5991 			err = -EPROTO;
5992 		return err ? ERR_PTR(err) : &udp_hdr(skb)->check;
5993 	}
5994 
5995 	return ERR_PTR(-EPROTO);
5996 }
5997 
5998 /* This value should be large enough to cover a tagged ethernet header plus
5999  * maximally sized IP and TCP or UDP headers.
6000  */
6001 #define MAX_IP_HDR_LEN 128
6002 
6003 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate)
6004 {
6005 	unsigned int off;
6006 	bool fragment;
6007 	__sum16 *csum;
6008 	int err;
6009 
6010 	fragment = false;
6011 
6012 	err = skb_maybe_pull_tail(skb,
6013 				  sizeof(struct iphdr),
6014 				  MAX_IP_HDR_LEN);
6015 	if (err < 0)
6016 		goto out;
6017 
6018 	if (ip_is_fragment(ip_hdr(skb)))
6019 		fragment = true;
6020 
6021 	off = ip_hdrlen(skb);
6022 
6023 	err = -EPROTO;
6024 
6025 	if (fragment)
6026 		goto out;
6027 
6028 	csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off);
6029 	if (IS_ERR(csum))
6030 		return PTR_ERR(csum);
6031 
6032 	if (recalculate)
6033 		*csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
6034 					   ip_hdr(skb)->daddr,
6035 					   skb->len - off,
6036 					   ip_hdr(skb)->protocol, 0);
6037 	err = 0;
6038 
6039 out:
6040 	return err;
6041 }
6042 
6043 /* This value should be large enough to cover a tagged ethernet header plus
6044  * an IPv6 header, all options, and a maximal TCP or UDP header.
6045  */
6046 #define MAX_IPV6_HDR_LEN 256
6047 
6048 #define OPT_HDR(type, skb, off) \
6049 	(type *)(skb_network_header(skb) + (off))
6050 
6051 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate)
6052 {
6053 	int err;
6054 	u8 nexthdr;
6055 	unsigned int off;
6056 	unsigned int len;
6057 	bool fragment;
6058 	bool done;
6059 	__sum16 *csum;
6060 
6061 	fragment = false;
6062 	done = false;
6063 
6064 	off = sizeof(struct ipv6hdr);
6065 
6066 	err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN);
6067 	if (err < 0)
6068 		goto out;
6069 
6070 	nexthdr = ipv6_hdr(skb)->nexthdr;
6071 
6072 	len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len);
6073 	while (off <= len && !done) {
6074 		switch (nexthdr) {
6075 		case IPPROTO_DSTOPTS:
6076 		case IPPROTO_HOPOPTS:
6077 		case IPPROTO_ROUTING: {
6078 			struct ipv6_opt_hdr *hp;
6079 
6080 			err = skb_maybe_pull_tail(skb,
6081 						  off +
6082 						  sizeof(struct ipv6_opt_hdr),
6083 						  off +
6084 						  sizeof(struct ipv6_opt_hdr));
6085 			if (err < 0)
6086 				goto out;
6087 
6088 			hp = OPT_HDR(struct ipv6_opt_hdr, skb, off);
6089 			nexthdr = hp->nexthdr;
6090 			off += ipv6_optlen(hp);
6091 			break;
6092 		}
6093 		case IPPROTO_AH: {
6094 			struct ip_auth_hdr *hp;
6095 
6096 			err = skb_maybe_pull_tail(skb,
6097 						  off +
6098 						  sizeof(struct ip_auth_hdr),
6099 						  off +
6100 						  sizeof(struct ip_auth_hdr));
6101 			if (err < 0)
6102 				goto out;
6103 
6104 			hp = OPT_HDR(struct ip_auth_hdr, skb, off);
6105 			nexthdr = hp->nexthdr;
6106 			off += ipv6_authlen(hp);
6107 			break;
6108 		}
6109 		case IPPROTO_FRAGMENT: {
6110 			struct frag_hdr *hp;
6111 
6112 			err = skb_maybe_pull_tail(skb,
6113 						  off +
6114 						  sizeof(struct frag_hdr),
6115 						  off +
6116 						  sizeof(struct frag_hdr));
6117 			if (err < 0)
6118 				goto out;
6119 
6120 			hp = OPT_HDR(struct frag_hdr, skb, off);
6121 
6122 			if (hp->frag_off & htons(IP6_OFFSET | IP6_MF))
6123 				fragment = true;
6124 
6125 			nexthdr = hp->nexthdr;
6126 			off += sizeof(struct frag_hdr);
6127 			break;
6128 		}
6129 		default:
6130 			done = true;
6131 			break;
6132 		}
6133 	}
6134 
6135 	err = -EPROTO;
6136 
6137 	if (!done || fragment)
6138 		goto out;
6139 
6140 	csum = skb_checksum_setup_ip(skb, nexthdr, off);
6141 	if (IS_ERR(csum))
6142 		return PTR_ERR(csum);
6143 
6144 	if (recalculate)
6145 		*csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
6146 					 &ipv6_hdr(skb)->daddr,
6147 					 skb->len - off, nexthdr, 0);
6148 	err = 0;
6149 
6150 out:
6151 	return err;
6152 }
6153 
6154 /**
6155  * skb_checksum_setup - set up partial checksum offset
6156  * @skb: the skb to set up
6157  * @recalculate: if true the pseudo-header checksum will be recalculated
6158  */
6159 int skb_checksum_setup(struct sk_buff *skb, bool recalculate)
6160 {
6161 	int err;
6162 
6163 	switch (skb->protocol) {
6164 	case htons(ETH_P_IP):
6165 		err = skb_checksum_setup_ipv4(skb, recalculate);
6166 		break;
6167 
6168 	case htons(ETH_P_IPV6):
6169 		err = skb_checksum_setup_ipv6(skb, recalculate);
6170 		break;
6171 
6172 	default:
6173 		err = -EPROTO;
6174 		break;
6175 	}
6176 
6177 	return err;
6178 }
6179 EXPORT_SYMBOL(skb_checksum_setup);
6180 
6181 /**
6182  * skb_checksum_maybe_trim - maybe trims the given skb
6183  * @skb: the skb to check
6184  * @transport_len: the data length beyond the network header
6185  *
6186  * Checks whether the given skb has data beyond the given transport length.
6187  * If so, returns a cloned skb trimmed to this transport length.
6188  * Otherwise returns the provided skb. Returns NULL in error cases
6189  * (e.g. transport_len exceeds skb length or out-of-memory).
6190  *
6191  * Caller needs to set the skb transport header and free any returned skb if it
6192  * differs from the provided skb.
6193  */
6194 static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb,
6195 					       unsigned int transport_len)
6196 {
6197 	struct sk_buff *skb_chk;
6198 	unsigned int len = skb_transport_offset(skb) + transport_len;
6199 	int ret;
6200 
6201 	if (skb->len < len)
6202 		return NULL;
6203 	else if (skb->len == len)
6204 		return skb;
6205 
6206 	skb_chk = skb_clone(skb, GFP_ATOMIC);
6207 	if (!skb_chk)
6208 		return NULL;
6209 
6210 	ret = pskb_trim_rcsum(skb_chk, len);
6211 	if (ret) {
6212 		kfree_skb(skb_chk);
6213 		return NULL;
6214 	}
6215 
6216 	return skb_chk;
6217 }
6218 
6219 /**
6220  * skb_checksum_trimmed - validate checksum of an skb
6221  * @skb: the skb to check
6222  * @transport_len: the data length beyond the network header
6223  * @skb_chkf: checksum function to use
6224  *
6225  * Applies the given checksum function skb_chkf to the provided skb.
6226  * Returns a checked and maybe trimmed skb. Returns NULL on error.
6227  *
6228  * If the skb has data beyond the given transport length, then a
6229  * trimmed & cloned skb is checked and returned.
6230  *
6231  * Caller needs to set the skb transport header and free any returned skb if it
6232  * differs from the provided skb.
6233  */
6234 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
6235 				     unsigned int transport_len,
6236 				     __sum16(*skb_chkf)(struct sk_buff *skb))
6237 {
6238 	struct sk_buff *skb_chk;
6239 	unsigned int offset = skb_transport_offset(skb);
6240 	__sum16 ret;
6241 
6242 	skb_chk = skb_checksum_maybe_trim(skb, transport_len);
6243 	if (!skb_chk)
6244 		goto err;
6245 
6246 	if (!pskb_may_pull(skb_chk, offset))
6247 		goto err;
6248 
6249 	skb_pull_rcsum(skb_chk, offset);
6250 	ret = skb_chkf(skb_chk);
6251 	skb_push_rcsum(skb_chk, offset);
6252 
6253 	if (ret)
6254 		goto err;
6255 
6256 	return skb_chk;
6257 
6258 err:
6259 	if (skb_chk && skb_chk != skb)
6260 		kfree_skb(skb_chk);
6261 
6262 	return NULL;
6263 
6264 }
6265 EXPORT_SYMBOL(skb_checksum_trimmed);
6266 
6267 void __skb_warn_lro_forwarding(const struct sk_buff *skb)
6268 {
6269 	net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n",
6270 			     skb->dev->name);
6271 }
6272 EXPORT_SYMBOL(__skb_warn_lro_forwarding);
6273 
6274 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
6275 {
6276 	if (head_stolen) {
6277 		skb_release_head_state(skb);
6278 		kfree_skbmem(skb);
6279 	} else {
6280 		__kfree_skb(skb);
6281 	}
6282 }
6283 EXPORT_SYMBOL(kfree_skb_partial);
6284 
6285 /**
6286  * skb_try_coalesce - try to merge skb to prior one
6287  * @to: prior buffer
6288  * @from: buffer to add
6289  * @fragstolen: pointer to boolean
6290  * @delta_truesize: how much more was allocated than was requested
6291  */
6292 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
6293 		      bool *fragstolen, int *delta_truesize)
6294 {
6295 	struct skb_shared_info *to_shinfo, *from_shinfo;
6296 	int i, delta, len = from->len;
6297 
6298 	*fragstolen = false;
6299 
6300 	if (skb_cloned(to))
6301 		return false;
6302 
6303 	/* In general, avoid mixing page_pool and non-page_pool allocated
6304 	 * pages within the same SKB. In theory we could take full
6305 	 * references if @from is cloned and !@to->pp_recycle but its
6306 	 * tricky (due to potential race with the clone disappearing) and
6307 	 * rare, so not worth dealing with.
6308 	 */
6309 	if (to->pp_recycle != from->pp_recycle)
6310 		return false;
6311 
6312 	/* All frags in an skb must either all be struct page-backed or all
6313 	 * belong to the same memory provider instance; do not coalesce skbs
6314 	 * that would mix them.
6315 	 */
6316 	if (skb_frags_readable(from) != skb_frags_readable(to))
6317 		return false;
6318 
6319 	if (len <= skb_tailroom(to) && skb_frags_readable(from)) {
6320 		if (len) {
6321 			if (unlikely(skb_copy_bits(from, 0,
6322 						   skb_tail_pointer(to),
6323 						   len))) {
6324 				DEBUG_NET_WARN_ON_ONCE(1);
6325 				return false;
6326 			}
6327 			skb_put(to, len);
6328 		}
6329 		*delta_truesize = 0;
6330 		return true;
6331 	}
6332 
6333 	to_shinfo = skb_shinfo(to);
6334 	from_shinfo = skb_shinfo(from);
6335 	if (to_shinfo->frag_list || from_shinfo->frag_list)
6336 		return false;
6337 	if (skb_zcopy(to) || skb_zcopy(from))
6338 		return false;
6339 
6340 	if (skb_headlen(from) != 0) {
6341 		struct page *page;
6342 		unsigned int offset;
6343 
6344 		if (to_shinfo->nr_frags +
6345 		    from_shinfo->nr_frags >= MAX_SKB_FRAGS)
6346 			return false;
6347 
6348 		if (skb_head_is_locked(from))
6349 			return false;
6350 
6351 		delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
6352 
6353 		page = virt_to_head_page(from->head);
6354 		offset = from->data - (unsigned char *)page_address(page);
6355 
6356 		skb_fill_page_desc(to, to_shinfo->nr_frags,
6357 				   page, offset, skb_headlen(from));
6358 		*fragstolen = true;
6359 	} else {
6360 		if (to_shinfo->nr_frags +
6361 		    from_shinfo->nr_frags > MAX_SKB_FRAGS)
6362 			return false;
6363 
6364 		delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from));
6365 	}
6366 
6367 	WARN_ON_ONCE(delta < len);
6368 
6369 	memcpy(to_shinfo->frags + to_shinfo->nr_frags,
6370 	       from_shinfo->frags,
6371 	       from_shinfo->nr_frags * sizeof(skb_frag_t));
6372 	to_shinfo->nr_frags += from_shinfo->nr_frags;
6373 	if (from_shinfo->nr_frags)
6374 		to_shinfo->flags |= from_shinfo->flags & SKBFL_SHARED_FRAG;
6375 
6376 	if (!skb_cloned(from))
6377 		from_shinfo->nr_frags = 0;
6378 
6379 	/* if the skb is not cloned this does nothing
6380 	 * since we set nr_frags to 0.
6381 	 */
6382 	for (i = 0; i < from_shinfo->nr_frags; i++)
6383 		skb_frag_ref(from, i);
6384 
6385 	to->truesize += delta;
6386 	to->len += len;
6387 	to->data_len += len;
6388 
6389 	*delta_truesize = delta;
6390 	return true;
6391 }
6392 EXPORT_SYMBOL(skb_try_coalesce);
6393 
6394 static void skb_ext_scrub(struct sk_buff *skb)
6395 {
6396 #if IS_ENABLED(CONFIG_INET_PSP)
6397 	struct psp_skb_ext *pse = skb_ext_find(skb, SKB_EXT_PSP);
6398 
6399 	if (pse) {
6400 		struct skb_ext *ext = skb->extensions;
6401 		struct psp_skb_ext tmp;
6402 
6403 		/* Nothing else to scrub if PSP is the only extension in ext. */
6404 		if (ext->chunks == SKB_EXT_CHUNKSIZEOF(*ext) +
6405 				   skb_ext_type_len[SKB_EXT_PSP])
6406 			return;
6407 
6408 		tmp = *pse;
6409 		skb_ext_reset(skb);
6410 		pse = skb_ext_add(skb, SKB_EXT_PSP);
6411 		if (pse)
6412 			memcpy(pse, &tmp, sizeof(*pse));
6413 		return;
6414 	}
6415 #endif
6416 	skb_ext_reset(skb);
6417 }
6418 
6419 /**
6420  * skb_scrub_packet - scrub an skb
6421  *
6422  * @skb: buffer to clean
6423  * @xnet: packet is crossing netns
6424  *
6425  * skb_scrub_packet can be used after encapsulating or decapsulating a packet
6426  * into/from a tunnel. Some information have to be cleared during these
6427  * operations.
6428  * skb_scrub_packet can also be used to clean a skb before injecting it in
6429  * another namespace (@xnet == true). We have to clear all information in the
6430  * skb that could impact namespace isolation. Note that PSP skb extension is
6431  * preserved across packet scrubbing by design.
6432  */
6433 void skb_scrub_packet(struct sk_buff *skb, bool xnet)
6434 {
6435 	skb->pkt_type = PACKET_HOST;
6436 	skb->skb_iif = 0;
6437 	skb->ignore_df = 0;
6438 	skb_dst_drop(skb);
6439 	skb_ext_scrub(skb);
6440 	nf_reset_ct(skb);
6441 	nf_reset_trace(skb);
6442 
6443 #ifdef CONFIG_NET_SWITCHDEV
6444 	skb->offload_fwd_mark = 0;
6445 	skb->offload_l3_fwd_mark = 0;
6446 #endif
6447 	ipvs_reset(skb);
6448 
6449 	if (!xnet)
6450 		return;
6451 
6452 	skb->mark = 0;
6453 	skb_clear_tstamp(skb);
6454 }
6455 EXPORT_SYMBOL_GPL(skb_scrub_packet);
6456 
6457 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb)
6458 {
6459 	int mac_len, meta_len;
6460 	void *meta;
6461 
6462 	if (skb_cow(skb, skb_headroom(skb)) < 0) {
6463 		kfree_skb(skb);
6464 		return NULL;
6465 	}
6466 
6467 	mac_len = skb->data - skb_mac_header(skb);
6468 	if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) {
6469 		memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb),
6470 			mac_len - VLAN_HLEN - ETH_TLEN);
6471 	}
6472 
6473 	meta_len = skb_metadata_len(skb);
6474 	if (meta_len) {
6475 		meta = skb_metadata_end(skb) - meta_len;
6476 		memmove(meta + VLAN_HLEN, meta, meta_len);
6477 	}
6478 
6479 	skb->mac_header += VLAN_HLEN;
6480 	return skb;
6481 }
6482 
6483 struct sk_buff *skb_vlan_untag(struct sk_buff *skb)
6484 {
6485 	struct vlan_hdr *vhdr;
6486 	u16 vlan_tci;
6487 
6488 	if (unlikely(skb_vlan_tag_present(skb))) {
6489 		/* vlan_tci is already set-up so leave this for another time */
6490 		return skb;
6491 	}
6492 
6493 	skb = skb_share_check(skb, GFP_ATOMIC);
6494 	if (unlikely(!skb))
6495 		goto err_free;
6496 	if (unlikely(!pskb_may_pull(skb, VLAN_HLEN)))
6497 		goto err_free;
6498 
6499 	vhdr = (struct vlan_hdr *)skb->data;
6500 	vlan_tci = ntohs(vhdr->h_vlan_TCI);
6501 	__vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci);
6502 
6503 	skb_pull_rcsum(skb, VLAN_HLEN);
6504 	vlan_set_encap_proto(skb, vhdr);
6505 
6506 	skb = skb_reorder_vlan_header(skb);
6507 	if (unlikely(!skb))
6508 		goto err_free;
6509 
6510 	skb_reset_network_header(skb);
6511 	if (!skb_transport_header_was_set(skb))
6512 		skb_reset_transport_header(skb);
6513 	skb_reset_mac_len(skb);
6514 
6515 	return skb;
6516 
6517 err_free:
6518 	kfree_skb(skb);
6519 	return NULL;
6520 }
6521 EXPORT_SYMBOL(skb_vlan_untag);
6522 
6523 int skb_ensure_writable(struct sk_buff *skb, unsigned int write_len)
6524 {
6525 	if (!pskb_may_pull(skb, write_len))
6526 		return -ENOMEM;
6527 
6528 	if (!skb_cloned(skb) || skb_clone_writable(skb, write_len))
6529 		return 0;
6530 
6531 	return pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
6532 }
6533 EXPORT_SYMBOL(skb_ensure_writable);
6534 
6535 int skb_ensure_writable_head_tail(struct sk_buff *skb, struct net_device *dev)
6536 {
6537 	int needed_headroom = dev->needed_headroom;
6538 	int needed_tailroom = dev->needed_tailroom;
6539 
6540 	/* For tail taggers, we need to pad short frames ourselves, to ensure
6541 	 * that the tail tag does not fail at its role of being at the end of
6542 	 * the packet, once the conduit interface pads the frame. Account for
6543 	 * that pad length here, and pad later.
6544 	 */
6545 	if (unlikely(needed_tailroom && skb->len < ETH_ZLEN))
6546 		needed_tailroom += ETH_ZLEN - skb->len;
6547 	/* skb_headroom() returns unsigned int... */
6548 	needed_headroom = max_t(int, needed_headroom - skb_headroom(skb), 0);
6549 	needed_tailroom = max_t(int, needed_tailroom - skb_tailroom(skb), 0);
6550 
6551 	if (likely(!needed_headroom && !needed_tailroom && !skb_cloned(skb)))
6552 		/* No reallocation needed, yay! */
6553 		return 0;
6554 
6555 	return pskb_expand_head(skb, needed_headroom, needed_tailroom,
6556 				GFP_ATOMIC);
6557 }
6558 EXPORT_SYMBOL(skb_ensure_writable_head_tail);
6559 
6560 /* remove VLAN header from packet and update csum accordingly.
6561  * expects a non skb_vlan_tag_present skb with a vlan tag payload
6562  */
6563 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci)
6564 {
6565 	int offset = skb->data - skb_mac_header(skb);
6566 	int err;
6567 
6568 	if (WARN_ONCE(offset,
6569 		      "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n",
6570 		      offset)) {
6571 		return -EINVAL;
6572 	}
6573 
6574 	err = skb_ensure_writable(skb, VLAN_ETH_HLEN);
6575 	if (unlikely(err))
6576 		return err;
6577 
6578 	skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
6579 
6580 	vlan_remove_tag(skb, vlan_tci);
6581 
6582 	skb->mac_header += VLAN_HLEN;
6583 
6584 	if (skb_network_offset(skb) < ETH_HLEN)
6585 		skb_set_network_header(skb, ETH_HLEN);
6586 
6587 	skb_reset_mac_len(skb);
6588 
6589 	return err;
6590 }
6591 EXPORT_SYMBOL(__skb_vlan_pop);
6592 
6593 /* Pop a vlan tag either from hwaccel or from payload.
6594  * Expects skb->data at mac header.
6595  */
6596 int skb_vlan_pop(struct sk_buff *skb)
6597 {
6598 	u16 vlan_tci;
6599 	__be16 vlan_proto;
6600 	int err;
6601 
6602 	if (likely(skb_vlan_tag_present(skb))) {
6603 		__vlan_hwaccel_clear_tag(skb);
6604 	} else {
6605 		if (unlikely(!eth_type_vlan(skb->protocol)))
6606 			return 0;
6607 
6608 		err = __skb_vlan_pop(skb, &vlan_tci);
6609 		if (err)
6610 			return err;
6611 	}
6612 	/* move next vlan tag to hw accel tag */
6613 	if (likely(!eth_type_vlan(skb->protocol)))
6614 		return 0;
6615 
6616 	vlan_proto = skb->protocol;
6617 	err = __skb_vlan_pop(skb, &vlan_tci);
6618 	if (unlikely(err))
6619 		return err;
6620 
6621 	__vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
6622 	return 0;
6623 }
6624 EXPORT_SYMBOL(skb_vlan_pop);
6625 
6626 /* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present).
6627  * Expects skb->data at mac header.
6628  */
6629 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci)
6630 {
6631 	if (skb_vlan_tag_present(skb)) {
6632 		int offset = skb->data - skb_mac_header(skb);
6633 		int err;
6634 
6635 		if (WARN_ONCE(offset,
6636 			      "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n",
6637 			      offset)) {
6638 			return -EINVAL;
6639 		}
6640 
6641 		err = __vlan_insert_tag(skb, skb->vlan_proto,
6642 					skb_vlan_tag_get(skb));
6643 		if (err)
6644 			return err;
6645 
6646 		skb->protocol = skb->vlan_proto;
6647 		skb->network_header -= VLAN_HLEN;
6648 
6649 		skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
6650 	}
6651 	__vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
6652 	return 0;
6653 }
6654 EXPORT_SYMBOL(skb_vlan_push);
6655 
6656 /**
6657  * skb_eth_pop() - Drop the Ethernet header at the head of a packet
6658  *
6659  * @skb: Socket buffer to modify
6660  *
6661  * Drop the Ethernet header of @skb.
6662  *
6663  * Expects that skb->data points to the mac header and that no VLAN tags are
6664  * present.
6665  *
6666  * Returns 0 on success, -errno otherwise.
6667  */
6668 int skb_eth_pop(struct sk_buff *skb)
6669 {
6670 	if (!pskb_may_pull(skb, ETH_HLEN) || skb_vlan_tagged(skb) ||
6671 	    skb_network_offset(skb) < ETH_HLEN)
6672 		return -EPROTO;
6673 
6674 	skb_pull_rcsum(skb, ETH_HLEN);
6675 	skb_reset_mac_header(skb);
6676 	skb_reset_mac_len(skb);
6677 
6678 	return 0;
6679 }
6680 EXPORT_SYMBOL(skb_eth_pop);
6681 
6682 /**
6683  * skb_eth_push() - Add a new Ethernet header at the head of a packet
6684  *
6685  * @skb: Socket buffer to modify
6686  * @dst: Destination MAC address of the new header
6687  * @src: Source MAC address of the new header
6688  *
6689  * Prepend @skb with a new Ethernet header.
6690  *
6691  * Expects that skb->data points to the mac header, which must be empty.
6692  *
6693  * Returns 0 on success, -errno otherwise.
6694  */
6695 int skb_eth_push(struct sk_buff *skb, const unsigned char *dst,
6696 		 const unsigned char *src)
6697 {
6698 	struct ethhdr *eth;
6699 	int err;
6700 
6701 	if (skb_network_offset(skb) || skb_vlan_tag_present(skb))
6702 		return -EPROTO;
6703 
6704 	err = skb_cow_head(skb, sizeof(*eth));
6705 	if (err < 0)
6706 		return err;
6707 
6708 	skb_push(skb, sizeof(*eth));
6709 	skb_reset_mac_header(skb);
6710 	skb_reset_mac_len(skb);
6711 
6712 	eth = eth_hdr(skb);
6713 	ether_addr_copy(eth->h_dest, dst);
6714 	ether_addr_copy(eth->h_source, src);
6715 	eth->h_proto = skb->protocol;
6716 
6717 	skb_postpush_rcsum(skb, eth, sizeof(*eth));
6718 
6719 	return 0;
6720 }
6721 EXPORT_SYMBOL(skb_eth_push);
6722 
6723 /* Update the ethertype of hdr and the skb csum value if required. */
6724 static void skb_mod_eth_type(struct sk_buff *skb, struct ethhdr *hdr,
6725 			     __be16 ethertype)
6726 {
6727 	if (skb->ip_summed == CHECKSUM_COMPLETE) {
6728 		__be16 diff[] = { ~hdr->h_proto, ethertype };
6729 
6730 		skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
6731 	}
6732 
6733 	hdr->h_proto = ethertype;
6734 }
6735 
6736 /**
6737  * skb_mpls_push() - push a new MPLS header after mac_len bytes from start of
6738  *                   the packet
6739  *
6740  * @skb: buffer
6741  * @mpls_lse: MPLS label stack entry to push
6742  * @mpls_proto: ethertype of the new MPLS header (expects 0x8847 or 0x8848)
6743  * @mac_len: length of the MAC header
6744  * @ethernet: flag to indicate if the resulting packet after skb_mpls_push is
6745  *            ethernet
6746  *
6747  * Expects skb->data at mac header.
6748  *
6749  * Returns 0 on success, -errno otherwise.
6750  */
6751 int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto,
6752 		  int mac_len, bool ethernet)
6753 {
6754 	struct mpls_shim_hdr *lse;
6755 	int err;
6756 
6757 	if (unlikely(!eth_p_mpls(mpls_proto)))
6758 		return -EINVAL;
6759 
6760 	/* Networking stack does not allow simultaneous Tunnel and MPLS GSO. */
6761 	if (skb->encapsulation)
6762 		return -EINVAL;
6763 
6764 	err = skb_cow_head(skb, MPLS_HLEN);
6765 	if (unlikely(err))
6766 		return err;
6767 
6768 	if (!skb->inner_protocol) {
6769 		skb_set_inner_network_header(skb, skb_network_offset(skb));
6770 		skb_set_inner_protocol(skb, skb->protocol);
6771 	}
6772 
6773 	skb_push(skb, MPLS_HLEN);
6774 	memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb),
6775 		mac_len);
6776 	skb_reset_mac_header(skb);
6777 	skb_set_network_header(skb, mac_len);
6778 	skb_reset_mac_len(skb);
6779 
6780 	lse = mpls_hdr(skb);
6781 	lse->label_stack_entry = mpls_lse;
6782 	skb_postpush_rcsum(skb, lse, MPLS_HLEN);
6783 
6784 	if (ethernet && mac_len >= ETH_HLEN)
6785 		skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto);
6786 	skb->protocol = mpls_proto;
6787 
6788 	return 0;
6789 }
6790 EXPORT_SYMBOL_GPL(skb_mpls_push);
6791 
6792 /**
6793  * skb_mpls_pop() - pop the outermost MPLS header
6794  *
6795  * @skb: buffer
6796  * @next_proto: ethertype of header after popped MPLS header
6797  * @mac_len: length of the MAC header
6798  * @ethernet: flag to indicate if the packet is ethernet
6799  *
6800  * Expects skb->data at mac header.
6801  *
6802  * Returns 0 on success, -errno otherwise.
6803  */
6804 int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len,
6805 		 bool ethernet)
6806 {
6807 	int err;
6808 
6809 	if (unlikely(!eth_p_mpls(skb->protocol)))
6810 		return 0;
6811 
6812 	err = skb_ensure_writable(skb, mac_len + MPLS_HLEN);
6813 	if (unlikely(err))
6814 		return err;
6815 
6816 	skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN);
6817 	memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb),
6818 		mac_len);
6819 
6820 	__skb_pull(skb, MPLS_HLEN);
6821 	skb_reset_mac_header(skb);
6822 	skb_set_network_header(skb, mac_len);
6823 
6824 	if (ethernet && mac_len >= ETH_HLEN) {
6825 		struct ethhdr *hdr;
6826 
6827 		/* use mpls_hdr() to get ethertype to account for VLANs. */
6828 		hdr = (struct ethhdr *)((void *)mpls_hdr(skb) - ETH_HLEN);
6829 		skb_mod_eth_type(skb, hdr, next_proto);
6830 	}
6831 	skb->protocol = next_proto;
6832 
6833 	/* The last label is gone, so the inner header recorded by
6834 	 * skb_mpls_push() no longer describes this packet. Drop it, or a
6835 	 * later push keeps the stale offset.
6836 	 */
6837 	if (!eth_p_mpls(next_proto))
6838 		skb->inner_protocol = 0;
6839 
6840 	return 0;
6841 }
6842 EXPORT_SYMBOL_GPL(skb_mpls_pop);
6843 
6844 /**
6845  * skb_mpls_update_lse() - modify outermost MPLS header and update csum
6846  *
6847  * @skb: buffer
6848  * @mpls_lse: new MPLS label stack entry to update to
6849  *
6850  * Expects skb->data at mac header.
6851  *
6852  * Returns 0 on success, -errno otherwise.
6853  */
6854 int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse)
6855 {
6856 	int err;
6857 
6858 	if (unlikely(!eth_p_mpls(skb->protocol)))
6859 		return -EINVAL;
6860 
6861 	err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN);
6862 	if (unlikely(err))
6863 		return err;
6864 
6865 	if (skb->ip_summed == CHECKSUM_COMPLETE) {
6866 		__be32 diff[] = { ~mpls_hdr(skb)->label_stack_entry, mpls_lse };
6867 
6868 		skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
6869 	}
6870 
6871 	mpls_hdr(skb)->label_stack_entry = mpls_lse;
6872 
6873 	return 0;
6874 }
6875 EXPORT_SYMBOL_GPL(skb_mpls_update_lse);
6876 
6877 /**
6878  * skb_mpls_dec_ttl() - decrement the TTL of the outermost MPLS header
6879  *
6880  * @skb: buffer
6881  *
6882  * Expects skb->data at mac header.
6883  *
6884  * Returns 0 on success, -errno otherwise.
6885  */
6886 int skb_mpls_dec_ttl(struct sk_buff *skb)
6887 {
6888 	u32 lse;
6889 	u8 ttl;
6890 
6891 	if (unlikely(!eth_p_mpls(skb->protocol)))
6892 		return -EINVAL;
6893 
6894 	if (!pskb_may_pull(skb, skb_network_offset(skb) + MPLS_HLEN))
6895 		return -ENOMEM;
6896 
6897 	lse = be32_to_cpu(mpls_hdr(skb)->label_stack_entry);
6898 	ttl = (lse & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT;
6899 	if (!ttl || !--ttl)
6900 		return -EINVAL;
6901 
6902 	lse &= ~MPLS_LS_TTL_MASK;
6903 	lse |= ttl << MPLS_LS_TTL_SHIFT;
6904 
6905 	return skb_mpls_update_lse(skb, cpu_to_be32(lse));
6906 }
6907 EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl);
6908 
6909 /**
6910  * alloc_skb_with_frags - allocate skb with page frags
6911  *
6912  * @header_len: size of linear part
6913  * @data_len: needed length in frags
6914  * @order: max page order desired.
6915  * @errcode: pointer to error code if any
6916  * @gfp_mask: allocation mask
6917  *
6918  * This can be used to allocate a paged skb, given a maximal order for frags.
6919  */
6920 struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
6921 				     unsigned long data_len,
6922 				     int order,
6923 				     int *errcode,
6924 				     gfp_t gfp_mask)
6925 {
6926 	unsigned long chunk;
6927 	struct sk_buff *skb;
6928 	struct page *page;
6929 	int nr_frags = 0;
6930 
6931 	*errcode = -EMSGSIZE;
6932 	if (unlikely(data_len > MAX_SKB_FRAGS * (PAGE_SIZE << order)))
6933 		return NULL;
6934 
6935 	*errcode = -ENOBUFS;
6936 	skb = alloc_skb(header_len, gfp_mask);
6937 	if (!skb)
6938 		return NULL;
6939 
6940 	while (data_len) {
6941 		if (nr_frags == MAX_SKB_FRAGS)
6942 			goto failure;
6943 		while (order && PAGE_ALIGN(data_len) < (PAGE_SIZE << order))
6944 			order--;
6945 
6946 		if (order) {
6947 			page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) |
6948 					   __GFP_COMP |
6949 					   __GFP_NOWARN,
6950 					   order);
6951 			if (!page) {
6952 				order--;
6953 				continue;
6954 			}
6955 		} else {
6956 			page = alloc_page(gfp_mask);
6957 			if (!page)
6958 				goto failure;
6959 		}
6960 		chunk = min_t(unsigned long, data_len,
6961 			      PAGE_SIZE << order);
6962 		skb_fill_page_desc(skb, nr_frags, page, 0, chunk);
6963 		nr_frags++;
6964 		skb->truesize += (PAGE_SIZE << order);
6965 		data_len -= chunk;
6966 	}
6967 	return skb;
6968 
6969 failure:
6970 	kfree_skb(skb);
6971 	return NULL;
6972 }
6973 EXPORT_SYMBOL(alloc_skb_with_frags);
6974 
6975 /* pskb_carve_inside_header() and pskb_carve_inside_nonlinear()
6976  * remove the first bytes of a packet and reallocate skb->head.
6977  *
6978  * Whatever headers were present before the operation are gone,
6979  * we must not leave stale offsets, otherwise users of this skb
6980  * (skb_dump(), drop_monitor, taps, ...) would read or pull garbage.
6981  */
6982 static void skb_carve_reset_headers(struct sk_buff *skb)
6983 {
6984 	skb_unset_mac_header(skb);
6985 	skb_unset_transport_header(skb);
6986 	skb_reset_network_header(skb);
6987 	skb->mac_len = 0;
6988 
6989 	/* Inner offsets have no "unset" marker, zero them so that
6990 	 * skb_inner_network_header_was_set() becomes false and no
6991 	 * consumer mistakes them for a real (and long gone) header.
6992 	 */
6993 	skb->inner_mac_header = 0;
6994 	skb->inner_network_header = 0;
6995 	skb->inner_transport_header = 0;
6996 	skb->inner_protocol = 0;
6997 	skb->encapsulation = 0;
6998 
6999 	if (skb->ip_summed == CHECKSUM_PARTIAL)
7000 		skb->ip_summed = CHECKSUM_NONE;
7001 }
7002 
7003 /* carve out the first off bytes from skb when off < headlen */
7004 static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,
7005 				    const int headlen, gfp_t gfp_mask)
7006 {
7007 	int i;
7008 	unsigned int size = skb_end_offset(skb);
7009 	int new_hlen = headlen - off;
7010 	u8 *data;
7011 
7012 	if (skb_pfmemalloc(skb))
7013 		gfp_mask |= __GFP_MEMALLOC;
7014 
7015 	data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL);
7016 	if (!data)
7017 		return -ENOMEM;
7018 	size = SKB_WITH_OVERHEAD(size);
7019 
7020 	/* Copy real data, and all frags */
7021 	skb_copy_from_linear_data_offset(skb, off, data, new_hlen);
7022 	skb->len -= off;
7023 
7024 	/* Remove SKBFL_MANAGED_FRAG_REFS instead of trying to honour it
7025 	 * while refcounting frags below.
7026 	 */
7027 	skb_zcopy_downgrade_managed(skb);
7028 
7029 	memcpy((struct skb_shared_info *)(data + size),
7030 	       skb_shinfo(skb),
7031 	       offsetof(struct skb_shared_info,
7032 			frags[skb_shinfo(skb)->nr_frags]));
7033 	if (skb_cloned(skb)) {
7034 		/* drop the old head gracefully */
7035 		if (skb_orphan_frags(skb, gfp_mask)) {
7036 			skb_kfree_head(data);
7037 			return -ENOMEM;
7038 		}
7039 		if (skb_zcopy(skb))
7040 			net_zcopy_get(skb_uarg(skb));
7041 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
7042 			skb_frag_ref(skb, i);
7043 		if (skb_has_frag_list(skb))
7044 			skb_clone_fraglist(skb);
7045 		skb_release_data(skb, SKB_CONSUMED);
7046 	} else {
7047 		/* we can reuse existing recount- all we did was
7048 		 * relocate values
7049 		 */
7050 		skb_free_head(skb);
7051 	}
7052 
7053 	skb->head = data;
7054 	skb->data = data;
7055 	skb->head_frag = 0;
7056 	skb_set_end_offset(skb, size);
7057 	skb_set_tail_pointer(skb, skb_headlen(skb));
7058 	skb_carve_reset_headers(skb);
7059 	skb->cloned = 0;
7060 	skb->hdr_len = 0;
7061 	skb->nohdr = 0;
7062 	atomic_set(&skb_shinfo(skb)->dataref, 1);
7063 
7064 	return 0;
7065 }
7066 
7067 static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp);
7068 
7069 /* carve out the first eat bytes from skb's frag_list. May recurse into
7070  * pskb_carve()
7071  */
7072 static int pskb_carve_frag_list(struct skb_shared_info *shinfo, int eat,
7073 				gfp_t gfp_mask)
7074 {
7075 	struct sk_buff *list = shinfo->frag_list;
7076 	struct sk_buff *clone = NULL;
7077 	struct sk_buff *insp = NULL;
7078 
7079 	do {
7080 		if (!list) {
7081 			pr_err("Not enough bytes to eat. Want %d\n", eat);
7082 			return -EFAULT;
7083 		}
7084 		if (list->len <= eat) {
7085 			/* Eaten as whole. */
7086 			eat -= list->len;
7087 			list = list->next;
7088 			insp = list;
7089 		} else {
7090 			/* Eaten partially. */
7091 			if (skb_shared(list)) {
7092 				clone = skb_clone(list, gfp_mask);
7093 				if (!clone)
7094 					return -ENOMEM;
7095 				insp = list->next;
7096 				list = clone;
7097 			} else {
7098 				/* This may be pulled without problems. */
7099 				insp = list;
7100 			}
7101 			if (pskb_carve(list, eat, gfp_mask) < 0) {
7102 				kfree_skb(clone);
7103 				return -ENOMEM;
7104 			}
7105 			break;
7106 		}
7107 	} while (eat);
7108 
7109 	/* Free pulled out fragments. */
7110 	while ((list = shinfo->frag_list) != insp) {
7111 		shinfo->frag_list = list->next;
7112 		consume_skb(list);
7113 	}
7114 	/* And insert new clone at head. */
7115 	if (clone) {
7116 		clone->next = list;
7117 		shinfo->frag_list = clone;
7118 	}
7119 	return 0;
7120 }
7121 
7122 /* carve off first len bytes from skb. Split line (off) is in the
7123  * non-linear part of skb
7124  */
7125 static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off,
7126 				       int pos, gfp_t gfp_mask)
7127 {
7128 	int i, k = 0;
7129 	unsigned int size = skb_end_offset(skb);
7130 	u8 *data;
7131 	const int nfrags = skb_shinfo(skb)->nr_frags;
7132 	struct skb_shared_info *shinfo;
7133 
7134 	if (skb_pfmemalloc(skb))
7135 		gfp_mask |= __GFP_MEMALLOC;
7136 
7137 	data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL);
7138 	if (!data)
7139 		return -ENOMEM;
7140 	size = SKB_WITH_OVERHEAD(size);
7141 
7142 	/* Remove SKBFL_MANAGED_FRAG_REFS instead of trying to honour it
7143 	 * while refcounting frags below.
7144 	 */
7145 	skb_zcopy_downgrade_managed(skb);
7146 
7147 	memcpy((struct skb_shared_info *)(data + size),
7148 	       skb_shinfo(skb), offsetof(struct skb_shared_info, frags[0]));
7149 	if (skb_orphan_frags(skb, gfp_mask)) {
7150 		skb_kfree_head(data);
7151 		return -ENOMEM;
7152 	}
7153 	shinfo = (struct skb_shared_info *)(data + size);
7154 	for (i = 0; i < nfrags; i++) {
7155 		int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]);
7156 
7157 		if (pos + fsize > off) {
7158 			shinfo->frags[k] = skb_shinfo(skb)->frags[i];
7159 
7160 			if (pos < off) {
7161 				/* Split frag.
7162 				 * We have two variants in this case:
7163 				 * 1. Move all the frag to the second
7164 				 *    part, if it is possible. F.e.
7165 				 *    this approach is mandatory for TUX,
7166 				 *    where splitting is expensive.
7167 				 * 2. Split is accurately. We make this.
7168 				 */
7169 				skb_frag_off_add(&shinfo->frags[0], off - pos);
7170 				skb_frag_size_sub(&shinfo->frags[0], off - pos);
7171 			}
7172 			skb_frag_ref(skb, i);
7173 			k++;
7174 		}
7175 		pos += fsize;
7176 	}
7177 	shinfo->nr_frags = k;
7178 	if (skb_has_frag_list(skb))
7179 		skb_clone_fraglist(skb);
7180 
7181 	/* split line is in frag list */
7182 	if (k == 0 && pskb_carve_frag_list(shinfo, off - pos, gfp_mask)) {
7183 		/* skb_frag_unref() is not needed here as shinfo->nr_frags = 0. */
7184 		if (skb_has_frag_list(skb))
7185 			kfree_skb_list(skb_shinfo(skb)->frag_list);
7186 		skb_kfree_head(data);
7187 		return -ENOMEM;
7188 	}
7189 	if (skb_zcopy(skb))
7190 		net_zcopy_get(skb_uarg(skb));
7191 	skb_release_data(skb, SKB_CONSUMED);
7192 
7193 	skb->head = data;
7194 	skb->head_frag = 0;
7195 	skb->data = data;
7196 	skb_set_end_offset(skb, size);
7197 	skb_reset_tail_pointer(skb);
7198 	skb_carve_reset_headers(skb);
7199 	skb->cloned   = 0;
7200 	skb->hdr_len  = 0;
7201 	skb->nohdr    = 0;
7202 	skb->len -= off;
7203 	skb->data_len = skb->len;
7204 	atomic_set(&skb_shinfo(skb)->dataref, 1);
7205 	return 0;
7206 }
7207 
7208 /* remove len bytes from the beginning of the skb */
7209 static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp)
7210 {
7211 	int headlen = skb_headlen(skb);
7212 
7213 	if (len < headlen)
7214 		return pskb_carve_inside_header(skb, len, headlen, gfp);
7215 	else
7216 		return pskb_carve_inside_nonlinear(skb, len, headlen, gfp);
7217 }
7218 
7219 /* Extract to_copy bytes starting at off from skb, and return this in
7220  * a new skb
7221  */
7222 struct sk_buff *pskb_extract(struct sk_buff *skb, int off,
7223 			     int to_copy, gfp_t gfp)
7224 {
7225 	struct sk_buff  *clone = skb_clone(skb, gfp);
7226 
7227 	if (!clone)
7228 		return NULL;
7229 
7230 	if (pskb_carve(clone, off, gfp) < 0 ||
7231 	    pskb_trim(clone, to_copy)) {
7232 		kfree_skb(clone);
7233 		return NULL;
7234 	}
7235 	return clone;
7236 }
7237 EXPORT_SYMBOL(pskb_extract);
7238 
7239 /**
7240  * skb_condense - try to get rid of fragments/frag_list if possible
7241  * @skb: buffer
7242  *
7243  * Can be used to save memory before skb is added to a busy queue.
7244  * If packet has bytes in frags and enough tail room in skb->head,
7245  * pull all of them, so that we can free the frags right now and adjust
7246  * truesize.
7247  * Notes:
7248  *	We do not reallocate skb->head thus can not fail.
7249  *	Caller must re-evaluate skb->truesize if needed.
7250  */
7251 void skb_condense(struct sk_buff *skb)
7252 {
7253 	if (skb->data_len) {
7254 		if (skb->data_len > skb->end - skb->tail ||
7255 		    skb_cloned(skb) || !skb_frags_readable(skb))
7256 			return;
7257 
7258 		/* Nice, we can free page frag(s) right now */
7259 		__pskb_pull_tail(skb, skb->data_len);
7260 	}
7261 	/* At this point, skb->truesize might be over estimated,
7262 	 * because skb had a fragment, and fragments do not tell
7263 	 * their truesize.
7264 	 * When we pulled its content into skb->head, fragment
7265 	 * was freed, but __pskb_pull_tail() could not possibly
7266 	 * adjust skb->truesize, not knowing the frag truesize.
7267 	 */
7268 	skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
7269 }
7270 EXPORT_SYMBOL(skb_condense);
7271 
7272 #ifdef CONFIG_SKB_EXTENSIONS
7273 static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id)
7274 {
7275 	return (void *)ext + (ext->offset[id] * SKB_EXT_ALIGN_VALUE);
7276 }
7277 
7278 /**
7279  * __skb_ext_alloc - allocate a new skb extensions storage
7280  *
7281  * @flags: See kmalloc().
7282  *
7283  * Returns the newly allocated pointer. The pointer can later attached to a
7284  * skb via __skb_ext_set().
7285  * Note: caller must handle the skb_ext as an opaque data.
7286  */
7287 struct skb_ext *__skb_ext_alloc(gfp_t flags)
7288 {
7289 	struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, flags);
7290 
7291 	if (new) {
7292 		memset(new->offset, 0, sizeof(new->offset));
7293 		refcount_set(&new->refcnt, 1);
7294 	}
7295 
7296 	return new;
7297 }
7298 
7299 static struct skb_ext *skb_ext_maybe_cow(struct skb_ext *old,
7300 					 unsigned int old_active)
7301 {
7302 	struct skb_ext *new;
7303 
7304 	if (refcount_read(&old->refcnt) == 1)
7305 		return old;
7306 
7307 	new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC);
7308 	if (!new)
7309 		return NULL;
7310 
7311 	memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE);
7312 	refcount_set(&new->refcnt, 1);
7313 
7314 #ifdef CONFIG_XFRM
7315 	if (old_active & (1 << SKB_EXT_SEC_PATH)) {
7316 		struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH);
7317 		unsigned int i;
7318 
7319 		for (i = 0; i < sp->len; i++)
7320 			xfrm_state_hold(sp->xvec[i]);
7321 	}
7322 #endif
7323 #ifdef CONFIG_MCTP_FLOWS
7324 	if (old_active & (1 << SKB_EXT_MCTP)) {
7325 		struct mctp_flow *flow = skb_ext_get_ptr(old, SKB_EXT_MCTP);
7326 
7327 		if (flow->key)
7328 			refcount_inc(&flow->key->refs);
7329 	}
7330 #endif
7331 	__skb_ext_put(old);
7332 	return new;
7333 }
7334 
7335 /**
7336  * __skb_ext_set - attach the specified extension storage to this skb
7337  * @skb: buffer
7338  * @id: extension id
7339  * @ext: extension storage previously allocated via __skb_ext_alloc()
7340  *
7341  * Existing extensions, if any, are cleared.
7342  *
7343  * Returns the pointer to the extension.
7344  */
7345 void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id,
7346 		    struct skb_ext *ext)
7347 {
7348 	unsigned int newlen, newoff = SKB_EXT_CHUNKSIZEOF(*ext);
7349 
7350 	skb_ext_put(skb);
7351 	newlen = newoff + skb_ext_type_len[id];
7352 	ext->chunks = newlen;
7353 	ext->offset[id] = newoff;
7354 	skb->extensions = ext;
7355 	skb->active_extensions = 1 << id;
7356 	return skb_ext_get_ptr(ext, id);
7357 }
7358 EXPORT_SYMBOL_NS_GPL(__skb_ext_set, "NETDEV_INTERNAL");
7359 
7360 /**
7361  * skb_ext_add - allocate space for given extension, COW if needed
7362  * @skb: buffer
7363  * @id: extension to allocate space for
7364  *
7365  * Allocates enough space for the given extension.
7366  * If the extension is already present, a pointer to that extension
7367  * is returned.
7368  *
7369  * If the skb was cloned, COW applies and the returned memory can be
7370  * modified without changing the extension space of clones buffers.
7371  *
7372  * Returns pointer to the extension or NULL on allocation failure.
7373  */
7374 void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id)
7375 {
7376 	struct skb_ext *new, *old = NULL;
7377 	unsigned int newlen, newoff;
7378 
7379 	if (skb->active_extensions) {
7380 		old = skb->extensions;
7381 
7382 		new = skb_ext_maybe_cow(old, skb->active_extensions);
7383 		if (!new)
7384 			return NULL;
7385 
7386 		if (__skb_ext_exist(new, id))
7387 			goto set_active;
7388 
7389 		newoff = new->chunks;
7390 	} else {
7391 		newoff = SKB_EXT_CHUNKSIZEOF(*new);
7392 
7393 		new = __skb_ext_alloc(GFP_ATOMIC);
7394 		if (!new)
7395 			return NULL;
7396 	}
7397 
7398 	newlen = newoff + skb_ext_type_len[id];
7399 	new->chunks = newlen;
7400 	new->offset[id] = newoff;
7401 set_active:
7402 	skb->slow_gro = 1;
7403 	skb->extensions = new;
7404 	skb->active_extensions |= 1 << id;
7405 	return skb_ext_get_ptr(new, id);
7406 }
7407 EXPORT_SYMBOL(skb_ext_add);
7408 
7409 #ifdef CONFIG_XFRM
7410 static void skb_ext_put_sp(struct sec_path *sp)
7411 {
7412 	unsigned int i;
7413 
7414 	if (!sp->len)
7415 		return;
7416 
7417 	for (i = 0; i < sp->len; i++)
7418 		xfrm_state_put(sp->xvec[i]);
7419 	sp->len = 0;
7420 }
7421 #endif
7422 
7423 #ifdef CONFIG_MCTP_FLOWS
7424 static void skb_ext_put_mctp(struct mctp_flow *flow)
7425 {
7426 	if (!flow->key)
7427 		return;
7428 
7429 	mctp_key_unref(flow->key);
7430 	flow->key = NULL;
7431 }
7432 #endif
7433 
7434 void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
7435 {
7436 	struct skb_ext *ext = skb->extensions;
7437 
7438 	skb->active_extensions &= ~(1 << id);
7439 	if (skb->active_extensions == 0) {
7440 		skb->extensions = NULL;
7441 		__skb_ext_put(ext);
7442 		return;
7443 	}
7444 
7445 	if (refcount_read(&ext->refcnt) > 1)
7446 		return;
7447 
7448 #ifdef CONFIG_XFRM
7449 	if (id == SKB_EXT_SEC_PATH)
7450 		skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH));
7451 #endif
7452 #ifdef CONFIG_MCTP_FLOWS
7453 	if (id == SKB_EXT_MCTP)
7454 		skb_ext_put_mctp(skb_ext_get_ptr(ext, SKB_EXT_MCTP));
7455 #endif
7456 }
7457 EXPORT_SYMBOL(__skb_ext_del);
7458 
7459 void __skb_ext_put(struct skb_ext *ext)
7460 {
7461 	/* If this is last clone, nothing can increment
7462 	 * it after check passes.  Avoids one atomic op.
7463 	 */
7464 	if (refcount_read(&ext->refcnt) == 1)
7465 		goto free_now;
7466 
7467 	if (!refcount_dec_and_test(&ext->refcnt))
7468 		return;
7469 free_now:
7470 #ifdef CONFIG_XFRM
7471 	if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH))
7472 		skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH));
7473 #endif
7474 #ifdef CONFIG_MCTP_FLOWS
7475 	if (__skb_ext_exist(ext, SKB_EXT_MCTP))
7476 		skb_ext_put_mctp(skb_ext_get_ptr(ext, SKB_EXT_MCTP));
7477 #endif
7478 
7479 	kmem_cache_free(skbuff_ext_cache, ext);
7480 }
7481 EXPORT_SYMBOL(__skb_ext_put);
7482 #endif /* CONFIG_SKB_EXTENSIONS */
7483 
7484 static void kfree_skb_napi_cache(struct sk_buff *skb)
7485 {
7486 	/* if SKB is a clone, don't handle this case */
7487 	if (skb->fclone != SKB_FCLONE_UNAVAILABLE) {
7488 		__kfree_skb(skb);
7489 		return;
7490 	}
7491 
7492 	local_bh_disable();
7493 	__napi_kfree_skb(skb, SKB_CONSUMED);
7494 	local_bh_enable();
7495 }
7496 
7497 DEFINE_STATIC_KEY_FALSE(skb_defer_disable_key);
7498 
7499 /**
7500  * skb_attempt_defer_free - queue skb for remote freeing
7501  * @skb: buffer
7502  *
7503  * Put @skb in a per-cpu list, using the cpu which
7504  * allocated the skb/pages to reduce false sharing
7505  * and memory zone spinlock contention.
7506  */
7507 void skb_attempt_defer_free(struct sk_buff *skb)
7508 {
7509 	struct skb_defer_node *sdn;
7510 	unsigned long defer_count;
7511 	unsigned int defer_max;
7512 	int cpu, my_cpu;
7513 	bool kick;
7514 
7515 	if (static_branch_unlikely(&skb_defer_disable_key))
7516 		goto nodefer;
7517 
7518 	/* zero copy notifications should not be delayed. */
7519 	if (skb_zcopy(skb))
7520 		goto nodefer;
7521 
7522 	cpu = skb->alloc_cpu;
7523 	my_cpu = raw_smp_processor_id();
7524 	if (cpu == my_cpu ||
7525 	    WARN_ON_ONCE(cpu >= nr_cpu_ids) ||
7526 	    !cpu_online(cpu)) {
7527 nodefer:	kfree_skb_napi_cache(skb);
7528 		return;
7529 	}
7530 
7531 	DEBUG_NET_WARN_ON_ONCE(skb_dst(skb));
7532 	DEBUG_NET_WARN_ON_ONCE(skb->destructor);
7533 	DEBUG_NET_WARN_ON_ONCE(skb_nfct(skb));
7534 
7535 	sdn = per_cpu_ptr(net_hotdata.skb_defer_nodes, cpu) + cpu_to_node(my_cpu);
7536 
7537 	defer_max = READ_ONCE(net_hotdata.sysctl_skb_defer_max);
7538 	defer_count = atomic_long_inc_return(&sdn->defer_count);
7539 
7540 	if (defer_count > defer_max) {
7541 		/* Undo the reservation. An llist_add() does not release one --
7542 		 * the skb stays charged until a flush subtracts the batch that
7543 		 * contains it -- so a rejected free must undo its own.
7544 		 */
7545 		atomic_long_dec(&sdn->defer_count);
7546 		goto nodefer;
7547 	}
7548 
7549 	llist_add(&skb->ll_node, &sdn->defer_list);
7550 
7551 	if (unlikely(!cpu_online(cpu) || my_cpu != raw_smp_processor_id())) {
7552 		skb_defer_node_flush(sdn);
7553 		return;
7554 	}
7555 
7556 	/* Half capacity, plus the skb that fills the queue. Equality alone can
7557 	 * be consumed early -- if the producer holding the kick value publishes
7558 	 * first, its purge drains only that skb and the reservations published
7559 	 * after it match nothing -- which would leave a full queue with nothing
7560 	 * pending.
7561 	 */
7562 	kick = (defer_count - 1) == (defer_max >> 1) || defer_count == defer_max;
7563 
7564 	/* Make sure to trigger NET_RX_SOFTIRQ on the remote CPU
7565 	 * if we are unlucky enough (this seems very unlikely).
7566 	 */
7567 	if (unlikely(kick))
7568 		kick_defer_list_purge(cpu);
7569 }
7570 
7571 static void skb_splice_csum_page(struct sk_buff *skb, struct page *page,
7572 				 size_t offset, size_t len,
7573 				 unsigned int csum_offset)
7574 {
7575 	const char *kaddr;
7576 	__wsum csum;
7577 
7578 	kaddr = kmap_local_page(page);
7579 	csum = csum_partial(kaddr + offset, len, 0);
7580 	kunmap_local(kaddr);
7581 	skb->csum = csum_block_add(skb->csum, csum, csum_offset);
7582 }
7583 
7584 /**
7585  * skb_splice_from_iter - Splice (or copy) pages to skbuff
7586  * @skb: The buffer to add pages to
7587  * @iter: Iterator representing the pages to be added
7588  * @maxsize: Maximum amount of pages to be added
7589  *
7590  * This is a common helper function for supporting MSG_SPLICE_PAGES.  It
7591  * extracts pages from an iterator and adds them to the socket buffer if
7592  * possible, copying them to fragments if not possible (such as if they're slab
7593  * pages).
7594  *
7595  * Returns the amount of data spliced/copied or -EMSGSIZE if there's
7596  * insufficient space in the buffer to transfer anything.
7597  */
7598 ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter,
7599 			     ssize_t maxsize)
7600 {
7601 	size_t frag_limit = READ_ONCE(net_hotdata.sysctl_max_skb_frags);
7602 	struct page *pages[8], **ppages = pages;
7603 	ssize_t spliced = 0, ret = 0;
7604 	unsigned int i;
7605 
7606 	while (iter->count > 0) {
7607 		ssize_t space, nr, len;
7608 		size_t off;
7609 
7610 		ret = -EMSGSIZE;
7611 		space = frag_limit - skb_shinfo(skb)->nr_frags;
7612 		if (space < 0)
7613 			break;
7614 
7615 		/* We might be able to coalesce without increasing nr_frags */
7616 		nr = clamp_t(size_t, space, 1, ARRAY_SIZE(pages));
7617 
7618 		len = iov_iter_extract_pages(iter, &ppages, maxsize, nr, 0, &off);
7619 		if (len <= 0) {
7620 			ret = len ?: -EIO;
7621 			break;
7622 		}
7623 
7624 		i = 0;
7625 		do {
7626 			struct page *page = pages[i++];
7627 			size_t part = min_t(size_t, PAGE_SIZE - off, len);
7628 
7629 			ret = -EIO;
7630 			if (WARN_ON_ONCE(!sendpage_ok(page)))
7631 				goto out;
7632 
7633 			ret = skb_append_pagefrags(skb, page, off, part,
7634 						   frag_limit);
7635 			if (ret < 0) {
7636 				iov_iter_revert(iter, len);
7637 				goto out;
7638 			}
7639 
7640 			if (skb->ip_summed == CHECKSUM_NONE)
7641 				skb_splice_csum_page(skb, page, off, part,
7642 						     skb->len + spliced);
7643 
7644 			off = 0;
7645 			spliced += part;
7646 			maxsize -= part;
7647 			len -= part;
7648 		} while (len > 0);
7649 
7650 		if (maxsize <= 0)
7651 			break;
7652 	}
7653 
7654 out:
7655 	skb_len_add(skb, spliced);
7656 	return spliced ?: ret;
7657 }
7658 EXPORT_SYMBOL(skb_splice_from_iter);
7659 
7660 static __always_inline
7661 size_t memcpy_from_iter_csum(void *iter_from, size_t progress,
7662 			     size_t len, void *to, void *priv2)
7663 {
7664 	__wsum *csum = priv2;
7665 	__wsum next = csum_partial_copy_nocheck(iter_from, to + progress, len);
7666 
7667 	*csum = csum_block_add(*csum, next, progress);
7668 	return 0;
7669 }
7670 
7671 static __always_inline
7672 size_t copy_from_user_iter_csum(void __user *iter_from, size_t progress,
7673 				size_t len, void *to, void *priv2)
7674 {
7675 	__wsum next, *csum = priv2;
7676 
7677 	next = csum_and_copy_from_user(iter_from, to + progress, len);
7678 	*csum = csum_block_add(*csum, next, progress);
7679 	return next ? 0 : len;
7680 }
7681 
7682 bool csum_and_copy_from_iter_full(void *addr, size_t bytes,
7683 				  __wsum *csum, struct iov_iter *i)
7684 {
7685 	size_t copied;
7686 
7687 	if (WARN_ON_ONCE(!i->data_source))
7688 		return false;
7689 	copied = iterate_and_advance2(i, bytes, addr, csum,
7690 				      copy_from_user_iter_csum,
7691 				      memcpy_from_iter_csum);
7692 	if (likely(copied == bytes))
7693 		return true;
7694 	iov_iter_revert(i, copied);
7695 	return false;
7696 }
7697 EXPORT_SYMBOL(csum_and_copy_from_iter_full);
7698 
7699 /**
7700  * net_iov_inc_ref - acquire a non-page_pool reference on a net_iov
7701  * @niov: net_iov to reference
7702  *
7703  * Acquires a non-pp backing reference (binding->ref) on @niov regardless of
7704  * page_pool membership, warning if @niov does not support non-pp references.
7705  * Counterpart to net_iov_dec_ref().
7706  */
7707 void net_iov_inc_ref(struct net_iov *niov)
7708 {
7709 	if (net_is_devmem_iov(niov))
7710 		net_devmem_net_iov_inc_ref(niov);
7711 	else
7712 		DEBUG_NET_WARN_ON_ONCE(true);
7713 }
7714 EXPORT_SYMBOL(net_iov_inc_ref);
7715 
7716 /**
7717  * net_iov_dec_ref - release a non-page_pool reference on a net_iov
7718  * @niov: net_iov to unreference
7719  *
7720  * Drops a non-pp backing reference (binding->ref) on @niov regardless of
7721  * page_pool membership, warning if @niov does not support non-pp references.
7722  * Counterpart to net_iov_inc_ref().
7723  */
7724 void net_iov_dec_ref(struct net_iov *niov)
7725 {
7726 	if (net_is_devmem_iov(niov))
7727 		net_devmem_net_iov_dec_ref(niov);
7728 	else
7729 		DEBUG_NET_WARN_ON_ONCE(true);
7730 }
7731 EXPORT_SYMBOL(net_iov_dec_ref);
7732 
7733 struct vlan_type_depth __vlan_get_protocol_offset(const struct sk_buff *skb,
7734 						  __be16 type,
7735 						  int mac_offset)
7736 {
7737 	unsigned int vlan_depth = skb->mac_len, parse_depth = VLAN_MAX_DEPTH;
7738 
7739 	/* if type is 802.1Q/AD then the header should already be
7740 	 * present at mac_len - VLAN_HLEN (if mac_len > 0), or at
7741 	 * ETH_HLEN otherwise
7742 	 */
7743 	if (vlan_depth) {
7744 		if (WARN_ON_ONCE(vlan_depth < VLAN_HLEN))
7745 			return (struct vlan_type_depth) { 0 };
7746 		vlan_depth -= VLAN_HLEN;
7747 	} else {
7748 		vlan_depth = ETH_HLEN;
7749 	}
7750 	do {
7751 		struct vlan_hdr vhdr, *vh;
7752 
7753 		vh = skb_header_pointer(skb, mac_offset + vlan_depth,
7754 					sizeof(vhdr), &vhdr);
7755 		if (unlikely(!vh || !--parse_depth))
7756 			return (struct vlan_type_depth) { 0 };
7757 
7758 		type = vh->h_vlan_encapsulated_proto;
7759 		vlan_depth += VLAN_HLEN;
7760 	} while (eth_type_vlan(type));
7761 
7762 	return (struct vlan_type_depth) {
7763 		.type = type,
7764 		.depth = vlan_depth
7765 	};
7766 }
7767 EXPORT_SYMBOL(__vlan_get_protocol_offset);
7768