xref: /linux/net/core/skbuff.c (revision abccf171fa8e97954e74934bde811d76a5410cbd)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  *	Routines having to do with the 'struct sk_buff' memory handlers.
4  *
5  *	Authors:	Alan Cox <alan@lxorguk.ukuu.org.uk>
6  *			Florian La Roche <rzsfl@rz.uni-sb.de>
7  *
8  *	Fixes:
9  *		Alan Cox	:	Fixed the worst of the load
10  *					balancer bugs.
11  *		Dave Platt	:	Interrupt stacking fix.
12  *	Richard Kooijman	:	Timestamp fixes.
13  *		Alan Cox	:	Changed buffer format.
14  *		Alan Cox	:	destructor hook for AF_UNIX etc.
15  *		Linus Torvalds	:	Better skb_clone.
16  *		Alan Cox	:	Added skb_copy.
17  *		Alan Cox	:	Added all the changed routines Linus
18  *					only put in the headers
19  *		Ray VanTassle	:	Fixed --skb->lock in free
20  *		Alan Cox	:	skb_copy copy arp field
21  *		Andi Kleen	:	slabified it.
22  *		Robert Olsson	:	Removed skb_head_pool
23  *
24  *	NOTE:
25  *		The __skb_ routines should be called with interrupts
26  *	disabled, or you better be *real* sure that the operation is atomic
27  *	with respect to whatever list is being frobbed (e.g. via lock_sock()
28  *	or via disabling bottom half handlers, etc).
29  */
30 
31 /*
32  *	The functions in this file will not compile correctly with gcc 2.4.x
33  */
34 
35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36 
37 #include <linux/module.h>
38 #include <linux/types.h>
39 #include <linux/kernel.h>
40 #include <linux/mm.h>
41 #include <linux/interrupt.h>
42 #include <linux/in.h>
43 #include <linux/inet.h>
44 #include <linux/slab.h>
45 #include <linux/tcp.h>
46 #include <linux/udp.h>
47 #include <linux/sctp.h>
48 #include <linux/netdevice.h>
49 #ifdef CONFIG_NET_CLS_ACT
50 #include <net/pkt_sched.h>
51 #endif
52 #include <linux/string.h>
53 #include <linux/skbuff.h>
54 #include <linux/skbuff_ref.h>
55 #include <linux/splice.h>
56 #include <linux/cache.h>
57 #include <linux/rtnetlink.h>
58 #include <linux/init.h>
59 #include <linux/scatterlist.h>
60 #include <linux/errqueue.h>
61 #include <linux/prefetch.h>
62 #include <linux/bitfield.h>
63 #include <linux/if_vlan.h>
64 #include <linux/mpls.h>
65 #include <linux/kcov.h>
66 #include <linux/iov_iter.h>
67 #include <linux/crc32.h>
68 #include <kunit/visibility.h>
69 
70 #include <net/protocol.h>
71 #include <net/dst.h>
72 #include <net/sock.h>
73 #include <net/checksum.h>
74 #include <net/gro.h>
75 #include <net/gso.h>
76 #include <net/hotdata.h>
77 #include <net/ip6_checksum.h>
78 #include <net/xfrm.h>
79 #include <net/mpls.h>
80 #include <net/mptcp.h>
81 #include <net/mctp.h>
82 #include <net/tcp.h>
83 #include <net/can.h>
84 #include <net/page_pool/helpers.h>
85 #include <net/psp/types.h>
86 #include <net/dropreason.h>
87 #include <net/xdp_sock.h>
88 
89 #include <linux/uaccess.h>
90 #include <trace/events/skb.h>
91 #include <linux/highmem.h>
92 #include <linux/capability.h>
93 #include <linux/user_namespace.h>
94 #include <linux/indirect_call_wrapper.h>
95 #include <linux/textsearch.h>
96 
97 #include "dev.h"
98 #include "devmem.h"
99 #include "net-sysfs.h"
100 #include "netmem_priv.h"
101 
102 #ifdef CONFIG_SKB_EXTENSIONS
103 static struct kmem_cache *skbuff_ext_cache __ro_after_init;
104 #endif
105 
106 #define GRO_MAX_HEAD_PAD (GRO_MAX_HEAD + NET_SKB_PAD + NET_IP_ALIGN)
107 #define SKB_SMALL_HEAD_SIZE SKB_HEAD_ALIGN(max(MAX_TCP_HEADER, \
108 					       GRO_MAX_HEAD_PAD))
109 
110 /* SKB_SMALL_HEAD_CACHE_SIZE is the size used for the skbuff_small_head
111  * kmem_cache. The non-power-of-2 padding is kept for historical reasons and
112  * to avoid potential collisions with generic kmalloc bucket sizes.
113  */
114 #define SKB_SMALL_HEAD_CACHE_SIZE					\
115 	(is_power_of_2(SKB_SMALL_HEAD_SIZE) ?			\
116 		(SKB_SMALL_HEAD_SIZE + L1_CACHE_BYTES) :	\
117 		SKB_SMALL_HEAD_SIZE)
118 
119 #define SKB_SMALL_HEAD_HEADROOM						\
120 	SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE)
121 
122 /* kcm_write_msgs() relies on casting paged frags to bio_vec to use
123  * iov_iter_bvec(). These static asserts ensure the cast is valid is long as the
124  * netmem is a page.
125  */
126 static_assert(offsetof(struct bio_vec, bv_page) ==
127 	      offsetof(skb_frag_t, netmem));
128 static_assert(sizeof_field(struct bio_vec, bv_page) ==
129 	      sizeof_field(skb_frag_t, netmem));
130 
131 static_assert(offsetof(struct bio_vec, bv_len) == offsetof(skb_frag_t, len));
132 static_assert(sizeof_field(struct bio_vec, bv_len) ==
133 	      sizeof_field(skb_frag_t, len));
134 
135 static_assert(offsetof(struct bio_vec, bv_offset) ==
136 	      offsetof(skb_frag_t, offset));
137 static_assert(sizeof_field(struct bio_vec, bv_offset) ==
138 	      sizeof_field(skb_frag_t, offset));
139 
140 #undef FN
141 #define FN(reason) [SKB_DROP_REASON_##reason] = #reason,
142 static const char * const drop_reasons[] = {
143 	[SKB_CONSUMED] = "CONSUMED",
144 	DEFINE_DROP_REASON(FN, FN)
145 };
146 
147 static const struct drop_reason_list drop_reasons_core = {
148 	.reasons = drop_reasons,
149 	.n_reasons = ARRAY_SIZE(drop_reasons),
150 };
151 
152 const struct drop_reason_list __rcu *
153 drop_reasons_by_subsys[SKB_DROP_REASON_SUBSYS_NUM] = {
154 	[SKB_DROP_REASON_SUBSYS_CORE] = RCU_INITIALIZER(&drop_reasons_core),
155 };
156 EXPORT_SYMBOL(drop_reasons_by_subsys);
157 
158 /**
159  * drop_reasons_register_subsys - register another drop reason subsystem
160  * @subsys: the subsystem to register, must not be the core
161  * @list: the list of drop reasons within the subsystem, must point to
162  *	a statically initialized list
163  */
164 void drop_reasons_register_subsys(enum skb_drop_reason_subsys subsys,
165 				  const struct drop_reason_list *list)
166 {
167 	if (WARN(subsys <= SKB_DROP_REASON_SUBSYS_CORE ||
168 		 subsys >= ARRAY_SIZE(drop_reasons_by_subsys),
169 		 "invalid subsystem %d\n", subsys))
170 		return;
171 
172 	/* must point to statically allocated memory, so INIT is OK */
173 	RCU_INIT_POINTER(drop_reasons_by_subsys[subsys], list);
174 }
175 EXPORT_SYMBOL_GPL(drop_reasons_register_subsys);
176 
177 /**
178  * drop_reasons_unregister_subsys - unregister a drop reason subsystem
179  * @subsys: the subsystem to remove, must not be the core
180  *
181  * Note: This will synchronize_rcu() to ensure no users when it returns.
182  */
183 void drop_reasons_unregister_subsys(enum skb_drop_reason_subsys subsys)
184 {
185 	if (WARN(subsys <= SKB_DROP_REASON_SUBSYS_CORE ||
186 		 subsys >= ARRAY_SIZE(drop_reasons_by_subsys),
187 		 "invalid subsystem %d\n", subsys))
188 		return;
189 
190 	RCU_INIT_POINTER(drop_reasons_by_subsys[subsys], NULL);
191 
192 	synchronize_rcu();
193 }
194 EXPORT_SYMBOL_GPL(drop_reasons_unregister_subsys);
195 
196 /**
197  *	skb_panic - private function for out-of-line support
198  *	@skb:	buffer
199  *	@sz:	size
200  *	@addr:	address
201  *	@msg:	skb_over_panic or skb_under_panic
202  *
203  *	Out-of-line support for skb_put() and skb_push().
204  *	Called via the wrapper skb_over_panic() or skb_under_panic().
205  *	Keep out of line to prevent kernel bloat.
206  *	__builtin_return_address is not used because it is not always reliable.
207  */
208 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
209 		      const char msg[])
210 {
211 	pr_emerg("%s: text:%px len:%d put:%d head:%px data:%px tail:%#lx end:%#lx dev:%s\n",
212 		 msg, addr, skb->len, sz, skb->head, skb->data,
213 		 (unsigned long)skb->tail, (unsigned long)skb->end,
214 		 skb->dev ? skb->dev->name : "<NULL>");
215 	BUG();
216 }
217 
218 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
219 {
220 	skb_panic(skb, sz, addr, __func__);
221 }
222 
223 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
224 {
225 	skb_panic(skb, sz, addr, __func__);
226 }
227 
228 #define NAPI_SKB_CACHE_SIZE	128
229 #define NAPI_SKB_CACHE_BULK	32
230 #define NAPI_SKB_CACHE_FREE	32
231 
232 struct napi_alloc_cache {
233 	local_lock_t bh_lock;
234 	struct page_frag_cache page;
235 	unsigned int skb_count;
236 	void *skb_cache[NAPI_SKB_CACHE_SIZE];
237 };
238 
239 static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
240 static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache) = {
241 	.bh_lock = INIT_LOCAL_LOCK(bh_lock),
242 };
243 
244 void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask)
245 {
246 	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
247 	void *data;
248 
249 	fragsz = SKB_DATA_ALIGN(fragsz);
250 
251 	local_lock_nested_bh(&napi_alloc_cache.bh_lock);
252 	data = __page_frag_alloc_align(&nc->page, fragsz,
253 				       GFP_ATOMIC | __GFP_NOWARN, align_mask);
254 	local_unlock_nested_bh(&napi_alloc_cache.bh_lock);
255 	return data;
256 
257 }
258 EXPORT_SYMBOL(__napi_alloc_frag_align);
259 
260 void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask)
261 {
262 	void *data;
263 
264 	if (in_hardirq() || irqs_disabled()) {
265 		struct page_frag_cache *nc = this_cpu_ptr(&netdev_alloc_cache);
266 
267 		fragsz = SKB_DATA_ALIGN(fragsz);
268 		data = __page_frag_alloc_align(nc, fragsz,
269 					       GFP_ATOMIC | __GFP_NOWARN,
270 					       align_mask);
271 	} else {
272 		local_bh_disable();
273 		data = __napi_alloc_frag_align(fragsz, align_mask);
274 		local_bh_enable();
275 	}
276 	return data;
277 }
278 EXPORT_SYMBOL(__netdev_alloc_frag_align);
279 
280 /* Cache kmem_cache_size(net_hotdata.skbuff_cache) to help the compiler
281  * remove dead code (and skbuff_cache_size) when CONFIG_KASAN is unset.
282  */
283 static u32 skbuff_cache_size __read_mostly;
284 
285 static inline struct sk_buff *napi_skb_cache_get(bool alloc)
286 {
287 	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
288 	struct sk_buff *skb;
289 
290 	local_lock_nested_bh(&napi_alloc_cache.bh_lock);
291 	if (unlikely(!nc->skb_count)) {
292 		if (alloc && kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
293 						   GFP_ATOMIC | __GFP_NOWARN,
294 						   NAPI_SKB_CACHE_BULK,
295 						   nc->skb_cache))
296 			nc->skb_count = NAPI_SKB_CACHE_BULK;
297 		if (unlikely(!nc->skb_count)) {
298 			local_unlock_nested_bh(&napi_alloc_cache.bh_lock);
299 			return NULL;
300 		}
301 	}
302 
303 	skb = nc->skb_cache[--nc->skb_count];
304 	if (nc->skb_count)
305 		prefetch(nc->skb_cache[nc->skb_count - 1]);
306 	local_unlock_nested_bh(&napi_alloc_cache.bh_lock);
307 	kasan_mempool_unpoison_object(skb, skbuff_cache_size);
308 
309 	return skb;
310 }
311 
312 /*
313  * Only clear those fields we need to clear, not those that we will
314  * actually initialise later. Hence, don't put any more fields after
315  * the tail pointer in struct sk_buff!
316  */
317 static inline void skbuff_clear(struct sk_buff *skb)
318 {
319 	/* Replace memset(skb, 0, offsetof(struct sk_buff, tail))
320 	 * with two smaller memset(), with a barrier() between them.
321 	 * This forces the compiler to inline both calls.
322 	 */
323 	BUILD_BUG_ON(offsetof(struct sk_buff, tail) <= 128);
324 	memset(skb, 0, 128);
325 	barrier();
326 	memset((void *)skb + 128, 0, offsetof(struct sk_buff, tail) - 128);
327 }
328 
329 /**
330  * napi_skb_cache_get_bulk - obtain a number of zeroed skb heads from the cache
331  * @skbs: pointer to an at least @n-sized array to fill with skb pointers
332  * @n: number of entries to provide
333  *
334  * Tries to obtain @n &sk_buff entries from the NAPI percpu cache and writes
335  * the pointers into the provided array @skbs. If there are less entries
336  * available, tries to replenish the cache and bulk-allocates the diff from
337  * the MM layer if needed.
338  * The heads are being zeroed with either memset() or %__GFP_ZERO, so they are
339  * ready for {,__}build_skb_around() and don't have any data buffers attached.
340  * Must be called *only* from the BH context.
341  *
342  * Return: number of successfully allocated skbs (@n if no actual allocation
343  *	   needed or kmem_cache_alloc_bulk() didn't fail).
344  */
345 u32 napi_skb_cache_get_bulk(void **skbs, u32 n)
346 {
347 	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
348 	u32 bulk, total = n;
349 
350 	local_lock_nested_bh(&napi_alloc_cache.bh_lock);
351 
352 	if (nc->skb_count >= n)
353 		goto get;
354 
355 	/* No enough cached skbs. Try refilling the cache first */
356 	bulk = min(NAPI_SKB_CACHE_SIZE - nc->skb_count, NAPI_SKB_CACHE_BULK);
357 	if (kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
358 				  GFP_ATOMIC | __GFP_NOWARN, bulk,
359 				  &nc->skb_cache[nc->skb_count]))
360 		nc->skb_count += bulk;
361 	if (likely(nc->skb_count >= n))
362 		goto get;
363 
364 	/* Still not enough. Bulk-allocate the missing part directly, zeroed */
365 	if (kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
366 				  GFP_ATOMIC | __GFP_ZERO | __GFP_NOWARN,
367 				  n - nc->skb_count, &skbs[nc->skb_count]))
368 		n = nc->skb_count;
369 	if (likely(nc->skb_count >= n))
370 		goto get;
371 
372 	/* kmem_cache didn't allocate the number we need, limit the output */
373 	total -= n - nc->skb_count;
374 	n = nc->skb_count;
375 
376 get:
377 	for (u32 base = nc->skb_count - n, i = 0; i < n; i++) {
378 		skbs[i] = nc->skb_cache[base + i];
379 
380 		kasan_mempool_unpoison_object(skbs[i], skbuff_cache_size);
381 		skbuff_clear(skbs[i]);
382 	}
383 
384 	nc->skb_count -= n;
385 	local_unlock_nested_bh(&napi_alloc_cache.bh_lock);
386 
387 	return total;
388 }
389 EXPORT_SYMBOL_GPL(napi_skb_cache_get_bulk);
390 
391 static inline void __finalize_skb_around(struct sk_buff *skb, void *data,
392 					 unsigned int size)
393 {
394 	struct skb_shared_info *shinfo;
395 
396 	size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
397 
398 	/* Assumes caller memset cleared SKB */
399 	skb->truesize = SKB_TRUESIZE(size);
400 	refcount_set(&skb->users, 1);
401 	skb->head = data;
402 	skb->data = data;
403 	skb_reset_tail_pointer(skb);
404 	skb_set_end_offset(skb, size);
405 	skb->mac_header = (typeof(skb->mac_header))~0U;
406 	skb->transport_header = (typeof(skb->transport_header))~0U;
407 	skb->alloc_cpu = raw_smp_processor_id();
408 	/* make sure we initialize shinfo sequentially */
409 	shinfo = skb_shinfo(skb);
410 	memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
411 	atomic_set(&shinfo->dataref, 1);
412 
413 	skb_set_kcov_handle(skb, kcov_common_handle());
414 }
415 
416 static inline void *__slab_build_skb(void *data, unsigned int *size)
417 {
418 	void *resized;
419 
420 	/* Must find the allocation size (and grow it to match). */
421 	*size = ksize(data);
422 	/* krealloc() will immediately return "data" when
423 	 * "ksize(data)" is requested: it is the existing upper
424 	 * bounds. As a result, GFP_ATOMIC will be ignored. Note
425 	 * that this "new" pointer needs to be passed back to the
426 	 * caller for use so the __alloc_size hinting will be
427 	 * tracked correctly.
428 	 */
429 	resized = krealloc(data, *size, GFP_ATOMIC);
430 	WARN_ON_ONCE(resized != data);
431 	return resized;
432 }
433 
434 /* build_skb() variant which can operate on slab buffers.
435  * Note that this should be used sparingly as slab buffers
436  * cannot be combined efficiently by GRO!
437  */
438 struct sk_buff *slab_build_skb(void *data)
439 {
440 	struct sk_buff *skb;
441 	unsigned int size;
442 
443 	skb = kmem_cache_alloc(net_hotdata.skbuff_cache,
444 			       GFP_ATOMIC | __GFP_NOWARN);
445 	if (unlikely(!skb))
446 		return NULL;
447 
448 	skbuff_clear(skb);
449 	data = __slab_build_skb(data, &size);
450 	__finalize_skb_around(skb, data, size);
451 
452 	return skb;
453 }
454 EXPORT_SYMBOL(slab_build_skb);
455 
456 /* Caller must provide SKB that is memset cleared */
457 static void __build_skb_around(struct sk_buff *skb, void *data,
458 			       unsigned int frag_size)
459 {
460 	unsigned int size = frag_size;
461 
462 	/* frag_size == 0 is considered deprecated now. Callers
463 	 * using slab buffer should use slab_build_skb() instead.
464 	 */
465 	if (WARN_ONCE(size == 0, "Use slab_build_skb() instead"))
466 		data = __slab_build_skb(data, &size);
467 
468 	__finalize_skb_around(skb, data, size);
469 }
470 
471 /**
472  * __build_skb - build a network buffer
473  * @data: data buffer provided by caller
474  * @frag_size: size of data (must not be 0)
475  *
476  * Allocate a new &sk_buff. Caller provides space holding head and
477  * skb_shared_info. @data must have been allocated from the page
478  * allocator or vmalloc(). (A @frag_size of 0 to indicate a kmalloc()
479  * allocation is deprecated, and callers should use slab_build_skb()
480  * instead.)
481  * The return is the new skb buffer.
482  * On a failure the return is %NULL, and @data is not freed.
483  * Notes :
484  *  Before IO, driver allocates only data buffer where NIC put incoming frame
485  *  Driver should add room at head (NET_SKB_PAD) and
486  *  MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info))
487  *  After IO, driver calls build_skb(), to allocate sk_buff and populate it
488  *  before giving packet to stack.
489  *  RX rings only contains data buffers, not full skbs.
490  */
491 struct sk_buff *__build_skb(void *data, unsigned int frag_size)
492 {
493 	struct sk_buff *skb;
494 
495 	skb = kmem_cache_alloc(net_hotdata.skbuff_cache,
496 			       GFP_ATOMIC | __GFP_NOWARN);
497 	if (unlikely(!skb))
498 		return NULL;
499 
500 	skbuff_clear(skb);
501 	__build_skb_around(skb, data, frag_size);
502 
503 	return skb;
504 }
505 
506 /* build_skb() is wrapper over __build_skb(), that specifically
507  * takes care of skb->head and skb->pfmemalloc
508  */
509 struct sk_buff *build_skb(void *data, unsigned int frag_size)
510 {
511 	struct sk_buff *skb = __build_skb(data, frag_size);
512 
513 	if (likely(skb && frag_size)) {
514 		skb->head_frag = 1;
515 		skb_propagate_pfmemalloc(virt_to_head_page(data), skb);
516 	}
517 	return skb;
518 }
519 EXPORT_SYMBOL(build_skb);
520 
521 /**
522  * build_skb_around - build a network buffer around provided skb
523  * @skb: sk_buff provide by caller, must be memset cleared
524  * @data: data buffer provided by caller
525  * @frag_size: size of data
526  */
527 struct sk_buff *build_skb_around(struct sk_buff *skb,
528 				 void *data, unsigned int frag_size)
529 {
530 	if (unlikely(!skb))
531 		return NULL;
532 
533 	__build_skb_around(skb, data, frag_size);
534 
535 	if (frag_size) {
536 		skb->head_frag = 1;
537 		skb_propagate_pfmemalloc(virt_to_head_page(data), skb);
538 	}
539 	return skb;
540 }
541 EXPORT_SYMBOL(build_skb_around);
542 
543 /**
544  * __napi_build_skb - build a network buffer
545  * @data: data buffer provided by caller
546  * @frag_size: size of data
547  *
548  * Version of __build_skb() that uses NAPI percpu caches to obtain
549  * skbuff_head instead of inplace allocation.
550  *
551  * Returns a new &sk_buff on success, %NULL on allocation failure.
552  */
553 static struct sk_buff *__napi_build_skb(void *data, unsigned int frag_size)
554 {
555 	struct sk_buff *skb;
556 
557 	skb = napi_skb_cache_get(true);
558 	if (unlikely(!skb))
559 		return NULL;
560 
561 	skbuff_clear(skb);
562 	__build_skb_around(skb, data, frag_size);
563 
564 	return skb;
565 }
566 
567 /**
568  * napi_build_skb - build a network buffer
569  * @data: data buffer provided by caller
570  * @frag_size: size of data
571  *
572  * Version of __napi_build_skb() that takes care of skb->head_frag
573  * and skb->pfmemalloc when the data is a page or page fragment.
574  *
575  * Returns a new &sk_buff on success, %NULL on allocation failure.
576  */
577 struct sk_buff *napi_build_skb(void *data, unsigned int frag_size)
578 {
579 	struct sk_buff *skb = __napi_build_skb(data, frag_size);
580 
581 	if (likely(skb) && frag_size) {
582 		skb->head_frag = 1;
583 		skb_propagate_pfmemalloc(virt_to_head_page(data), skb);
584 	}
585 
586 	return skb;
587 }
588 EXPORT_SYMBOL(napi_build_skb);
589 
590 static void *kmalloc_pfmemalloc(size_t obj_size, gfp_t flags, int node)
591 {
592 	if (!gfp_pfmemalloc_allowed(flags))
593 		return NULL;
594 	if (!obj_size)
595 		return kmem_cache_alloc_node(net_hotdata.skb_small_head_cache,
596 					     flags, node);
597 	return kmalloc_node_track_caller(obj_size, flags, node);
598 }
599 
600 /*
601  * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells
602  * the caller if emergency pfmemalloc reserves are being used. If it is and
603  * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves
604  * may be used. Otherwise, the packet data may be discarded until enough
605  * memory is free
606  */
607 static void *kmalloc_reserve(unsigned int *size, gfp_t flags, int node,
608 			     struct sk_buff *skb)
609 {
610 	size_t obj_size;
611 	void *obj;
612 
613 	obj_size = SKB_HEAD_ALIGN(*size);
614 	if (obj_size <= SKB_SMALL_HEAD_CACHE_SIZE &&
615 	    !(flags & KMALLOC_NOT_NORMAL_BITS)) {
616 		obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache,
617 				flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
618 				node);
619 		*size = SKB_SMALL_HEAD_CACHE_SIZE;
620 		if (likely(obj))
621 			goto out;
622 		/* Try again but now we are using pfmemalloc reserves */
623 		if (skb)
624 			skb->pfmemalloc = true;
625 		return kmalloc_pfmemalloc(0, flags, node);
626 	}
627 
628 	obj_size = kmalloc_size_roundup(obj_size);
629 	/* The following cast might truncate high-order bits of obj_size, this
630 	 * is harmless because kmalloc(obj_size >= 2^32) will fail anyway.
631 	 */
632 	*size = (unsigned int)obj_size;
633 
634 	/*
635 	 * Try a regular allocation, when that fails and we're not entitled
636 	 * to the reserves, fail.
637 	 */
638 	obj = kmalloc_node_track_caller(obj_size,
639 					flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
640 					node);
641 	if (likely(obj))
642 		goto out;
643 
644 	/* Try again but now we are using pfmemalloc reserves */
645 	if (skb)
646 		skb->pfmemalloc = true;
647 	obj = kmalloc_pfmemalloc(obj_size, flags, node);
648 out:
649 	return obj;
650 }
651 
652 /* 	Allocate a new skbuff. We do this ourselves so we can fill in a few
653  *	'private' fields and also do memory statistics to find all the
654  *	[BEEP] leaks.
655  *
656  */
657 
658 /**
659  *	__alloc_skb	-	allocate a network buffer
660  *	@size: size to allocate
661  *	@gfp_mask: allocation mask
662  *	@flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache
663  *		instead of head cache and allocate a cloned (child) skb.
664  *		If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
665  *		allocations in case the data is required for writeback
666  *	@node: numa node to allocate memory on
667  *
668  *	Allocate a new &sk_buff. The returned buffer has no headroom and a
669  *	tail room of at least size bytes. The object has a reference count
670  *	of one. The return is the buffer. On a failure the return is %NULL.
671  *
672  *	Buffers may only be allocated from interrupts using a @gfp_mask of
673  *	%GFP_ATOMIC.
674  */
675 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
676 			    int flags, int node)
677 {
678 	struct sk_buff *skb = NULL;
679 	struct kmem_cache *cache;
680 	u8 *data;
681 
682 	if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
683 		gfp_mask |= __GFP_MEMALLOC;
684 
685 	if (flags & SKB_ALLOC_FCLONE) {
686 		cache = net_hotdata.skbuff_fclone_cache;
687 		goto fallback;
688 	}
689 	cache = net_hotdata.skbuff_cache;
690 	if (unlikely(node != NUMA_NO_NODE && node != numa_mem_id()))
691 		goto fallback;
692 
693 	if (flags & SKB_ALLOC_NAPI) {
694 		skb = napi_skb_cache_get(true);
695 		if (unlikely(!skb))
696 			return NULL;
697 	} else if (!in_hardirq() && !irqs_disabled()) {
698 		local_bh_disable();
699 		skb = napi_skb_cache_get(false);
700 		local_bh_enable();
701 	}
702 
703 	if (!skb) {
704 fallback:
705 		skb = kmem_cache_alloc_node(cache, gfp_mask & ~GFP_DMA, node);
706 		if (unlikely(!skb))
707 			return NULL;
708 	}
709 	skbuff_clear(skb);
710 
711 	/* We do our best to align skb_shared_info on a separate cache
712 	 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
713 	 * aligned memory blocks, unless SLUB/SLAB debug is enabled.
714 	 * Both skb->head and skb_shared_info are cache line aligned.
715 	 */
716 	data = kmalloc_reserve(&size, gfp_mask, node, skb);
717 	if (unlikely(!data))
718 		goto nodata;
719 	/* kmalloc_size_roundup() might give us more room than requested.
720 	 * Put skb_shared_info exactly at the end of allocated zone,
721 	 * to allow max possible filling before reallocation.
722 	 */
723 	__finalize_skb_around(skb, data, size);
724 
725 	if (flags & SKB_ALLOC_FCLONE) {
726 		struct sk_buff_fclones *fclones;
727 
728 		fclones = container_of(skb, struct sk_buff_fclones, skb1);
729 
730 		/* skb->fclone is a 2bits field.
731 		 * Replace expensive RMW (skb->fclone = SKB_FCLONE_ORIG)
732 		 * with a single OR.
733 		 */
734 		BUILD_BUG_ON(SKB_FCLONE_UNAVAILABLE != 0);
735 		DEBUG_NET_WARN_ON_ONCE(skb->fclone != SKB_FCLONE_UNAVAILABLE);
736 		skb->fclone |= SKB_FCLONE_ORIG;
737 
738 		refcount_set(&fclones->fclone_ref, 1);
739 	}
740 
741 	return skb;
742 
743 nodata:
744 	kmem_cache_free(cache, skb);
745 	return NULL;
746 }
747 EXPORT_SYMBOL(__alloc_skb);
748 
749 /**
750  *	__netdev_alloc_skb - allocate an skbuff for rx on a specific device
751  *	@dev: network device to receive on
752  *	@len: length to allocate
753  *	@gfp_mask: get_free_pages mask, passed to alloc_skb
754  *
755  *	Allocate a new &sk_buff and assign it a usage count of one. The
756  *	buffer has NET_SKB_PAD headroom built in. Users should allocate
757  *	the headroom they think they need without accounting for the
758  *	built in space. The built in space is used for optimisations.
759  *
760  *	%NULL is returned if there is no free memory.
761  */
762 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len,
763 				   gfp_t gfp_mask)
764 {
765 	struct page_frag_cache *nc;
766 	struct sk_buff *skb;
767 	bool pfmemalloc;
768 	void *data;
769 
770 	len += NET_SKB_PAD;
771 
772 	/* If requested length is either too small or too big,
773 	 * we use kmalloc() for skb->head allocation.
774 	 */
775 	if (len <= SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE) ||
776 	    len > SKB_WITH_OVERHEAD(PAGE_SIZE) ||
777 	    (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
778 		skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
779 		if (!skb)
780 			goto skb_fail;
781 		goto skb_success;
782 	}
783 
784 	len = SKB_HEAD_ALIGN(len);
785 
786 	if (sk_memalloc_socks())
787 		gfp_mask |= __GFP_MEMALLOC;
788 
789 	if (in_hardirq() || irqs_disabled()) {
790 		nc = this_cpu_ptr(&netdev_alloc_cache);
791 		data = page_frag_alloc(nc, len, gfp_mask);
792 		pfmemalloc = page_frag_cache_is_pfmemalloc(nc);
793 	} else {
794 		local_bh_disable();
795 		local_lock_nested_bh(&napi_alloc_cache.bh_lock);
796 
797 		nc = this_cpu_ptr(&napi_alloc_cache.page);
798 		data = page_frag_alloc(nc, len, gfp_mask);
799 		pfmemalloc = page_frag_cache_is_pfmemalloc(nc);
800 
801 		local_unlock_nested_bh(&napi_alloc_cache.bh_lock);
802 		local_bh_enable();
803 	}
804 
805 	if (unlikely(!data))
806 		return NULL;
807 
808 	skb = __build_skb(data, len);
809 	if (unlikely(!skb)) {
810 		skb_free_frag(data);
811 		return NULL;
812 	}
813 
814 	if (pfmemalloc)
815 		skb->pfmemalloc = 1;
816 	skb->head_frag = 1;
817 
818 skb_success:
819 	skb_reserve(skb, NET_SKB_PAD);
820 	skb->dev = dev;
821 
822 skb_fail:
823 	return skb;
824 }
825 EXPORT_SYMBOL(__netdev_alloc_skb);
826 
827 /**
828  *	napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance
829  *	@napi: napi instance this buffer was allocated for
830  *	@len: length to allocate
831  *
832  *	Allocate a new sk_buff for use in NAPI receive.  This buffer will
833  *	attempt to allocate the head from a special reserved region used
834  *	only for NAPI Rx allocation.  By doing this we can save several
835  *	CPU cycles by avoiding having to disable and re-enable IRQs.
836  *
837  *	%NULL is returned if there is no free memory.
838  */
839 struct sk_buff *napi_alloc_skb(struct napi_struct *napi, unsigned int len)
840 {
841 	gfp_t gfp_mask = GFP_ATOMIC | __GFP_NOWARN;
842 	struct napi_alloc_cache *nc;
843 	struct sk_buff *skb;
844 	bool pfmemalloc;
845 	void *data;
846 
847 	DEBUG_NET_WARN_ON_ONCE(!in_softirq());
848 	len += NET_SKB_PAD + NET_IP_ALIGN;
849 
850 	/* If requested length is either too small or too big,
851 	 * we use kmalloc() for skb->head allocation.
852 	 */
853 	if (len <= SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE) ||
854 	    len > SKB_WITH_OVERHEAD(PAGE_SIZE) ||
855 	    (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
856 		skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX | SKB_ALLOC_NAPI,
857 				  NUMA_NO_NODE);
858 		if (!skb)
859 			goto skb_fail;
860 		goto skb_success;
861 	}
862 
863 	len = SKB_HEAD_ALIGN(len);
864 
865 	if (sk_memalloc_socks())
866 		gfp_mask |= __GFP_MEMALLOC;
867 
868 	local_lock_nested_bh(&napi_alloc_cache.bh_lock);
869 	nc = this_cpu_ptr(&napi_alloc_cache);
870 
871 	data = page_frag_alloc(&nc->page, len, gfp_mask);
872 	pfmemalloc = page_frag_cache_is_pfmemalloc(&nc->page);
873 	local_unlock_nested_bh(&napi_alloc_cache.bh_lock);
874 
875 	if (unlikely(!data))
876 		return NULL;
877 
878 	skb = __napi_build_skb(data, len);
879 	if (unlikely(!skb)) {
880 		skb_free_frag(data);
881 		return NULL;
882 	}
883 
884 	if (pfmemalloc)
885 		skb->pfmemalloc = 1;
886 	skb->head_frag = 1;
887 
888 skb_success:
889 	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
890 	skb->dev = napi->dev;
891 
892 skb_fail:
893 	return skb;
894 }
895 EXPORT_SYMBOL(napi_alloc_skb);
896 
897 
898 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
899 			  unsigned int truesize)
900 {
901 	skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
902 
903 	DEBUG_NET_WARN_ON_ONCE(size > truesize);
904 
905 	skb_frag_size_add(frag, size);
906 	skb->len += size;
907 	skb->data_len += size;
908 	skb->truesize += truesize;
909 }
910 EXPORT_SYMBOL(skb_coalesce_rx_frag);
911 
912 static void skb_drop_list(struct sk_buff **listp)
913 {
914 	kfree_skb_list(*listp);
915 	*listp = NULL;
916 }
917 
918 static inline void skb_drop_fraglist(struct sk_buff *skb)
919 {
920 	skb_drop_list(&skb_shinfo(skb)->frag_list);
921 }
922 
923 static void skb_clone_fraglist(struct sk_buff *skb)
924 {
925 	struct sk_buff *list;
926 
927 	skb_walk_frags(skb, list)
928 		skb_get(list);
929 }
930 
931 /**
932  * skb_pp_cow_data() - copy skb data into page-pool backed storage
933  * @pool: page pool to allocate from
934  * @pskb: pointer to skb pointer, replaced with the copied skb on success
935  * @headroom: headroom to reserve in the copied skb
936  *
937  * skb_copy_bits() handles both frags[] and frag_list input. If the copied
938  * skb remains non-linear, it uses frags[], which is the representation used
939  * by XDP multi-buffer.
940  *
941  * Return: 0 on success or a negative errno on failure.
942  */
943 int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb,
944 		    unsigned int headroom)
945 {
946 #if IS_ENABLED(CONFIG_PAGE_POOL)
947 	u32 size, truesize, len, max_head_size, off;
948 	struct sk_buff *skb = *pskb, *nskb;
949 	int err, i, head_off;
950 	void *data;
951 
952 	max_head_size = SKB_WITH_OVERHEAD(PAGE_SIZE - headroom);
953 	if (skb->len > max_head_size + MAX_SKB_FRAGS * PAGE_SIZE)
954 		return -ENOMEM;
955 
956 	size = min_t(u32, skb->len, max_head_size);
957 	truesize = SKB_HEAD_ALIGN(size) + headroom;
958 	data = page_pool_dev_alloc_va(pool, &truesize);
959 	if (!data)
960 		return -ENOMEM;
961 
962 	nskb = napi_build_skb(data, truesize);
963 	if (!nskb) {
964 		page_pool_free_va(pool, data, true);
965 		return -ENOMEM;
966 	}
967 
968 	skb_reserve(nskb, headroom);
969 	skb_copy_header(nskb, skb);
970 	nskb->mac_len = skb->mac_len;
971 	skb_mark_for_recycle(nskb);
972 
973 	err = skb_copy_bits(skb, 0, nskb->data, size);
974 	if (err) {
975 		consume_skb(nskb);
976 		return err;
977 	}
978 	skb_put(nskb, size);
979 
980 	head_off = skb_headroom(nskb) - skb_headroom(skb);
981 	skb_headers_offset_update(nskb, head_off);
982 
983 	off = size;
984 	len = skb->len - off;
985 	for (i = 0; i < MAX_SKB_FRAGS && off < skb->len; i++) {
986 		struct page *page;
987 		u32 page_off;
988 
989 		size = min_t(u32, len, PAGE_SIZE);
990 		truesize = size;
991 
992 		page = page_pool_dev_alloc(pool, &page_off, &truesize);
993 		if (!page) {
994 			consume_skb(nskb);
995 			return -ENOMEM;
996 		}
997 
998 		skb_add_rx_frag(nskb, i, page, page_off, size, truesize);
999 		err = skb_copy_bits(skb, off, page_address(page) + page_off,
1000 				    size);
1001 		if (err) {
1002 			consume_skb(nskb);
1003 			return err;
1004 		}
1005 
1006 		len -= size;
1007 		off += size;
1008 	}
1009 
1010 	consume_skb(skb);
1011 	*pskb = nskb;
1012 
1013 	return 0;
1014 #else
1015 	return -EOPNOTSUPP;
1016 #endif
1017 }
1018 EXPORT_SYMBOL(skb_pp_cow_data);
1019 
1020 int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb,
1021 			 const struct bpf_prog *prog)
1022 {
1023 	if (!prog->aux->xdp_has_frags)
1024 		return -EINVAL;
1025 
1026 	return skb_pp_cow_data(pool, pskb, XDP_PACKET_HEADROOM);
1027 }
1028 EXPORT_SYMBOL(skb_cow_data_for_xdp);
1029 
1030 #if IS_ENABLED(CONFIG_PAGE_POOL)
1031 /**
1032  * napi_pp_put_page - release a page_pool reference on a netmem
1033  * @netmem: netmem to unreference
1034  *
1035  * Drops a page_pool reference (pp_ref_count) on @netmem if it belongs to a
1036  * page_pool. Counterpart to napi_pp_get_page().
1037  *
1038  * Return: true if @netmem belongs to a page_pool, false otherwise.
1039  */
1040 bool napi_pp_put_page(netmem_ref netmem)
1041 {
1042 	netmem = netmem_compound_head(netmem);
1043 
1044 	if (unlikely(!netmem_is_pp(netmem)))
1045 		return false;
1046 
1047 	page_pool_put_full_netmem(netmem_get_pp(netmem), netmem, false);
1048 
1049 	return true;
1050 }
1051 EXPORT_SYMBOL(napi_pp_put_page);
1052 
1053 /**
1054  * napi_pp_get_page - acquire a page_pool reference on a netmem
1055  * @netmem: netmem to reference
1056  *
1057  * Acquires a page_pool reference (pp_ref_count) on @netmem if it belongs to a
1058  * page_pool. Counterpart to napi_pp_put_page().
1059  *
1060  * Return: true if @netmem belongs to a page_pool, false otherwise.
1061  */
1062 bool napi_pp_get_page(netmem_ref netmem)
1063 {
1064 	netmem = netmem_compound_head(netmem);
1065 
1066 	if (unlikely(!netmem_is_pp(netmem)))
1067 		return false;
1068 
1069 	page_pool_ref_netmem(netmem);
1070 
1071 	return true;
1072 }
1073 EXPORT_SYMBOL(napi_pp_get_page);
1074 #endif
1075 
1076 static bool skb_pp_recycle(struct sk_buff *skb, void *data)
1077 {
1078 	if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle)
1079 		return false;
1080 	return napi_pp_put_page(page_to_netmem(virt_to_page(data)));
1081 }
1082 
1083 static void skb_kfree_head(void *head)
1084 {
1085 	kfree(head);
1086 }
1087 
1088 static void skb_free_head(struct sk_buff *skb)
1089 {
1090 	unsigned char *head = skb->head;
1091 
1092 	if (skb->head_frag) {
1093 		if (skb_pp_recycle(skb, head))
1094 			return;
1095 		skb_free_frag(head);
1096 	} else {
1097 		skb_kfree_head(head);
1098 	}
1099 }
1100 
1101 static void skb_release_data(struct sk_buff *skb, enum skb_drop_reason reason)
1102 {
1103 	struct skb_shared_info *shinfo = skb_shinfo(skb);
1104 	int i;
1105 
1106 	if (!skb_data_unref(skb, shinfo))
1107 		return;
1108 
1109 	if (skb_zcopy(skb)) {
1110 		bool skip_unref = shinfo->flags & SKBFL_MANAGED_FRAG_REFS;
1111 
1112 		skb_zcopy_clear(skb, true);
1113 		if (skip_unref)
1114 			goto free_head;
1115 	}
1116 
1117 	for (i = 0; i < shinfo->nr_frags; i++)
1118 		skb_netmem_unref(skb_frag_netmem(&shinfo->frags[i]),
1119 				 skb->pp_recycle);
1120 
1121 free_head:
1122 	if (shinfo->frag_list)
1123 		kfree_skb_list_reason(shinfo->frag_list, reason);
1124 
1125 	skb_free_head(skb);
1126 }
1127 
1128 /*
1129  *	Free an skbuff by memory without cleaning the state.
1130  */
1131 static void kfree_skbmem(struct sk_buff *skb)
1132 {
1133 	struct sk_buff_fclones *fclones;
1134 
1135 	switch (skb->fclone) {
1136 	case SKB_FCLONE_UNAVAILABLE:
1137 		kmem_cache_free(net_hotdata.skbuff_cache, skb);
1138 		return;
1139 
1140 	case SKB_FCLONE_ORIG:
1141 		fclones = container_of(skb, struct sk_buff_fclones, skb1);
1142 
1143 		/* We usually free the clone (TX completion) before original skb
1144 		 * This test would have no chance to be true for the clone,
1145 		 * while here, branch prediction will be good.
1146 		 */
1147 		if (refcount_read(&fclones->fclone_ref) == 1)
1148 			goto fastpath;
1149 		break;
1150 
1151 	default: /* SKB_FCLONE_CLONE */
1152 		fclones = container_of(skb, struct sk_buff_fclones, skb2);
1153 		break;
1154 	}
1155 	if (!refcount_dec_and_test(&fclones->fclone_ref))
1156 		return;
1157 fastpath:
1158 	kmem_cache_free(net_hotdata.skbuff_fclone_cache, fclones);
1159 }
1160 
1161 void skb_release_head_state(struct sk_buff *skb)
1162 {
1163 	skb_dst_drop(skb);
1164 	if (skb->destructor) {
1165 		DEBUG_NET_WARN_ON_ONCE(in_hardirq());
1166 #ifdef CONFIG_INET
1167 		INDIRECT_CALL_4(skb->destructor,
1168 				tcp_wfree, __sock_wfree, sock_wfree,
1169 				xsk_destruct_skb,
1170 				skb);
1171 #else
1172 		INDIRECT_CALL_2(skb->destructor,
1173 				sock_wfree, xsk_destruct_skb,
1174 				skb);
1175 
1176 #endif
1177 		skb->destructor = NULL;
1178 		skb->sk = NULL;
1179 	}
1180 	nf_reset_ct(skb);
1181 	skb_ext_reset(skb);
1182 }
1183 
1184 /* Free everything but the sk_buff shell. */
1185 static void skb_release_all(struct sk_buff *skb, enum skb_drop_reason reason)
1186 {
1187 	skb_release_head_state(skb);
1188 	if (likely(skb->head))
1189 		skb_release_data(skb, reason);
1190 }
1191 
1192 /**
1193  *	__kfree_skb - private function
1194  *	@skb: buffer
1195  *
1196  *	Free an sk_buff. Release anything attached to the buffer.
1197  *	Clean the state. This is an internal helper function. Users should
1198  *	always call kfree_skb
1199  */
1200 
1201 void __kfree_skb(struct sk_buff *skb)
1202 {
1203 	skb_release_all(skb, SKB_DROP_REASON_NOT_SPECIFIED);
1204 	kfree_skbmem(skb);
1205 }
1206 EXPORT_SYMBOL(__kfree_skb);
1207 
1208 static __always_inline
1209 bool __sk_skb_reason_drop(const struct sock *sk, struct sk_buff *skb,
1210 			  enum skb_drop_reason reason)
1211 {
1212 	if (unlikely(!skb_unref(skb)))
1213 		return false;
1214 
1215 	DEBUG_NET_WARN_ON_ONCE(reason == SKB_NOT_DROPPED_YET ||
1216 			       u32_get_bits(reason,
1217 					    SKB_DROP_REASON_SUBSYS_MASK) >=
1218 				SKB_DROP_REASON_SUBSYS_NUM);
1219 
1220 	if (reason == SKB_CONSUMED)
1221 		trace_consume_skb(skb, __builtin_return_address(0));
1222 	else
1223 		trace_kfree_skb(skb, __builtin_return_address(0), reason, sk);
1224 	return true;
1225 }
1226 
1227 /**
1228  *	sk_skb_reason_drop - free an sk_buff with special reason
1229  *	@sk: the socket to receive @skb, or NULL if not applicable
1230  *	@skb: buffer to free
1231  *	@reason: reason why this skb is dropped
1232  *
1233  *	Drop a reference to the buffer and free it if the usage count has hit
1234  *	zero. Meanwhile, pass the receiving socket and drop reason to
1235  *	'kfree_skb' tracepoint.
1236  */
1237 void __fix_address
1238 sk_skb_reason_drop(const struct sock *sk, struct sk_buff *skb,
1239 		   enum skb_drop_reason reason)
1240 {
1241 	if (__sk_skb_reason_drop(sk, skb, reason))
1242 		__kfree_skb(skb);
1243 }
1244 EXPORT_SYMBOL(sk_skb_reason_drop);
1245 
1246 #define KFREE_SKB_BULK_SIZE	16
1247 
1248 struct skb_free_array {
1249 	unsigned int skb_count;
1250 	void *skb_array[KFREE_SKB_BULK_SIZE];
1251 };
1252 
1253 static void kfree_skb_add_bulk(struct sk_buff *skb,
1254 			       struct skb_free_array *sa,
1255 			       enum skb_drop_reason reason)
1256 {
1257 	/* if SKB is a clone, don't handle this case */
1258 	if (unlikely(skb->fclone != SKB_FCLONE_UNAVAILABLE)) {
1259 		__kfree_skb(skb);
1260 		return;
1261 	}
1262 
1263 	skb_release_all(skb, reason);
1264 	sa->skb_array[sa->skb_count++] = skb;
1265 
1266 	if (unlikely(sa->skb_count == KFREE_SKB_BULK_SIZE)) {
1267 		kmem_cache_free_bulk(net_hotdata.skbuff_cache, KFREE_SKB_BULK_SIZE,
1268 				     sa->skb_array);
1269 		sa->skb_count = 0;
1270 	}
1271 }
1272 
1273 void __fix_address
1274 kfree_skb_list_reason(struct sk_buff *segs, enum skb_drop_reason reason)
1275 {
1276 	struct skb_free_array sa;
1277 
1278 	sa.skb_count = 0;
1279 
1280 	while (segs) {
1281 		struct sk_buff *next = segs->next;
1282 
1283 		if (__sk_skb_reason_drop(NULL, segs, reason)) {
1284 			skb_poison_list(segs);
1285 			kfree_skb_add_bulk(segs, &sa, reason);
1286 		}
1287 
1288 		segs = next;
1289 	}
1290 
1291 	if (sa.skb_count)
1292 		kmem_cache_free_bulk(net_hotdata.skbuff_cache, sa.skb_count, sa.skb_array);
1293 }
1294 EXPORT_SYMBOL(kfree_skb_list_reason);
1295 
1296 /* Dump skb information and contents.
1297  *
1298  * Must only be called from net_ratelimit()-ed paths.
1299  *
1300  * Dumps whole packets if full_pkt, only headers otherwise.
1301  */
1302 void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt)
1303 {
1304 	struct skb_shared_info *sh = skb_shinfo(skb);
1305 	struct net_device *dev = skb->dev;
1306 	struct sock *sk = skb->sk;
1307 	struct sk_buff *list_skb;
1308 	bool has_mac, has_trans;
1309 	int headroom, tailroom;
1310 	int i, len, seg_len;
1311 
1312 	if (full_pkt)
1313 		len = skb->len;
1314 	else
1315 		len = min_t(int, skb->len, MAX_HEADER + 128);
1316 
1317 	headroom = skb_headroom(skb);
1318 	tailroom = skb_tailroom(skb);
1319 
1320 	has_mac = skb_mac_header_was_set(skb);
1321 	has_trans = skb_transport_header_was_set(skb);
1322 
1323 	printk("%sskb len=%u data_len=%u headroom=%u headlen=%u tailroom=%u\n"
1324 	       "end-tail=%u mac=(%d,%d) mac_len=%u net=(%d,%d) trans=%d\n"
1325 	       "shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n"
1326 	       "csum(0x%x start=%u offset=%u ip_summed=%u complete_sw=%u valid=%u level=%u)\n"
1327 	       "hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n"
1328 	       "priority=0x%x mark=0x%x alloc_cpu=%u vlan_all=0x%x\n"
1329 	       "encapsulation=%d inner(proto=0x%04x, mac=%u, net=%u, trans=%u)\n",
1330 	       level, skb->len, skb->data_len, headroom, skb_headlen(skb),
1331 	       tailroom, skb->end - skb->tail,
1332 	       has_mac ? skb->mac_header : -1,
1333 	       has_mac ? skb_mac_header_len(skb) : -1,
1334 	       skb->mac_len,
1335 	       skb->network_header,
1336 	       has_trans ? skb_network_header_len(skb) : -1,
1337 	       has_trans ? skb->transport_header : -1,
1338 	       sh->tx_flags, sh->nr_frags,
1339 	       sh->gso_size, sh->gso_type, sh->gso_segs,
1340 	       skb->csum, skb->csum_start, skb->csum_offset, skb->ip_summed,
1341 	       skb->csum_complete_sw, skb->csum_valid, skb->csum_level,
1342 	       skb->hash, skb->sw_hash, skb->l4_hash,
1343 	       ntohs(skb->protocol), skb->pkt_type, skb->skb_iif,
1344 	       skb->priority, skb->mark, skb->alloc_cpu, skb->vlan_all,
1345 	       skb->encapsulation, skb->inner_protocol, skb->inner_mac_header,
1346 	       skb->inner_network_header, skb->inner_transport_header);
1347 
1348 	if (dev)
1349 		printk("%sdev name=%s feat=%pNF\n",
1350 		       level, dev->name, &dev->features);
1351 	if (sk)
1352 		printk("%ssk family=%hu type=%u proto=%u\n",
1353 		       level, sk->sk_family, sk->sk_type, sk->sk_protocol);
1354 
1355 	if (full_pkt && headroom)
1356 		print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET,
1357 			       16, 1, skb->head, headroom, false);
1358 
1359 	seg_len = min_t(int, skb_headlen(skb), len);
1360 	if (seg_len)
1361 		print_hex_dump(level, "skb linear:   ", DUMP_PREFIX_OFFSET,
1362 			       16, 1, skb->data, seg_len, false);
1363 	len -= seg_len;
1364 
1365 	if (full_pkt && tailroom)
1366 		print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET,
1367 			       16, 1, skb_tail_pointer(skb), tailroom, false);
1368 
1369 	for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) {
1370 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1371 		u32 p_off, p_len, copied;
1372 		struct page *p;
1373 		u8 *vaddr;
1374 
1375 		if (skb_frag_is_net_iov(frag)) {
1376 			printk("%sskb frag %d: not readable\n", level, i);
1377 			len -= skb_frag_size(frag);
1378 			if (!len)
1379 				break;
1380 			continue;
1381 		}
1382 
1383 		skb_frag_foreach_page(frag, skb_frag_off(frag),
1384 				      skb_frag_size(frag), p, p_off, p_len,
1385 				      copied) {
1386 			seg_len = min_t(int, p_len, len);
1387 			vaddr = kmap_atomic(p);
1388 			print_hex_dump(level, "skb frag:     ",
1389 				       DUMP_PREFIX_OFFSET,
1390 				       16, 1, vaddr + p_off, seg_len, false);
1391 			kunmap_atomic(vaddr);
1392 			len -= seg_len;
1393 			if (!len)
1394 				break;
1395 		}
1396 	}
1397 
1398 	if (full_pkt && skb_has_frag_list(skb)) {
1399 		printk("skb fraglist:\n");
1400 		skb_walk_frags(skb, list_skb)
1401 			skb_dump(level, list_skb, true);
1402 	}
1403 }
1404 EXPORT_SYMBOL(skb_dump);
1405 
1406 /**
1407  *	skb_tx_error - report an sk_buff xmit error
1408  *	@skb: buffer that triggered an error
1409  *
1410  *	Report xmit error if a device callback is tracking this skb.
1411  *	skb must be freed afterwards.
1412  *
1413  *	Does nothing for a cloned skb: the zerocopy state lives in
1414  *	skb_shinfo(), which the clones share.
1415  */
1416 void skb_tx_error(struct sk_buff *skb)
1417 {
1418 	if (skb && !skb_cloned(skb)) {
1419 		skb_zcopy_downgrade_managed(skb);
1420 		skb_zcopy_clear(skb, true);
1421 	}
1422 }
1423 EXPORT_SYMBOL(skb_tx_error);
1424 
1425 #ifdef CONFIG_TRACEPOINTS
1426 /**
1427  *	consume_skb - free an skbuff
1428  *	@skb: buffer to free
1429  *
1430  *	Drop a ref to the buffer and free it if the usage count has hit zero
1431  *	Functions identically to kfree_skb, but kfree_skb assumes that the frame
1432  *	is being dropped after a failure and notes that
1433  */
1434 void consume_skb(struct sk_buff *skb)
1435 {
1436 	if (!skb_unref(skb))
1437 		return;
1438 
1439 	trace_consume_skb(skb, __builtin_return_address(0));
1440 	__kfree_skb(skb);
1441 }
1442 EXPORT_SYMBOL(consume_skb);
1443 #endif
1444 
1445 /**
1446  *	__consume_stateless_skb - free an skbuff, assuming it is stateless
1447  *	@skb: buffer to free
1448  *
1449  *	Alike consume_skb(), but this variant assumes that this is the last
1450  *	skb reference and all the head states have been already dropped
1451  */
1452 void __consume_stateless_skb(struct sk_buff *skb)
1453 {
1454 	trace_consume_skb(skb, __builtin_return_address(0));
1455 	skb_release_data(skb, SKB_CONSUMED);
1456 	kfree_skbmem(skb);
1457 }
1458 
1459 static void napi_skb_cache_put(struct sk_buff *skb)
1460 {
1461 	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
1462 
1463 	if (!kasan_mempool_poison_object(skb))
1464 		return;
1465 
1466 	local_lock_nested_bh(&napi_alloc_cache.bh_lock);
1467 	nc->skb_cache[nc->skb_count++] = skb;
1468 
1469 	if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) {
1470 		u32 i, remaining = NAPI_SKB_CACHE_SIZE - NAPI_SKB_CACHE_FREE;
1471 
1472 		for (i = remaining; i < NAPI_SKB_CACHE_SIZE; i++)
1473 			kasan_mempool_unpoison_object(nc->skb_cache[i],
1474 						skbuff_cache_size);
1475 
1476 		kmem_cache_free_bulk(net_hotdata.skbuff_cache,
1477 				     NAPI_SKB_CACHE_FREE,
1478 				     nc->skb_cache + remaining);
1479 		nc->skb_count = remaining;
1480 	}
1481 	local_unlock_nested_bh(&napi_alloc_cache.bh_lock);
1482 }
1483 
1484 void __napi_kfree_skb(struct sk_buff *skb, enum skb_drop_reason reason)
1485 {
1486 	skb_release_all(skb, reason);
1487 	napi_skb_cache_put(skb);
1488 }
1489 
1490 void napi_skb_free_stolen_head(struct sk_buff *skb)
1491 {
1492 	if (unlikely(skb->slow_gro)) {
1493 		nf_reset_ct(skb);
1494 		skb_dst_drop(skb);
1495 		skb_ext_put(skb);
1496 		skb_orphan(skb);
1497 		skb->slow_gro = 0;
1498 	}
1499 	if (skb->fclone != SKB_FCLONE_UNAVAILABLE) {
1500 		kfree_skbmem(skb);
1501 		return;
1502 	}
1503 	napi_skb_cache_put(skb);
1504 }
1505 
1506 /**
1507  * napi_consume_skb() - consume skb in NAPI context, try to feed skb cache
1508  * @skb: buffer to free
1509  * @budget: NAPI budget
1510  *
1511  * Non-zero @budget must come from the @budget argument passed by the core
1512  * to a NAPI poll function. Note that core may pass budget of 0 to NAPI poll
1513  * for example when polling for netpoll / netconsole.
1514  *
1515  * Passing @budget of 0 is safe from any context, it turns this function
1516  * into dev_consume_skb_any().
1517  */
1518 void napi_consume_skb(struct sk_buff *skb, int budget)
1519 {
1520 	if (unlikely(!budget || !skb)) {
1521 		dev_consume_skb_any(skb);
1522 		return;
1523 	}
1524 
1525 	DEBUG_NET_WARN_ON_ONCE(!in_softirq());
1526 
1527 	if (!static_branch_unlikely(&skb_defer_disable_key) &&
1528 	    skb->alloc_cpu != smp_processor_id() && !skb_shared(skb)) {
1529 		skb_release_head_state(skb);
1530 		return skb_attempt_defer_free(skb);
1531 	}
1532 
1533 	if (!skb_unref(skb))
1534 		return;
1535 
1536 	/* if reaching here SKB is ready to free */
1537 	trace_consume_skb(skb, __builtin_return_address(0));
1538 
1539 	/* if SKB is a clone, don't handle this case */
1540 	if (skb->fclone != SKB_FCLONE_UNAVAILABLE) {
1541 		__kfree_skb(skb);
1542 		return;
1543 	}
1544 
1545 	skb_release_all(skb, SKB_CONSUMED);
1546 	napi_skb_cache_put(skb);
1547 }
1548 EXPORT_SYMBOL(napi_consume_skb);
1549 
1550 /* Make sure a field is contained by headers group */
1551 #define CHECK_SKB_FIELD(field) \
1552 	BUILD_BUG_ON(offsetof(struct sk_buff, field) !=		\
1553 		     offsetof(struct sk_buff, headers.field));	\
1554 
1555 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
1556 {
1557 	new->tstamp		= old->tstamp;
1558 	/* We do not copy old->sk */
1559 	new->dev		= old->dev;
1560 	memcpy(new->cb, old->cb, sizeof(old->cb));
1561 	skb_dst_copy(new, old);
1562 	__skb_ext_copy(new, old);
1563 	__nf_copy(new, old, false);
1564 
1565 	/* Note : this field could be in the headers group.
1566 	 * It is not yet because we do not want to have a 16 bit hole
1567 	 */
1568 	new->queue_mapping = old->queue_mapping;
1569 
1570 	memcpy(&new->headers, &old->headers, sizeof(new->headers));
1571 	CHECK_SKB_FIELD(protocol);
1572 	CHECK_SKB_FIELD(csum);
1573 	CHECK_SKB_FIELD(hash);
1574 	CHECK_SKB_FIELD(priority);
1575 	CHECK_SKB_FIELD(skb_iif);
1576 	CHECK_SKB_FIELD(vlan_proto);
1577 	CHECK_SKB_FIELD(vlan_tci);
1578 	CHECK_SKB_FIELD(transport_header);
1579 	CHECK_SKB_FIELD(network_header);
1580 	CHECK_SKB_FIELD(mac_header);
1581 	CHECK_SKB_FIELD(inner_protocol);
1582 	CHECK_SKB_FIELD(inner_transport_header);
1583 	CHECK_SKB_FIELD(inner_network_header);
1584 	CHECK_SKB_FIELD(inner_mac_header);
1585 	CHECK_SKB_FIELD(mark);
1586 #ifdef CONFIG_NETWORK_SECMARK
1587 	CHECK_SKB_FIELD(secmark);
1588 #endif
1589 #ifdef CONFIG_NET_RX_BUSY_POLL
1590 	CHECK_SKB_FIELD(napi_id);
1591 #endif
1592 #ifdef CONFIG_XPS
1593 	CHECK_SKB_FIELD(sender_cpu);
1594 #endif
1595 #ifdef CONFIG_NET_SCHED
1596 	CHECK_SKB_FIELD(tc_index);
1597 #endif
1598 
1599 }
1600 
1601 /*
1602  * You should not add any new code to this function.  Add it to
1603  * __copy_skb_header above instead.
1604  */
1605 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
1606 {
1607 #define C(x) n->x = skb->x
1608 
1609 	n->next = n->prev = NULL;
1610 	n->sk = NULL;
1611 	__copy_skb_header(n, skb);
1612 
1613 	C(len);
1614 	C(data_len);
1615 	C(mac_len);
1616 	n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
1617 	n->cloned = 1;
1618 	n->nohdr = 0;
1619 	n->peeked = 0;
1620 	C(pfmemalloc);
1621 	C(pp_recycle);
1622 	n->destructor = NULL;
1623 	C(alloc_cpu);
1624 	C(tail);
1625 	C(end);
1626 	C(head);
1627 	C(head_frag);
1628 	C(data);
1629 	C(truesize);
1630 	refcount_set(&n->users, 1);
1631 
1632 	atomic_inc(&(skb_shinfo(skb)->dataref));
1633 	skb->cloned = 1;
1634 
1635 	return n;
1636 #undef C
1637 }
1638 
1639 /**
1640  * alloc_skb_for_msg() - allocate sk_buff to wrap frag list forming a msg
1641  * @first: first sk_buff of the msg
1642  */
1643 struct sk_buff *alloc_skb_for_msg(struct sk_buff *first)
1644 {
1645 	struct sk_buff *n;
1646 
1647 	n = alloc_skb(0, GFP_ATOMIC);
1648 	if (!n)
1649 		return NULL;
1650 
1651 	n->len = first->len;
1652 	n->data_len = first->len;
1653 	n->truesize = first->truesize;
1654 
1655 	skb_shinfo(n)->frag_list = first;
1656 
1657 	__copy_skb_header(n, first);
1658 	n->destructor = NULL;
1659 
1660 	return n;
1661 }
1662 EXPORT_SYMBOL_GPL(alloc_skb_for_msg);
1663 
1664 /**
1665  *	skb_morph	-	morph one skb into another
1666  *	@dst: the skb to receive the contents
1667  *	@src: the skb to supply the contents
1668  *
1669  *	This is identical to skb_clone except that the target skb is
1670  *	supplied by the user.
1671  *
1672  *	The target skb is returned upon exit.
1673  */
1674 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
1675 {
1676 	skb_release_all(dst, SKB_CONSUMED);
1677 	return __skb_clone(dst, src);
1678 }
1679 EXPORT_SYMBOL_GPL(skb_morph);
1680 
1681 int mm_account_pinned_pages(struct mmpin *mmp, size_t size)
1682 {
1683 	unsigned long max_pg, num_pg, new_pg, old_pg, rlim;
1684 	struct user_struct *user;
1685 
1686 	if (capable(CAP_IPC_LOCK) || !size)
1687 		return 0;
1688 
1689 	rlim = rlimit(RLIMIT_MEMLOCK);
1690 	if (rlim == RLIM_INFINITY)
1691 		return 0;
1692 
1693 	num_pg = (size >> PAGE_SHIFT) + 2;	/* worst case */
1694 	max_pg = rlim >> PAGE_SHIFT;
1695 	user = mmp->user ? : current_user();
1696 
1697 	old_pg = atomic_long_read(&user->locked_vm);
1698 	do {
1699 		new_pg = old_pg + num_pg;
1700 		if (new_pg > max_pg)
1701 			return -ENOBUFS;
1702 	} while (!atomic_long_try_cmpxchg(&user->locked_vm, &old_pg, new_pg));
1703 
1704 	if (!mmp->user) {
1705 		mmp->user = get_uid(user);
1706 		mmp->num_pg = num_pg;
1707 	} else {
1708 		mmp->num_pg += num_pg;
1709 	}
1710 
1711 	return 0;
1712 }
1713 EXPORT_SYMBOL_GPL(mm_account_pinned_pages);
1714 
1715 void mm_unaccount_pinned_pages(struct mmpin *mmp)
1716 {
1717 	if (mmp->user) {
1718 		atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm);
1719 		free_uid(mmp->user);
1720 	}
1721 }
1722 EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages);
1723 
1724 static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size,
1725 					    bool devmem)
1726 {
1727 	struct ubuf_info_msgzc *uarg;
1728 	struct sk_buff *skb;
1729 
1730 	WARN_ON_ONCE(!in_task());
1731 
1732 	skb = sock_omalloc(sk, 0, GFP_KERNEL);
1733 	if (!skb)
1734 		return NULL;
1735 
1736 	BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb));
1737 	uarg = (void *)skb->cb;
1738 	uarg->mmp.user = NULL;
1739 
1740 	if (likely(!devmem) && mm_account_pinned_pages(&uarg->mmp, size)) {
1741 		kfree_skb(skb);
1742 		return NULL;
1743 	}
1744 
1745 	uarg->ubuf.ops = &msg_zerocopy_ubuf_ops;
1746 	uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
1747 	uarg->len = 1;
1748 	uarg->bytelen = size;
1749 	uarg->zerocopy = 1;
1750 	uarg->ubuf.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN;
1751 	refcount_set(&uarg->ubuf.refcnt, 1);
1752 	sock_hold(sk);
1753 
1754 	return &uarg->ubuf;
1755 }
1756 
1757 static inline struct sk_buff *skb_from_uarg(struct ubuf_info_msgzc *uarg)
1758 {
1759 	return container_of((void *)uarg, struct sk_buff, cb);
1760 }
1761 
1762 struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
1763 				       struct ubuf_info *uarg, bool devmem)
1764 {
1765 	if (uarg) {
1766 		struct ubuf_info_msgzc *uarg_zc;
1767 		const u32 byte_limit = 1 << 19;		/* limit to a few TSO */
1768 		u32 bytelen, next;
1769 
1770 		/* there might be non MSG_ZEROCOPY users */
1771 		if (uarg->ops != &msg_zerocopy_ubuf_ops)
1772 			return NULL;
1773 
1774 		/* realloc only when socket is locked (TCP, UDP cork),
1775 		 * so uarg->len and sk_zckey access is serialized
1776 		 */
1777 		if (!sock_owned_by_user(sk)) {
1778 			WARN_ON_ONCE(1);
1779 			return NULL;
1780 		}
1781 
1782 		uarg_zc = uarg_to_msgzc(uarg);
1783 		bytelen = uarg_zc->bytelen + size;
1784 		if (uarg_zc->len == USHRT_MAX - 1 || bytelen > byte_limit) {
1785 			/* TCP can create new skb to attach new uarg */
1786 			if (sk->sk_type == SOCK_STREAM)
1787 				goto new_alloc;
1788 			return NULL;
1789 		}
1790 
1791 		next = (u32)atomic_read(&sk->sk_zckey);
1792 		if ((u32)(uarg_zc->id + uarg_zc->len) == next) {
1793 			if (likely(!devmem) &&
1794 			    mm_account_pinned_pages(&uarg_zc->mmp, size))
1795 				return NULL;
1796 			uarg_zc->len++;
1797 			uarg_zc->bytelen = bytelen;
1798 			atomic_set(&sk->sk_zckey, ++next);
1799 
1800 			/* no extra ref when appending to datagram (MSG_MORE) */
1801 			if (sk->sk_type == SOCK_STREAM)
1802 				net_zcopy_get(uarg);
1803 
1804 			return uarg;
1805 		}
1806 	}
1807 
1808 new_alloc:
1809 	return msg_zerocopy_alloc(sk, size, devmem);
1810 }
1811 EXPORT_SYMBOL_GPL(msg_zerocopy_realloc);
1812 
1813 static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len)
1814 {
1815 	struct sock_exterr_skb *serr = SKB_EXT_ERR(skb);
1816 	u32 old_lo, old_hi;
1817 	u64 sum_len;
1818 
1819 	old_lo = serr->ee.ee_info;
1820 	old_hi = serr->ee.ee_data;
1821 	sum_len = old_hi - old_lo + 1ULL + len;
1822 
1823 	if (sum_len >= (1ULL << 32))
1824 		return false;
1825 
1826 	if (lo != old_hi + 1)
1827 		return false;
1828 
1829 	serr->ee.ee_data += len;
1830 	return true;
1831 }
1832 
1833 static void __msg_zerocopy_callback(struct ubuf_info_msgzc *uarg)
1834 {
1835 	struct sk_buff *tail, *skb = skb_from_uarg(uarg);
1836 	struct sock_exterr_skb *serr;
1837 	struct sock *sk = skb->sk;
1838 	struct sk_buff_head *q;
1839 	unsigned long flags;
1840 	bool is_zerocopy;
1841 	u32 lo, hi;
1842 	u16 len;
1843 
1844 	mm_unaccount_pinned_pages(&uarg->mmp);
1845 
1846 	/* if !len, there was only 1 call, and it was aborted
1847 	 * so do not queue a completion notification
1848 	 */
1849 	if (!uarg->len || sock_flag(sk, SOCK_DEAD))
1850 		goto release;
1851 
1852 	len = uarg->len;
1853 	lo = uarg->id;
1854 	hi = uarg->id + len - 1;
1855 	is_zerocopy = uarg->zerocopy;
1856 
1857 	serr = SKB_EXT_ERR(skb);
1858 	memset(serr, 0, sizeof(*serr));
1859 	serr->ee.ee_errno = 0;
1860 	serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY;
1861 	serr->ee.ee_data = hi;
1862 	serr->ee.ee_info = lo;
1863 	if (!is_zerocopy)
1864 		serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED;
1865 
1866 	q = &sk->sk_error_queue;
1867 	spin_lock_irqsave(&q->lock, flags);
1868 	tail = skb_peek_tail(q);
1869 	if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY ||
1870 	    !skb_zerocopy_notify_extend(tail, lo, len)) {
1871 		__skb_queue_tail(q, skb);
1872 		skb = NULL;
1873 	}
1874 	spin_unlock_irqrestore(&q->lock, flags);
1875 
1876 	sk_error_report(sk);
1877 
1878 release:
1879 	consume_skb(skb);
1880 	sock_put(sk);
1881 }
1882 
1883 static void msg_zerocopy_complete(struct sk_buff *skb, struct ubuf_info *uarg,
1884 				  bool success)
1885 {
1886 	struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg);
1887 
1888 	uarg_zc->zerocopy = uarg_zc->zerocopy & success;
1889 
1890 	if (refcount_dec_and_test(&uarg->refcnt))
1891 		__msg_zerocopy_callback(uarg_zc);
1892 }
1893 
1894 void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
1895 {
1896 	struct sock *sk = skb_from_uarg(uarg_to_msgzc(uarg))->sk;
1897 
1898 	atomic_dec(&sk->sk_zckey);
1899 	uarg_to_msgzc(uarg)->len--;
1900 
1901 	if (have_uref)
1902 		msg_zerocopy_complete(NULL, uarg, true);
1903 }
1904 EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort);
1905 
1906 const struct ubuf_info_ops msg_zerocopy_ubuf_ops = {
1907 	.complete = msg_zerocopy_complete,
1908 };
1909 EXPORT_SYMBOL_GPL(msg_zerocopy_ubuf_ops);
1910 
1911 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
1912 			     struct msghdr *msg, int len,
1913 			     struct ubuf_info *uarg,
1914 			     struct net_devmem_dmabuf_binding *binding)
1915 {
1916 	unsigned int orig_truesize = skb->truesize;
1917 	int err, orig_len = skb->len;
1918 
1919 	if (uarg->ops->link_skb) {
1920 		err = uarg->ops->link_skb(skb, uarg);
1921 		if (err)
1922 			return err;
1923 	} else {
1924 		struct ubuf_info *orig_uarg = skb_zcopy(skb);
1925 
1926 		/* An skb can only point to one uarg. This edge case happens
1927 		 * when TCP appends to an skb, but zerocopy_realloc triggered
1928 		 * a new alloc.
1929 		 */
1930 		if (orig_uarg && uarg != orig_uarg)
1931 			return -EEXIST;
1932 	}
1933 
1934 	err = __zerocopy_sg_from_iter(msg, sk, skb, &msg->msg_iter, len,
1935 				      binding);
1936 	if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
1937 		u32 delta_truesize = skb->truesize - orig_truesize;
1938 		struct sock *save_sk = skb->sk;
1939 
1940 		/* Streams do not free skb on error. Reset to prev state. */
1941 		iov_iter_revert(&msg->msg_iter, skb->len - orig_len);
1942 		skb->sk = sk;
1943 		if (!___pskb_trim(skb, orig_len)) {
1944 			skb->truesize -= delta_truesize;
1945 			sk_wmem_queued_add(sk, -delta_truesize);
1946 			if (!skb_zcopy_pure(skb))
1947 				sk_mem_uncharge(sk, delta_truesize);
1948 		}
1949 		skb->sk = save_sk;
1950 		return err;
1951 	}
1952 
1953 	skb_zcopy_set(skb, uarg, NULL);
1954 	return skb->len - orig_len;
1955 }
1956 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);
1957 
1958 void __skb_zcopy_downgrade_managed(struct sk_buff *skb)
1959 {
1960 	int i;
1961 
1962 	skb_shinfo(skb)->flags &= ~SKBFL_MANAGED_FRAG_REFS;
1963 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1964 		skb_frag_ref(skb, i);
1965 }
1966 EXPORT_SYMBOL_GPL(__skb_zcopy_downgrade_managed);
1967 
1968 static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig,
1969 			      gfp_t gfp_mask)
1970 {
1971 	if (skb_zcopy(orig)) {
1972 		if (skb_zcopy(nskb)) {
1973 			/* !gfp_mask callers are verified to !skb_zcopy(nskb) */
1974 			if (!gfp_mask) {
1975 				WARN_ON_ONCE(1);
1976 				return -ENOMEM;
1977 			}
1978 			if (skb_uarg(nskb) == skb_uarg(orig))
1979 				return 0;
1980 			if (skb_copy_ubufs(nskb, GFP_ATOMIC))
1981 				return -EIO;
1982 		}
1983 		skb_zcopy_set(nskb, skb_uarg(orig), NULL);
1984 	}
1985 	return 0;
1986 }
1987 
1988 /**
1989  *	skb_copy_ubufs	-	copy userspace skb frags buffers to kernel
1990  *	@skb: the skb to modify
1991  *	@gfp_mask: allocation priority
1992  *
1993  *	This must be called on skb with SKBFL_ZEROCOPY_ENABLE.
1994  *	It will copy all frags into kernel and drop the reference
1995  *	to userspace pages.
1996  *
1997  *	If this function is called from an interrupt gfp_mask() must be
1998  *	%GFP_ATOMIC.
1999  *
2000  *	Returns 0 on success or a negative error code on failure
2001  *	to allocate kernel memory to copy to.
2002  */
2003 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
2004 {
2005 	int num_frags = skb_shinfo(skb)->nr_frags;
2006 	struct page *page, *head = NULL;
2007 	int i, order, psize, new_frags;
2008 	u32 d_off;
2009 
2010 	if (!skb_frags_readable(skb))
2011 		return -EFAULT;
2012 
2013 	if (skb_shared(skb) || skb_unclone(skb, gfp_mask))
2014 		return -EINVAL;
2015 
2016 	if (!num_frags)
2017 		goto release;
2018 
2019 	/* We might have to allocate high order pages, so compute what minimum
2020 	 * page order is needed.
2021 	 */
2022 	order = 0;
2023 	while ((PAGE_SIZE << order) * MAX_SKB_FRAGS < __skb_pagelen(skb))
2024 		order++;
2025 	psize = (PAGE_SIZE << order);
2026 
2027 	new_frags = (__skb_pagelen(skb) + psize - 1) >> (PAGE_SHIFT + order);
2028 	for (i = 0; i < new_frags; i++) {
2029 		page = alloc_pages(gfp_mask | __GFP_COMP, order);
2030 		if (!page) {
2031 			while (head) {
2032 				struct page *next = (struct page *)page_private(head);
2033 				put_page(head);
2034 				head = next;
2035 			}
2036 			return -ENOMEM;
2037 		}
2038 		set_page_private(page, (unsigned long)head);
2039 		head = page;
2040 	}
2041 
2042 	page = head;
2043 	d_off = 0;
2044 	for (i = 0; i < num_frags; i++) {
2045 		skb_frag_t *f = &skb_shinfo(skb)->frags[i];
2046 		u32 p_off, p_len, copied;
2047 		struct page *p;
2048 		u8 *vaddr;
2049 
2050 		skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f),
2051 				      p, p_off, p_len, copied) {
2052 			u32 copy, done = 0;
2053 			vaddr = kmap_atomic(p);
2054 
2055 			while (done < p_len) {
2056 				if (d_off == psize) {
2057 					d_off = 0;
2058 					page = (struct page *)page_private(page);
2059 				}
2060 				copy = min_t(u32, psize - d_off, p_len - done);
2061 				memcpy(page_address(page) + d_off,
2062 				       vaddr + p_off + done, copy);
2063 				done += copy;
2064 				d_off += copy;
2065 			}
2066 			kunmap_atomic(vaddr);
2067 		}
2068 	}
2069 
2070 	/* skb frags release userspace buffers */
2071 	for (i = 0; i < num_frags; i++)
2072 		skb_frag_unref(skb, i);
2073 
2074 	/* skb frags point to kernel buffers */
2075 	for (i = 0; i < new_frags - 1; i++) {
2076 		__skb_fill_netmem_desc(skb, i, page_to_netmem(head), 0, psize);
2077 		head = (struct page *)page_private(head);
2078 	}
2079 	__skb_fill_netmem_desc(skb, new_frags - 1, page_to_netmem(head), 0,
2080 			       d_off);
2081 	skb_shinfo(skb)->nr_frags = new_frags;
2082 
2083 release:
2084 	skb_zcopy_clear(skb, false);
2085 	return 0;
2086 }
2087 EXPORT_SYMBOL_GPL(skb_copy_ubufs);
2088 
2089 /**
2090  *	skb_clone	-	duplicate an sk_buff
2091  *	@skb: buffer to clone
2092  *	@gfp_mask: allocation priority
2093  *
2094  *	Duplicate an &sk_buff. The new one is not owned by a socket. Both
2095  *	copies share the same packet data but not structure. The new
2096  *	buffer has a reference count of 1. If the allocation fails the
2097  *	function returns %NULL otherwise the new buffer is returned.
2098  *
2099  *	If this function is called from an interrupt gfp_mask() must be
2100  *	%GFP_ATOMIC.
2101  */
2102 
2103 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
2104 {
2105 	struct sk_buff_fclones *fclones = container_of(skb,
2106 						       struct sk_buff_fclones,
2107 						       skb1);
2108 	struct sk_buff *n;
2109 
2110 	if (skb_orphan_frags(skb, gfp_mask))
2111 		return NULL;
2112 
2113 	if (skb->fclone == SKB_FCLONE_ORIG &&
2114 	    refcount_read(&fclones->fclone_ref) == 1) {
2115 		n = &fclones->skb2;
2116 		refcount_set(&fclones->fclone_ref, 2);
2117 		n->fclone = SKB_FCLONE_CLONE;
2118 	} else {
2119 		if (skb_pfmemalloc(skb))
2120 			gfp_mask |= __GFP_MEMALLOC;
2121 
2122 		n = kmem_cache_alloc(net_hotdata.skbuff_cache, gfp_mask);
2123 		if (!n)
2124 			return NULL;
2125 
2126 		n->fclone = SKB_FCLONE_UNAVAILABLE;
2127 	}
2128 
2129 	return __skb_clone(n, skb);
2130 }
2131 EXPORT_SYMBOL(skb_clone);
2132 
2133 void skb_headers_offset_update(struct sk_buff *skb, int off)
2134 {
2135 	/* Only adjust this if it actually is csum_start rather than csum */
2136 	if (skb->ip_summed == CHECKSUM_PARTIAL)
2137 		skb->csum_start += off;
2138 	/* {transport,network,mac}_header and tail are relative to skb->head */
2139 	skb->transport_header += off;
2140 	skb->network_header   += off;
2141 	if (skb_mac_header_was_set(skb))
2142 		skb->mac_header += off;
2143 	skb->inner_transport_header += off;
2144 	skb->inner_network_header += off;
2145 	skb->inner_mac_header += off;
2146 }
2147 EXPORT_SYMBOL(skb_headers_offset_update);
2148 
2149 void skb_copy_header(struct sk_buff *new, const struct sk_buff *old)
2150 {
2151 	__copy_skb_header(new, old);
2152 
2153 	skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
2154 	skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
2155 	skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
2156 }
2157 EXPORT_SYMBOL(skb_copy_header);
2158 
2159 static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
2160 {
2161 	if (skb_pfmemalloc(skb))
2162 		return SKB_ALLOC_RX;
2163 	return 0;
2164 }
2165 
2166 /**
2167  *	skb_copy	-	create private copy of an sk_buff
2168  *	@skb: buffer to copy
2169  *	@gfp_mask: allocation priority
2170  *
2171  *	Make a copy of both an &sk_buff and its data. This is used when the
2172  *	caller wishes to modify the data and needs a private copy of the
2173  *	data to alter. Returns %NULL on failure or the pointer to the buffer
2174  *	on success. The returned buffer has a reference count of 1.
2175  *
2176  *	As by-product this function converts non-linear &sk_buff to linear
2177  *	one, so that &sk_buff becomes completely private and caller is allowed
2178  *	to modify all the data of returned buffer. This means that this
2179  *	function is not recommended for use in circumstances when only
2180  *	header is going to be modified. Use pskb_copy() instead.
2181  */
2182 
2183 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
2184 {
2185 	struct sk_buff *n;
2186 	unsigned int size;
2187 	int headerlen;
2188 
2189 	if (!skb_frags_readable(skb))
2190 		return NULL;
2191 
2192 	if (WARN_ON_ONCE(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST))
2193 		return NULL;
2194 
2195 	headerlen = skb_headroom(skb);
2196 	size = skb_end_offset(skb) + skb->data_len;
2197 	n = __alloc_skb(size, gfp_mask,
2198 			skb_alloc_rx_flag(skb), NUMA_NO_NODE);
2199 	if (!n)
2200 		return NULL;
2201 
2202 	/* Set the data pointer */
2203 	skb_reserve(n, headerlen);
2204 	/* Set the tail pointer and length */
2205 	skb_put(n, skb->len);
2206 
2207 	if (unlikely(skb_copy_bits(skb, -headerlen, n->head,
2208 				   headerlen + skb->len))) {
2209 		DEBUG_NET_WARN_ON_ONCE(1);
2210 		kfree_skb(n);
2211 		return NULL;
2212 	}
2213 
2214 	skb_copy_header(n, skb);
2215 	return n;
2216 }
2217 EXPORT_SYMBOL(skb_copy);
2218 
2219 /**
2220  *	__pskb_copy_fclone	-  create copy of an sk_buff with private head.
2221  *	@skb: buffer to copy
2222  *	@headroom: headroom of new skb
2223  *	@gfp_mask: allocation priority
2224  *	@fclone: if true allocate the copy of the skb from the fclone
2225  *	cache instead of the head cache; it is recommended to set this
2226  *	to true for the cases where the copy will likely be cloned
2227  *
2228  *	Make a copy of both an &sk_buff and part of its data, located
2229  *	in header. Fragmented data remain shared. This is used when
2230  *	the caller wishes to modify only header of &sk_buff and needs
2231  *	private copy of the header to alter. Returns %NULL on failure
2232  *	or the pointer to the buffer on success.
2233  *	The returned buffer has a reference count of 1.
2234  */
2235 
2236 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
2237 				   gfp_t gfp_mask, bool fclone)
2238 {
2239 	unsigned int size = skb_headlen(skb) + headroom;
2240 	int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0);
2241 	struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE);
2242 
2243 	if (!n)
2244 		goto out;
2245 
2246 	/* Set the data pointer */
2247 	skb_reserve(n, headroom);
2248 	/* Set the tail pointer and length */
2249 	skb_put(n, skb_headlen(skb));
2250 	/* Copy the bytes */
2251 	skb_copy_from_linear_data(skb, n->data, n->len);
2252 
2253 	n->truesize += skb->data_len;
2254 	n->data_len  = skb->data_len;
2255 	n->len	     = skb->len;
2256 
2257 	if (skb_shinfo(skb)->nr_frags) {
2258 		int i;
2259 
2260 		if (skb_orphan_frags(skb, gfp_mask) ||
2261 		    skb_zerocopy_clone(n, skb, gfp_mask)) {
2262 			kfree_skb(n);
2263 			n = NULL;
2264 			goto out;
2265 		}
2266 		n->pp_recycle = skb->pp_recycle;
2267 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2268 			skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
2269 			skb_frag_ref(n, i);
2270 		}
2271 		skb_shinfo(n)->nr_frags = i;
2272 		skb_shinfo(n)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG;
2273 	}
2274 
2275 	if (skb_has_frag_list(skb)) {
2276 		skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
2277 		skb_clone_fraglist(n);
2278 	}
2279 
2280 	skb_copy_header(n, skb);
2281 out:
2282 	return n;
2283 }
2284 EXPORT_SYMBOL(__pskb_copy_fclone);
2285 
2286 /**
2287  *	pskb_expand_head - reallocate header of &sk_buff
2288  *	@skb: buffer to reallocate
2289  *	@nhead: room to add at head
2290  *	@ntail: room to add at tail
2291  *	@gfp_mask: allocation priority
2292  *
2293  *	Expands (or creates identical copy, if @nhead and @ntail are zero)
2294  *	header of @skb. &sk_buff itself is not changed. &sk_buff MUST have
2295  *	reference count of 1. Returns zero in the case of success or error,
2296  *	if expansion failed. In the last case, &sk_buff is not changed.
2297  *
2298  *	All the pointers pointing into skb header may change and must be
2299  *	reloaded after call to this function.
2300  *
2301  *	Note: If you skb_push() the start of the buffer after reallocating the
2302  *	header, call skb_postpush_data_move() first to move the metadata out of
2303  *	the way before writing to &sk_buff->data.
2304  */
2305 
2306 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
2307 		     gfp_t gfp_mask)
2308 {
2309 	unsigned int osize = skb_end_offset(skb);
2310 	unsigned int size = osize + nhead + ntail;
2311 	long off;
2312 	u8 *data;
2313 	int i;
2314 
2315 	if (unlikely(nhead < 0)) {
2316 		DEBUG_NET_WARN_ON_ONCE(1);
2317 		return -EINVAL;
2318 	}
2319 
2320 	if (unlikely(skb_shared(skb))) {
2321 		DEBUG_NET_WARN_ON_ONCE(1);
2322 		return -EINVAL;
2323 	}
2324 
2325 	skb_zcopy_downgrade_managed(skb);
2326 
2327 	if (skb_pfmemalloc(skb))
2328 		gfp_mask |= __GFP_MEMALLOC;
2329 
2330 	data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL);
2331 	if (!data)
2332 		goto nodata;
2333 	size = SKB_WITH_OVERHEAD(size);
2334 
2335 	/* Copy only real data... and, alas, header. This should be
2336 	 * optimized for the cases when header is void.
2337 	 */
2338 	memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
2339 
2340 	memcpy((struct skb_shared_info *)(data + size),
2341 	       skb_shinfo(skb),
2342 	       offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
2343 
2344 	/*
2345 	 * if shinfo is shared we must drop the old head gracefully, but if it
2346 	 * is not we can just drop the old head and let the existing refcount
2347 	 * be since all we did is relocate the values
2348 	 */
2349 	if (skb_cloned(skb)) {
2350 		if (skb_orphan_frags(skb, gfp_mask))
2351 			goto nofrags;
2352 		if (skb_zcopy(skb))
2353 			net_zcopy_get(skb_uarg(skb));
2354 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
2355 			skb_frag_ref(skb, i);
2356 
2357 		if (skb_has_frag_list(skb))
2358 			skb_clone_fraglist(skb);
2359 
2360 		skb_release_data(skb, SKB_CONSUMED);
2361 	} else {
2362 		skb_free_head(skb);
2363 	}
2364 	off = (data + nhead) - skb->head;
2365 
2366 	skb->head     = data;
2367 	skb->head_frag = 0;
2368 	skb->data    += off;
2369 
2370 	skb_set_end_offset(skb, size);
2371 #ifdef NET_SKBUFF_DATA_USES_OFFSET
2372 	off           = nhead;
2373 #endif
2374 	skb->tail	      += off;
2375 	skb_headers_offset_update(skb, nhead);
2376 	skb->cloned   = 0;
2377 	skb->hdr_len  = 0;
2378 	skb->nohdr    = 0;
2379 	atomic_set(&skb_shinfo(skb)->dataref, 1);
2380 
2381 	/* It is not generally safe to change skb->truesize.
2382 	 * For the moment, we really care of rx path, or
2383 	 * when skb is orphaned (not attached to a socket).
2384 	 */
2385 	if (!skb->sk || skb->destructor == sock_edemux)
2386 		skb->truesize += size - osize;
2387 
2388 	return 0;
2389 
2390 nofrags:
2391 	skb_kfree_head(data);
2392 nodata:
2393 	return -ENOMEM;
2394 }
2395 EXPORT_SYMBOL(pskb_expand_head);
2396 
2397 /* Make private copy of skb with writable head and some headroom */
2398 
2399 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
2400 {
2401 	struct sk_buff *skb2;
2402 	int delta = headroom - skb_headroom(skb);
2403 
2404 	if (delta <= 0)
2405 		skb2 = pskb_copy(skb, GFP_ATOMIC);
2406 	else {
2407 		skb2 = skb_clone(skb, GFP_ATOMIC);
2408 		if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
2409 					     GFP_ATOMIC)) {
2410 			kfree_skb(skb2);
2411 			skb2 = NULL;
2412 		}
2413 	}
2414 	return skb2;
2415 }
2416 EXPORT_SYMBOL(skb_realloc_headroom);
2417 
2418 /* Note: We plan to rework this in linux-6.4 */
2419 int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri)
2420 {
2421 	unsigned int saved_end_offset, saved_truesize;
2422 	struct skb_shared_info *shinfo;
2423 	int res;
2424 
2425 	saved_end_offset = skb_end_offset(skb);
2426 	saved_truesize = skb->truesize;
2427 
2428 	res = pskb_expand_head(skb, 0, 0, pri);
2429 	if (res)
2430 		return res;
2431 
2432 	skb->truesize = saved_truesize;
2433 
2434 	if (likely(skb_end_offset(skb) == saved_end_offset))
2435 		return 0;
2436 
2437 	shinfo = skb_shinfo(skb);
2438 
2439 	/* We are about to change back skb->end,
2440 	 * we need to move skb_shinfo() to its new location.
2441 	 */
2442 	memmove(skb->head + saved_end_offset,
2443 		shinfo,
2444 		offsetof(struct skb_shared_info, frags[shinfo->nr_frags]));
2445 
2446 	skb_set_end_offset(skb, saved_end_offset);
2447 
2448 	return 0;
2449 }
2450 
2451 /**
2452  *	skb_expand_head - reallocate header of &sk_buff
2453  *	@skb: buffer to reallocate
2454  *	@headroom: needed headroom
2455  *
2456  *	Unlike skb_realloc_headroom, this one does not allocate a new skb
2457  *	if possible; copies skb->sk to new skb as needed
2458  *	and frees original skb in case of failures.
2459  *
2460  *	It expect increased headroom and generates warning otherwise.
2461  */
2462 
2463 struct sk_buff *skb_expand_head(struct sk_buff *skb, unsigned int headroom)
2464 {
2465 	int delta = headroom - skb_headroom(skb);
2466 	int osize = skb_end_offset(skb);
2467 	struct sock *sk = skb->sk;
2468 
2469 	if (WARN_ONCE(delta <= 0,
2470 		      "%s is expecting an increase in the headroom", __func__))
2471 		return skb;
2472 
2473 	delta = SKB_DATA_ALIGN(delta);
2474 	/* pskb_expand_head() might crash, if skb is shared. */
2475 	if (skb_shared(skb) || !is_skb_wmem(skb)) {
2476 		struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
2477 
2478 		if (unlikely(!nskb))
2479 			goto fail;
2480 
2481 		if (sk)
2482 			skb_set_owner_w(nskb, sk);
2483 		consume_skb(skb);
2484 		skb = nskb;
2485 	}
2486 	if (pskb_expand_head(skb, delta, 0, GFP_ATOMIC))
2487 		goto fail;
2488 
2489 	if (sk && is_skb_wmem(skb)) {
2490 		delta = skb_end_offset(skb) - osize;
2491 		refcount_add(delta, &sk->sk_wmem_alloc);
2492 		skb->truesize += delta;
2493 	}
2494 	return skb;
2495 
2496 fail:
2497 	kfree_skb(skb);
2498 	return NULL;
2499 }
2500 EXPORT_SYMBOL(skb_expand_head);
2501 
2502 /**
2503  *	skb_copy_expand	-	copy and expand sk_buff
2504  *	@skb: buffer to copy
2505  *	@newheadroom: new free bytes at head
2506  *	@newtailroom: new free bytes at tail
2507  *	@gfp_mask: allocation priority
2508  *
2509  *	Make a copy of both an &sk_buff and its data and while doing so
2510  *	allocate additional space.
2511  *
2512  *	This is used when the caller wishes to modify the data and needs a
2513  *	private copy of the data to alter as well as more space for new fields.
2514  *	Returns %NULL on failure or the pointer to the buffer
2515  *	on success. The returned buffer has a reference count of 1.
2516  *
2517  *	You must pass %GFP_ATOMIC as the allocation priority if this function
2518  *	is called from an interrupt.
2519  */
2520 struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
2521 				int newheadroom, int newtailroom,
2522 				gfp_t gfp_mask)
2523 {
2524 	/*
2525 	 *	Allocate the copy buffer
2526 	 */
2527 	int head_copy_len, head_copy_off;
2528 	struct sk_buff *n;
2529 	int oldheadroom;
2530 
2531 	if (!skb_frags_readable(skb))
2532 		return NULL;
2533 
2534 	if (WARN_ON_ONCE(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST))
2535 		return NULL;
2536 
2537 	oldheadroom = skb_headroom(skb);
2538 	n = __alloc_skb(newheadroom + skb->len + newtailroom,
2539 			gfp_mask, skb_alloc_rx_flag(skb),
2540 			NUMA_NO_NODE);
2541 	if (!n)
2542 		return NULL;
2543 
2544 	skb_reserve(n, newheadroom);
2545 
2546 	/* Set the tail pointer and length */
2547 	skb_put(n, skb->len);
2548 
2549 	head_copy_len = oldheadroom;
2550 	head_copy_off = 0;
2551 	if (newheadroom <= head_copy_len)
2552 		head_copy_len = newheadroom;
2553 	else
2554 		head_copy_off = newheadroom - head_copy_len;
2555 
2556 	/* Copy the linear header and data. */
2557 	if (unlikely(skb_copy_bits(skb, -head_copy_len,
2558 				   n->head + head_copy_off,
2559 				   skb->len + head_copy_len))) {
2560 		DEBUG_NET_WARN_ON_ONCE(1);
2561 		kfree_skb(n);
2562 		return NULL;
2563 	}
2564 
2565 	skb_copy_header(n, skb);
2566 
2567 	skb_headers_offset_update(n, newheadroom - oldheadroom);
2568 
2569 	return n;
2570 }
2571 EXPORT_SYMBOL(skb_copy_expand);
2572 
2573 /**
2574  *	__skb_pad		-	zero pad the tail of an skb
2575  *	@skb: buffer to pad
2576  *	@pad: space to pad
2577  *	@free_on_error: free buffer on error
2578  *
2579  *	Ensure that a buffer is followed by a padding area that is zero
2580  *	filled. Used by network drivers which may DMA or transfer data
2581  *	beyond the buffer end onto the wire.
2582  *
2583  *	May return error in out of memory cases. The skb is freed on error
2584  *	if @free_on_error is true.
2585  */
2586 
2587 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error)
2588 {
2589 	int err;
2590 	int ntail;
2591 
2592 	/* If the skbuff is non linear tailroom is always zero.. */
2593 	if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
2594 		memset(skb->data+skb->len, 0, pad);
2595 		return 0;
2596 	}
2597 
2598 	ntail = skb->data_len + pad - (skb->end - skb->tail);
2599 	if (likely(skb_cloned(skb) || ntail > 0)) {
2600 		err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
2601 		if (unlikely(err))
2602 			goto free_skb;
2603 	}
2604 
2605 	/* FIXME: The use of this function with non-linear skb's really needs
2606 	 * to be audited.
2607 	 */
2608 	err = skb_linearize(skb);
2609 	if (unlikely(err))
2610 		goto free_skb;
2611 
2612 	memset(skb->data + skb->len, 0, pad);
2613 	return 0;
2614 
2615 free_skb:
2616 	if (free_on_error)
2617 		kfree_skb(skb);
2618 	return err;
2619 }
2620 EXPORT_SYMBOL(__skb_pad);
2621 
2622 /**
2623  *	pskb_put - add data to the tail of a potentially fragmented buffer
2624  *	@skb: start of the buffer to use
2625  *	@tail: tail fragment of the buffer to use
2626  *	@len: amount of data to add
2627  *
2628  *	This function extends the used data area of the potentially
2629  *	fragmented buffer. @tail must be the last fragment of @skb -- or
2630  *	@skb itself. If this would exceed the total buffer size the kernel
2631  *	will panic. A pointer to the first byte of the extra data is
2632  *	returned.
2633  */
2634 
2635 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len)
2636 {
2637 	if (tail != skb) {
2638 		skb->data_len += len;
2639 		skb->len += len;
2640 	}
2641 	return skb_put(tail, len);
2642 }
2643 EXPORT_SYMBOL_GPL(pskb_put);
2644 
2645 /**
2646  *	skb_put - add data to a buffer
2647  *	@skb: buffer to use
2648  *	@len: amount of data to add
2649  *
2650  *	This function extends the used data area of the buffer. If this would
2651  *	exceed the total buffer size the kernel will panic. A pointer to the
2652  *	first byte of the extra data is returned.
2653  */
2654 void *skb_put(struct sk_buff *skb, unsigned int len)
2655 {
2656 	void *tmp = skb_tail_pointer(skb);
2657 	SKB_LINEAR_ASSERT(skb);
2658 	skb->tail += len;
2659 	skb->len  += len;
2660 	if (unlikely(skb->tail > skb->end))
2661 		skb_over_panic(skb, len, __builtin_return_address(0));
2662 	return tmp;
2663 }
2664 EXPORT_SYMBOL(skb_put);
2665 
2666 /**
2667  *	skb_push - add data to the start of a buffer
2668  *	@skb: buffer to use
2669  *	@len: amount of data to add
2670  *
2671  *	This function extends the used data area of the buffer at the buffer
2672  *	start. If this would exceed the total buffer headroom the kernel will
2673  *	panic. A pointer to the first byte of the extra data is returned.
2674  */
2675 void *skb_push(struct sk_buff *skb, unsigned int len)
2676 {
2677 	skb->data -= len;
2678 	skb->len  += len;
2679 	if (unlikely(skb->data < skb->head))
2680 		skb_under_panic(skb, len, __builtin_return_address(0));
2681 	return skb->data;
2682 }
2683 EXPORT_SYMBOL(skb_push);
2684 
2685 /**
2686  *	skb_pull - remove data from the start of a buffer
2687  *	@skb: buffer to use
2688  *	@len: amount of data to remove
2689  *
2690  *	This function removes data from the start of a buffer, returning
2691  *	the memory to the headroom. A pointer to the next data in the buffer
2692  *	is returned. Once the data has been pulled future pushes will overwrite
2693  *	the old data.
2694  */
2695 void *skb_pull(struct sk_buff *skb, unsigned int len)
2696 {
2697 	return skb_pull_inline(skb, len);
2698 }
2699 EXPORT_SYMBOL(skb_pull);
2700 
2701 /**
2702  *	skb_pull_data - remove data from the start of a buffer returning its
2703  *	original position.
2704  *	@skb: buffer to use
2705  *	@len: amount of data to remove
2706  *
2707  *	This function removes data from the start of a buffer, returning
2708  *	the memory to the headroom. A pointer to the original data in the buffer
2709  *	is returned after checking if there is enough data to pull. Once the
2710  *	data has been pulled future pushes will overwrite the old data.
2711  */
2712 void *skb_pull_data(struct sk_buff *skb, size_t len)
2713 {
2714 	void *data = skb->data;
2715 
2716 	if (skb->len < len)
2717 		return NULL;
2718 
2719 	skb_pull(skb, len);
2720 
2721 	return data;
2722 }
2723 EXPORT_SYMBOL(skb_pull_data);
2724 
2725 /**
2726  *	skb_trim - remove end from a buffer
2727  *	@skb: buffer to alter
2728  *	@len: new length
2729  *
2730  *	Cut the length of a buffer down by removing data from the tail. If
2731  *	the buffer is already under the length specified it is not modified.
2732  *	The skb must be linear.
2733  */
2734 void skb_trim(struct sk_buff *skb, unsigned int len)
2735 {
2736 	if (skb->len > len)
2737 		__skb_trim(skb, len);
2738 }
2739 EXPORT_SYMBOL(skb_trim);
2740 
2741 /* Trims skb to length len. It can change skb pointers.
2742  */
2743 
2744 int ___pskb_trim(struct sk_buff *skb, unsigned int len)
2745 {
2746 	struct sk_buff **fragp;
2747 	struct sk_buff *frag;
2748 	int offset = skb_headlen(skb);
2749 	int nfrags = skb_shinfo(skb)->nr_frags;
2750 	int i;
2751 	int err;
2752 
2753 	if (skb_cloned(skb) &&
2754 	    unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
2755 		return err;
2756 
2757 	i = 0;
2758 	if (offset >= len)
2759 		goto drop_pages;
2760 
2761 	for (; i < nfrags; i++) {
2762 		int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
2763 
2764 		if (end < len) {
2765 			offset = end;
2766 			continue;
2767 		}
2768 
2769 		skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
2770 
2771 drop_pages:
2772 		skb_shinfo(skb)->nr_frags = i;
2773 
2774 		for (; i < nfrags; i++)
2775 			skb_frag_unref(skb, i);
2776 
2777 		if (skb_has_frag_list(skb))
2778 			skb_drop_fraglist(skb);
2779 		goto done;
2780 	}
2781 
2782 	for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
2783 	     fragp = &frag->next) {
2784 		int end = offset + frag->len;
2785 
2786 		if (skb_shared(frag)) {
2787 			struct sk_buff *nfrag;
2788 
2789 			nfrag = skb_clone(frag, GFP_ATOMIC);
2790 			if (unlikely(!nfrag))
2791 				return -ENOMEM;
2792 
2793 			nfrag->next = frag->next;
2794 			consume_skb(frag);
2795 			frag = nfrag;
2796 			*fragp = frag;
2797 		}
2798 
2799 		if (end < len) {
2800 			offset = end;
2801 			continue;
2802 		}
2803 
2804 		if (end > len &&
2805 		    unlikely((err = pskb_trim(frag, len - offset))))
2806 			return err;
2807 
2808 		if (frag->next)
2809 			skb_drop_list(&frag->next);
2810 		break;
2811 	}
2812 
2813 done:
2814 	if (len > skb_headlen(skb)) {
2815 		skb->data_len -= skb->len - len;
2816 		skb->len       = len;
2817 	} else {
2818 		skb->len       = len;
2819 		skb->data_len  = 0;
2820 		skb_set_tail_pointer(skb, len);
2821 	}
2822 	if (!skb_shinfo(skb)->nr_frags && !skb_has_frag_list(skb))
2823 		skb->unreadable = 0;
2824 
2825 	if (!skb->sk || skb->destructor == sock_edemux)
2826 		skb_condense(skb);
2827 	return 0;
2828 }
2829 EXPORT_SYMBOL(___pskb_trim);
2830 
2831 static int pskb_trim_rcsum_complete(struct sk_buff *skb, unsigned int len)
2832 {
2833 	int delta = skb->len - len;
2834 
2835 	if (skb_frags_readable(skb)) {
2836 		skb->csum = csum_block_sub(skb->csum,
2837 					   skb_checksum(skb, len, delta, 0),
2838 					   len);
2839 		return 0;
2840 	}
2841 
2842 	if (len > skb_headlen(skb))
2843 		return -EFAULT;
2844 
2845 	/* The trimmed bytes are unreadable, but the remaining packet can be
2846 	 * checksummed by software after trimming.
2847 	 */
2848 	skb->ip_summed = CHECKSUM_NONE;
2849 	return 0;
2850 }
2851 
2852 /* Note : use pskb_trim_rcsum() instead of calling this directly
2853  */
2854 int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len)
2855 {
2856 	if (skb->ip_summed == CHECKSUM_COMPLETE) {
2857 		int err;
2858 
2859 		err = pskb_trim_rcsum_complete(skb, len);
2860 		if (err)
2861 			return err;
2862 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
2863 		int hdlen = (len > skb_headlen(skb)) ? skb_headlen(skb) : len;
2864 		int offset = skb_checksum_start_offset(skb) + skb->csum_offset;
2865 
2866 		if (offset + sizeof(__sum16) > hdlen)
2867 			return -EINVAL;
2868 	}
2869 	return __pskb_trim(skb, len);
2870 }
2871 EXPORT_SYMBOL(pskb_trim_rcsum_slow);
2872 
2873 /**
2874  *	__pskb_pull_tail - advance tail of skb header
2875  *	@skb: buffer to reallocate
2876  *	@delta: number of bytes to advance tail
2877  *
2878  *	The function makes a sense only on a fragmented &sk_buff,
2879  *	it expands header moving its tail forward and copying necessary
2880  *	data from fragmented part.
2881  *
2882  *	&sk_buff MUST have reference count of 1.
2883  *
2884  *	Returns %NULL (and &sk_buff does not change) if pull failed
2885  *	or value of new tail of skb in the case of success.
2886  *
2887  *	All the pointers pointing into skb header may change and must be
2888  *	reloaded after call to this function.
2889  */
2890 
2891 /* Moves tail of skb head forward, copying data from fragmented part,
2892  * when it is necessary.
2893  * 1. It may fail due to malloc failure.
2894  * 2. It may change skb pointers.
2895  *
2896  * It is pretty complicated. Luckily, it is called only in exceptional cases.
2897  */
2898 void *__pskb_pull_tail(struct sk_buff *skb, int delta)
2899 {
2900 	/* If skb has not enough free space at tail, get new one
2901 	 * plus 128 bytes for future expansions. If we have enough
2902 	 * room at tail, reallocate without expansion only if skb is cloned.
2903 	 */
2904 	int i, k, eat = (skb->tail + delta) - skb->end;
2905 
2906 	if (!skb_frags_readable(skb))
2907 		return NULL;
2908 
2909 	if (eat > 0 || skb_cloned(skb)) {
2910 		if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
2911 				     GFP_ATOMIC))
2912 			return NULL;
2913 	}
2914 
2915 	BUG_ON(skb_copy_bits(skb, skb_headlen(skb),
2916 			     skb_tail_pointer(skb), delta));
2917 
2918 	/* Optimization: no fragments, no reasons to preestimate
2919 	 * size of pulled pages. Superb.
2920 	 */
2921 	if (!skb_has_frag_list(skb))
2922 		goto pull_pages;
2923 
2924 	/* Estimate size of pulled pages. */
2925 	eat = delta;
2926 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2927 		int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
2928 
2929 		if (size >= eat)
2930 			goto pull_pages;
2931 		eat -= size;
2932 	}
2933 
2934 	/* If we need update frag list, we are in troubles.
2935 	 * Certainly, it is possible to add an offset to skb data,
2936 	 * but taking into account that pulling is expected to
2937 	 * be very rare operation, it is worth to fight against
2938 	 * further bloating skb head and crucify ourselves here instead.
2939 	 * Pure masohism, indeed. 8)8)
2940 	 */
2941 	if (eat) {
2942 		struct sk_buff *list = skb_shinfo(skb)->frag_list;
2943 		struct sk_buff *clone = NULL;
2944 		struct sk_buff *insp = NULL;
2945 
2946 		do {
2947 			if (list->len <= eat) {
2948 				/* Eaten as whole. */
2949 				eat -= list->len;
2950 				list = list->next;
2951 				insp = list;
2952 			} else {
2953 				/* Eaten partially. */
2954 				if (skb_is_gso(skb) && !list->head_frag &&
2955 				    skb_headlen(list))
2956 					skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2957 
2958 				if (skb_shared(list)) {
2959 					/* Sucks! We need to fork list. :-( */
2960 					clone = skb_clone(list, GFP_ATOMIC);
2961 					if (!clone)
2962 						return NULL;
2963 					insp = list->next;
2964 					list = clone;
2965 				} else {
2966 					/* This may be pulled without
2967 					 * problems. */
2968 					insp = list;
2969 				}
2970 				if (!pskb_pull(list, eat)) {
2971 					kfree_skb(clone);
2972 					return NULL;
2973 				}
2974 				break;
2975 			}
2976 		} while (eat);
2977 
2978 		/* Free pulled out fragments. */
2979 		while ((list = skb_shinfo(skb)->frag_list) != insp) {
2980 			skb_shinfo(skb)->frag_list = list->next;
2981 			consume_skb(list);
2982 		}
2983 		/* And insert new clone at head. */
2984 		if (clone) {
2985 			clone->next = list;
2986 			skb_shinfo(skb)->frag_list = clone;
2987 		}
2988 	}
2989 	/* Success! Now we may commit changes to skb data. */
2990 
2991 pull_pages:
2992 	eat = delta;
2993 	k = 0;
2994 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2995 		int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
2996 
2997 		if (size <= eat) {
2998 			skb_frag_unref(skb, i);
2999 			eat -= size;
3000 		} else {
3001 			skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
3002 
3003 			*frag = skb_shinfo(skb)->frags[i];
3004 			if (eat) {
3005 				skb_frag_off_add(frag, eat);
3006 				skb_frag_size_sub(frag, eat);
3007 				if (!i)
3008 					goto end;
3009 				eat = 0;
3010 			}
3011 			k++;
3012 		}
3013 	}
3014 	skb_shinfo(skb)->nr_frags = k;
3015 
3016 end:
3017 	skb->tail     += delta;
3018 	skb->data_len -= delta;
3019 
3020 	if (!skb->data_len)
3021 		skb_zcopy_clear(skb, false);
3022 
3023 	return skb_tail_pointer(skb);
3024 }
3025 EXPORT_SYMBOL(__pskb_pull_tail);
3026 
3027 /**
3028  *	skb_copy_bits - copy bits from skb to kernel buffer
3029  *	@skb: source skb
3030  *	@offset: offset in source
3031  *	@to: destination buffer
3032  *	@len: number of bytes to copy
3033  *
3034  *	Copy the specified number of bytes from the source skb to the
3035  *	destination buffer.
3036  *
3037  *	CAUTION ! :
3038  *		If its prototype is ever changed,
3039  *		check arch/{*}/net/{*}.S files,
3040  *		since it is called from BPF assembly code.
3041  */
3042 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
3043 {
3044 	int start = skb_headlen(skb);
3045 	struct sk_buff *frag_iter;
3046 	int i, copy;
3047 
3048 	if (offset > (int)skb->len - len)
3049 		goto fault;
3050 
3051 	/* Copy header. */
3052 	if ((copy = start - offset) > 0) {
3053 		if (copy > len)
3054 			copy = len;
3055 		skb_copy_from_linear_data_offset(skb, offset, to, copy);
3056 		if ((len -= copy) == 0)
3057 			return 0;
3058 		offset += copy;
3059 		to     += copy;
3060 	}
3061 
3062 	if (!skb_frags_readable(skb))
3063 		goto fault;
3064 
3065 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3066 		int end;
3067 		skb_frag_t *f = &skb_shinfo(skb)->frags[i];
3068 
3069 		WARN_ON(start > offset + len);
3070 
3071 		end = start + skb_frag_size(f);
3072 		if ((copy = end - offset) > 0) {
3073 			u32 p_off, p_len, copied;
3074 			struct page *p;
3075 			u8 *vaddr;
3076 
3077 			if (copy > len)
3078 				copy = len;
3079 
3080 			skb_frag_foreach_page(f,
3081 					      skb_frag_off(f) + offset - start,
3082 					      copy, p, p_off, p_len, copied) {
3083 				vaddr = kmap_atomic(p);
3084 				memcpy(to + copied, vaddr + p_off, p_len);
3085 				kunmap_atomic(vaddr);
3086 			}
3087 
3088 			if ((len -= copy) == 0)
3089 				return 0;
3090 			offset += copy;
3091 			to     += copy;
3092 		}
3093 		start = end;
3094 	}
3095 
3096 	skb_walk_frags(skb, frag_iter) {
3097 		int end;
3098 
3099 		WARN_ON(start > offset + len);
3100 
3101 		end = start + frag_iter->len;
3102 		if ((copy = end - offset) > 0) {
3103 			if (copy > len)
3104 				copy = len;
3105 			if (skb_copy_bits(frag_iter, offset - start, to, copy))
3106 				goto fault;
3107 			if ((len -= copy) == 0)
3108 				return 0;
3109 			offset += copy;
3110 			to     += copy;
3111 		}
3112 		start = end;
3113 	}
3114 
3115 	if (!len)
3116 		return 0;
3117 
3118 fault:
3119 	return -EFAULT;
3120 }
3121 EXPORT_SYMBOL(skb_copy_bits);
3122 
3123 /*
3124  * Callback from splice_to_pipe(), if we need to release some pages
3125  * at the end of the spd in case we error'ed out in filling the pipe.
3126  */
3127 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
3128 {
3129 	put_page(spd->pages[i]);
3130 }
3131 
3132 static struct page *linear_to_page(struct page *page, unsigned int *len,
3133 				   unsigned int *offset,
3134 				   struct sock *sk)
3135 {
3136 	struct page_frag *pfrag = sk_page_frag(sk);
3137 
3138 	if (!sk_page_frag_refill(sk, pfrag))
3139 		return NULL;
3140 
3141 	*len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);
3142 
3143 	memcpy(page_address(pfrag->page) + pfrag->offset,
3144 	       page_address(page) + *offset, *len);
3145 	*offset = pfrag->offset;
3146 	pfrag->offset += *len;
3147 
3148 	return pfrag->page;
3149 }
3150 
3151 static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
3152 			     struct page *page,
3153 			     unsigned int offset)
3154 {
3155 	return	spd->nr_pages &&
3156 		spd->pages[spd->nr_pages - 1] == page &&
3157 		(spd->partial[spd->nr_pages - 1].offset +
3158 		 spd->partial[spd->nr_pages - 1].len == offset);
3159 }
3160 
3161 /*
3162  * Fill page/offset/length into spd, if it can hold more pages.
3163  */
3164 static bool spd_fill_page(struct splice_pipe_desc *spd, struct page *page,
3165 			  unsigned int *len, unsigned int offset, bool linear,
3166 			  struct sock *sk)
3167 {
3168 	if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
3169 		return true;
3170 
3171 	if (linear) {
3172 		page = linear_to_page(page, len, &offset, sk);
3173 		if (!page)
3174 			return true;
3175 	}
3176 	if (spd_can_coalesce(spd, page, offset)) {
3177 		spd->partial[spd->nr_pages - 1].len += *len;
3178 		return false;
3179 	}
3180 	get_page(page);
3181 	spd->pages[spd->nr_pages] = page;
3182 	spd->partial[spd->nr_pages].len = *len;
3183 	spd->partial[spd->nr_pages].offset = offset;
3184 	spd->nr_pages++;
3185 
3186 	return false;
3187 }
3188 
3189 static bool __splice_segment(struct page *page, unsigned int poff,
3190 			     unsigned int plen, unsigned int *off,
3191 			     unsigned int *len,
3192 			     struct splice_pipe_desc *spd, bool linear,
3193 			     struct sock *sk)
3194 {
3195 	if (!*len)
3196 		return true;
3197 
3198 	/* skip this segment if already processed */
3199 	if (*off >= plen) {
3200 		*off -= plen;
3201 		return false;
3202 	}
3203 
3204 	/* ignore any bits we already processed */
3205 	poff += *off;
3206 	plen -= *off;
3207 	*off = 0;
3208 
3209 	do {
3210 		unsigned int flen = min(*len, plen);
3211 
3212 		if (spd_fill_page(spd, page, &flen, poff, linear, sk))
3213 			return true;
3214 		poff += flen;
3215 		plen -= flen;
3216 		*len -= flen;
3217 		if (!*len)
3218 			return true;
3219 	} while (plen);
3220 
3221 	return false;
3222 }
3223 
3224 /*
3225  * Map linear and fragment data from the skb to spd. It reports true if the
3226  * pipe is full or if we already spliced the requested length.
3227  */
3228 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
3229 			      unsigned int *offset, unsigned int *len,
3230 			      struct splice_pipe_desc *spd, struct sock *sk)
3231 {
3232 	struct sk_buff *iter;
3233 	int seg;
3234 
3235 	/* map the linear part :
3236 	 * If skb->head_frag is set, this 'linear' part is backed by a
3237 	 * fragment, and if the head is not shared with any clones then
3238 	 * we can avoid a copy since we own the head portion of this page.
3239 	 */
3240 	if (__splice_segment(virt_to_page(skb->data),
3241 			     (unsigned long) skb->data & (PAGE_SIZE - 1),
3242 			     skb_headlen(skb),
3243 			     offset, len, spd,
3244 			     skb_head_is_locked(skb),
3245 			     sk))
3246 		return true;
3247 
3248 	/*
3249 	 * then map the fragments
3250 	 */
3251 	if (!skb_frags_readable(skb))
3252 		return false;
3253 
3254 	for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
3255 		const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
3256 
3257 		if (WARN_ON_ONCE(!skb_frag_page(f)))
3258 			return false;
3259 
3260 		if (__splice_segment(skb_frag_page(f),
3261 				     skb_frag_off(f), skb_frag_size(f),
3262 				     offset, len, spd, false, sk))
3263 			return true;
3264 	}
3265 
3266 	skb_walk_frags(skb, iter) {
3267 		if (*offset >= iter->len) {
3268 			*offset -= iter->len;
3269 			continue;
3270 		}
3271 		/* __skb_splice_bits() only fails if the output has no room
3272 		 * left, so no point in going over the frag_list for the error
3273 		 * case.
3274 		 */
3275 		if (__skb_splice_bits(iter, pipe, offset, len, spd, sk))
3276 			return true;
3277 	}
3278 
3279 	return false;
3280 }
3281 
3282 /*
3283  * Map data from the skb to a pipe. Should handle both the linear part,
3284  * the fragments, and the frag list.
3285  */
3286 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
3287 		    struct pipe_inode_info *pipe, unsigned int tlen,
3288 		    unsigned int flags)
3289 {
3290 	struct partial_page partial[MAX_SKB_FRAGS];
3291 	struct page *pages[MAX_SKB_FRAGS];
3292 	struct splice_pipe_desc spd = {
3293 		.pages = pages,
3294 		.partial = partial,
3295 		.nr_pages_max = MAX_SKB_FRAGS,
3296 		.ops = &default_pipe_buf_ops,
3297 		.spd_release = sock_spd_release,
3298 	};
3299 	int ret = 0;
3300 
3301 	__skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk);
3302 
3303 	if (spd.nr_pages)
3304 		ret = splice_to_pipe(pipe, &spd);
3305 
3306 	return ret;
3307 }
3308 EXPORT_SYMBOL_GPL(skb_splice_bits);
3309 
3310 static int sendmsg_locked(struct sock *sk, struct msghdr *msg)
3311 {
3312 	struct socket *sock = sk->sk_socket;
3313 	size_t size = msg_data_left(msg);
3314 
3315 	if (!sock)
3316 		return -EINVAL;
3317 
3318 	if (!sock->ops->sendmsg_locked)
3319 		return sock_no_sendmsg_locked(sk, msg, size);
3320 
3321 	return sock->ops->sendmsg_locked(sk, msg, size);
3322 }
3323 
3324 static int sendmsg_unlocked(struct sock *sk, struct msghdr *msg)
3325 {
3326 	struct socket *sock = sk->sk_socket;
3327 
3328 	if (!sock)
3329 		return -EINVAL;
3330 	return sock_sendmsg(sock, msg);
3331 }
3332 
3333 typedef int (*sendmsg_func)(struct sock *sk, struct msghdr *msg);
3334 static int __skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset,
3335 			   int len, sendmsg_func sendmsg, int flags)
3336 {
3337 	int more_hint = sk_is_tcp(sk) ? MSG_MORE : 0;
3338 	unsigned int orig_len = len;
3339 	struct sk_buff *head = skb;
3340 	unsigned short fragidx;
3341 	int slen, ret;
3342 
3343 do_frag_list:
3344 
3345 	/* Deal with head data */
3346 	while (offset < skb_headlen(skb) && len) {
3347 		struct kvec kv;
3348 		struct msghdr msg;
3349 
3350 		slen = min_t(int, len, skb_headlen(skb) - offset);
3351 		kv.iov_base = skb->data + offset;
3352 		kv.iov_len = slen;
3353 		memset(&msg, 0, sizeof(msg));
3354 		msg.msg_flags = MSG_DONTWAIT | flags;
3355 		if (slen < len)
3356 			msg.msg_flags |= more_hint;
3357 
3358 		iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, &kv, 1, slen);
3359 		ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked,
3360 				      sendmsg_unlocked, sk, &msg);
3361 		if (ret <= 0)
3362 			goto error;
3363 
3364 		offset += ret;
3365 		len -= ret;
3366 	}
3367 
3368 	/* All the data was skb head? */
3369 	if (!len)
3370 		goto out;
3371 
3372 	/* Make offset relative to start of frags */
3373 	offset -= skb_headlen(skb);
3374 
3375 	/* Find where we are in frag list */
3376 	for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
3377 		skb_frag_t *frag  = &skb_shinfo(skb)->frags[fragidx];
3378 
3379 		if (offset < skb_frag_size(frag))
3380 			break;
3381 
3382 		offset -= skb_frag_size(frag);
3383 	}
3384 
3385 	for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
3386 		skb_frag_t *frag  = &skb_shinfo(skb)->frags[fragidx];
3387 
3388 		slen = min_t(size_t, len, skb_frag_size(frag) - offset);
3389 
3390 		while (slen) {
3391 			struct bio_vec bvec;
3392 			struct msghdr msg = {
3393 				.msg_flags = MSG_SPLICE_PAGES | MSG_DONTWAIT |
3394 					     flags,
3395 			};
3396 
3397 			if (slen < len)
3398 				msg.msg_flags |= more_hint;
3399 			bvec_set_page(&bvec, skb_frag_page(frag), slen,
3400 				      skb_frag_off(frag) + offset);
3401 			iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1,
3402 				      slen);
3403 
3404 			ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked,
3405 					      sendmsg_unlocked, sk, &msg);
3406 			if (ret <= 0)
3407 				goto error;
3408 
3409 			len -= ret;
3410 			offset += ret;
3411 			slen -= ret;
3412 		}
3413 
3414 		offset = 0;
3415 	}
3416 
3417 	if (len) {
3418 		/* Process any frag lists */
3419 
3420 		if (skb == head) {
3421 			if (skb_has_frag_list(skb)) {
3422 				skb = skb_shinfo(skb)->frag_list;
3423 				goto do_frag_list;
3424 			}
3425 		} else if (skb->next) {
3426 			skb = skb->next;
3427 			goto do_frag_list;
3428 		}
3429 	}
3430 
3431 out:
3432 	return orig_len - len;
3433 
3434 error:
3435 	return orig_len == len ? ret : orig_len - len;
3436 }
3437 
3438 /* Send skb data on a socket. Socket must be locked. */
3439 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
3440 			 int len)
3441 {
3442 	return __skb_send_sock(sk, skb, offset, len, sendmsg_locked, 0);
3443 }
3444 EXPORT_SYMBOL_GPL(skb_send_sock_locked);
3445 
3446 int skb_send_sock_locked_with_flags(struct sock *sk, struct sk_buff *skb,
3447 				    int offset, int len, int flags)
3448 {
3449 	return __skb_send_sock(sk, skb, offset, len, sendmsg_locked, flags);
3450 }
3451 EXPORT_SYMBOL_GPL(skb_send_sock_locked_with_flags);
3452 
3453 /* Send skb data on a socket. Socket must be unlocked. */
3454 int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len)
3455 {
3456 	return __skb_send_sock(sk, skb, offset, len, sendmsg_unlocked, 0);
3457 }
3458 
3459 /**
3460  *	skb_store_bits - store bits from kernel buffer to skb
3461  *	@skb: destination buffer
3462  *	@offset: offset in destination
3463  *	@from: source buffer
3464  *	@len: number of bytes to copy
3465  *
3466  *	Copy the specified number of bytes from the source buffer to the
3467  *	destination skb.  This function handles all the messy bits of
3468  *	traversing fragment lists and such.
3469  */
3470 
3471 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
3472 {
3473 	int start = skb_headlen(skb);
3474 	struct sk_buff *frag_iter;
3475 	int i, copy;
3476 
3477 	if (offset > (int)skb->len - len)
3478 		goto fault;
3479 
3480 	if ((copy = start - offset) > 0) {
3481 		if (copy > len)
3482 			copy = len;
3483 		skb_copy_to_linear_data_offset(skb, offset, from, copy);
3484 		if ((len -= copy) == 0)
3485 			return 0;
3486 		offset += copy;
3487 		from += copy;
3488 	}
3489 
3490 	if (!skb_frags_readable(skb))
3491 		goto fault;
3492 
3493 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3494 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3495 		int end;
3496 
3497 		WARN_ON(start > offset + len);
3498 
3499 		end = start + skb_frag_size(frag);
3500 		if ((copy = end - offset) > 0) {
3501 			u32 p_off, p_len, copied;
3502 			struct page *p;
3503 			u8 *vaddr;
3504 
3505 			if (copy > len)
3506 				copy = len;
3507 
3508 			skb_frag_foreach_page(frag,
3509 					      skb_frag_off(frag) + offset - start,
3510 					      copy, p, p_off, p_len, copied) {
3511 				vaddr = kmap_atomic(p);
3512 				memcpy(vaddr + p_off, from + copied, p_len);
3513 				kunmap_atomic(vaddr);
3514 			}
3515 
3516 			if ((len -= copy) == 0)
3517 				return 0;
3518 			offset += copy;
3519 			from += copy;
3520 		}
3521 		start = end;
3522 	}
3523 
3524 	skb_walk_frags(skb, frag_iter) {
3525 		int end;
3526 
3527 		WARN_ON(start > offset + len);
3528 
3529 		end = start + frag_iter->len;
3530 		if ((copy = end - offset) > 0) {
3531 			if (copy > len)
3532 				copy = len;
3533 			if (skb_store_bits(frag_iter, offset - start,
3534 					   from, copy))
3535 				goto fault;
3536 			if ((len -= copy) == 0)
3537 				return 0;
3538 			offset += copy;
3539 			from += copy;
3540 		}
3541 		start = end;
3542 	}
3543 	if (!len)
3544 		return 0;
3545 
3546 fault:
3547 	return -EFAULT;
3548 }
3549 EXPORT_SYMBOL(skb_store_bits);
3550 
3551 /* Checksum skb data. */
3552 __wsum skb_checksum(const struct sk_buff *skb, int offset, int len, __wsum csum)
3553 {
3554 	int start = skb_headlen(skb);
3555 	int i, copy = start - offset;
3556 	struct sk_buff *frag_iter;
3557 	int pos = 0;
3558 
3559 	/* Checksum header. */
3560 	if (copy > 0) {
3561 		if (copy > len)
3562 			copy = len;
3563 		csum = csum_partial(skb->data + offset, copy, csum);
3564 		if ((len -= copy) == 0)
3565 			return csum;
3566 		offset += copy;
3567 		pos	= copy;
3568 	}
3569 
3570 	if (WARN_ON_ONCE(!skb_frags_readable(skb)))
3571 		return 0;
3572 
3573 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3574 		int end;
3575 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3576 
3577 		WARN_ON(start > offset + len);
3578 
3579 		end = start + skb_frag_size(frag);
3580 		if ((copy = end - offset) > 0) {
3581 			u32 p_off, p_len, copied;
3582 			struct page *p;
3583 			__wsum csum2;
3584 			u8 *vaddr;
3585 
3586 			if (copy > len)
3587 				copy = len;
3588 
3589 			skb_frag_foreach_page(frag,
3590 					      skb_frag_off(frag) + offset - start,
3591 					      copy, p, p_off, p_len, copied) {
3592 				vaddr = kmap_atomic(p);
3593 				csum2 = csum_partial(vaddr + p_off, p_len, 0);
3594 				kunmap_atomic(vaddr);
3595 				csum = csum_block_add(csum, csum2, pos);
3596 				pos += p_len;
3597 			}
3598 
3599 			if (!(len -= copy))
3600 				return csum;
3601 			offset += copy;
3602 		}
3603 		start = end;
3604 	}
3605 
3606 	skb_walk_frags(skb, frag_iter) {
3607 		int end;
3608 
3609 		WARN_ON(start > offset + len);
3610 
3611 		end = start + frag_iter->len;
3612 		if ((copy = end - offset) > 0) {
3613 			__wsum csum2;
3614 			if (copy > len)
3615 				copy = len;
3616 			csum2 = skb_checksum(frag_iter, offset - start, copy,
3617 					     0);
3618 			csum = csum_block_add(csum, csum2, pos);
3619 			if ((len -= copy) == 0)
3620 				return csum;
3621 			offset += copy;
3622 			pos    += copy;
3623 		}
3624 		start = end;
3625 	}
3626 	if (unlikely(len)) {
3627 		DEBUG_NET_WARN_ON_ONCE(1);
3628 		return 0;
3629 	}
3630 
3631 	return csum;
3632 }
3633 EXPORT_SYMBOL(skb_checksum);
3634 
3635 /* Both of above in one bottle. */
3636 
3637 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
3638 				    u8 *to, int len)
3639 {
3640 	int start = skb_headlen(skb);
3641 	int i, copy = start - offset;
3642 	struct sk_buff *frag_iter;
3643 	int pos = 0;
3644 	__wsum csum = 0;
3645 
3646 	/* Copy header. */
3647 	if (copy > 0) {
3648 		if (copy > len)
3649 			copy = len;
3650 		csum = csum_partial_copy_nocheck(skb->data + offset, to,
3651 						 copy);
3652 		if ((len -= copy) == 0)
3653 			return csum;
3654 		offset += copy;
3655 		to     += copy;
3656 		pos	= copy;
3657 	}
3658 
3659 	if (!skb_frags_readable(skb)) {
3660 		/* Don't hand the caller a buffer with stale bytes in it. */
3661 		if (len > 0)
3662 			memset(to, 0, len);
3663 		return 0;
3664 	}
3665 
3666 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3667 		int end;
3668 
3669 		WARN_ON(start > offset + len);
3670 
3671 		end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
3672 		if ((copy = end - offset) > 0) {
3673 			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3674 			u32 p_off, p_len, copied;
3675 			struct page *p;
3676 			__wsum csum2;
3677 			u8 *vaddr;
3678 
3679 			if (copy > len)
3680 				copy = len;
3681 
3682 			skb_frag_foreach_page(frag,
3683 					      skb_frag_off(frag) + offset - start,
3684 					      copy, p, p_off, p_len, copied) {
3685 				vaddr = kmap_atomic(p);
3686 				csum2 = csum_partial_copy_nocheck(vaddr + p_off,
3687 								  to + copied,
3688 								  p_len);
3689 				kunmap_atomic(vaddr);
3690 				csum = csum_block_add(csum, csum2, pos);
3691 				pos += p_len;
3692 			}
3693 
3694 			if (!(len -= copy))
3695 				return csum;
3696 			offset += copy;
3697 			to     += copy;
3698 		}
3699 		start = end;
3700 	}
3701 
3702 	skb_walk_frags(skb, frag_iter) {
3703 		__wsum csum2;
3704 		int end;
3705 
3706 		WARN_ON(start > offset + len);
3707 
3708 		end = start + frag_iter->len;
3709 		if ((copy = end - offset) > 0) {
3710 			if (copy > len)
3711 				copy = len;
3712 			csum2 = skb_copy_and_csum_bits(frag_iter,
3713 						       offset - start,
3714 						       to, copy);
3715 			csum = csum_block_add(csum, csum2, pos);
3716 			if ((len -= copy) == 0)
3717 				return csum;
3718 			offset += copy;
3719 			to     += copy;
3720 			pos    += copy;
3721 		}
3722 		start = end;
3723 	}
3724 	if (unlikely(len)) {
3725 		DEBUG_NET_WARN_ON_ONCE(1);
3726 		/* Don't hand the caller a buffer with stale bytes in it. */
3727 		if (len > 0)
3728 			memset(to, 0, len);
3729 		return 0;
3730 	}
3731 	return csum;
3732 }
3733 EXPORT_SYMBOL(skb_copy_and_csum_bits);
3734 
3735 #ifdef CONFIG_NET_CRC32C
3736 u32 skb_crc32c(const struct sk_buff *skb, int offset, int len, u32 crc)
3737 {
3738 	int start = skb_headlen(skb);
3739 	int i, copy = start - offset;
3740 	struct sk_buff *frag_iter;
3741 
3742 	if (copy > 0) {
3743 		copy = min(copy, len);
3744 		crc = crc32c(crc, skb->data + offset, copy);
3745 		len -= copy;
3746 		if (len == 0)
3747 			return crc;
3748 		offset += copy;
3749 	}
3750 
3751 	if (WARN_ON_ONCE(!skb_frags_readable(skb)))
3752 		return 0;
3753 
3754 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3755 		int end;
3756 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3757 
3758 		WARN_ON(start > offset + len);
3759 
3760 		end = start + skb_frag_size(frag);
3761 		copy = end - offset;
3762 		if (copy > 0) {
3763 			u32 p_off, p_len, copied;
3764 			struct page *p;
3765 			u8 *vaddr;
3766 
3767 			copy = min(copy, len);
3768 			skb_frag_foreach_page(frag,
3769 					      skb_frag_off(frag) + offset - start,
3770 					      copy, p, p_off, p_len, copied) {
3771 				vaddr = kmap_atomic(p);
3772 				crc = crc32c(crc, vaddr + p_off, p_len);
3773 				kunmap_atomic(vaddr);
3774 			}
3775 			len -= copy;
3776 			if (len == 0)
3777 				return crc;
3778 			offset += copy;
3779 		}
3780 		start = end;
3781 	}
3782 
3783 	skb_walk_frags(skb, frag_iter) {
3784 		int end;
3785 
3786 		WARN_ON(start > offset + len);
3787 
3788 		end = start + frag_iter->len;
3789 		copy = end - offset;
3790 		if (copy > 0) {
3791 			copy = min(copy, len);
3792 			crc = skb_crc32c(frag_iter, offset - start, copy, crc);
3793 			len -= copy;
3794 			if (len == 0)
3795 				return crc;
3796 			offset += copy;
3797 		}
3798 		start = end;
3799 	}
3800 	if (unlikely(len)) {
3801 		DEBUG_NET_WARN_ON_ONCE(1);
3802 		return 0;
3803 	}
3804 
3805 	return crc;
3806 }
3807 EXPORT_SYMBOL(skb_crc32c);
3808 #endif /* CONFIG_NET_CRC32C */
3809 
3810 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len)
3811 {
3812 	__sum16 sum;
3813 
3814 	sum = csum_fold(skb_checksum(skb, 0, len, skb->csum));
3815 	/* See comments in __skb_checksum_complete(). */
3816 	if (likely(!sum)) {
3817 		if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
3818 		    !skb->csum_complete_sw)
3819 			netdev_rx_csum_fault(skb->dev, skb);
3820 	}
3821 	if (!skb_shared(skb))
3822 		skb->csum_valid = !sum;
3823 	return sum;
3824 }
3825 EXPORT_SYMBOL(__skb_checksum_complete_head);
3826 
3827 /* This function assumes skb->csum already holds pseudo header's checksum,
3828  * which has been changed from the hardware checksum, for example, by
3829  * __skb_checksum_validate_complete(). And, the original skb->csum must
3830  * have been validated unsuccessfully for CHECKSUM_COMPLETE case.
3831  *
3832  * It returns non-zero if the recomputed checksum is still invalid, otherwise
3833  * zero. The new checksum is stored back into skb->csum unless the skb is
3834  * shared.
3835  */
3836 __sum16 __skb_checksum_complete(struct sk_buff *skb)
3837 {
3838 	__wsum csum;
3839 	__sum16 sum;
3840 
3841 	csum = skb_checksum(skb, 0, skb->len, 0);
3842 
3843 	sum = csum_fold(csum_add(skb->csum, csum));
3844 	/* This check is inverted, because we already knew the hardware
3845 	 * checksum is invalid before calling this function. So, if the
3846 	 * re-computed checksum is valid instead, then we have a mismatch
3847 	 * between the original skb->csum and skb_checksum(). This means either
3848 	 * the original hardware checksum is incorrect or we screw up skb->csum
3849 	 * when moving skb->data around.
3850 	 */
3851 	if (likely(!sum)) {
3852 		if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
3853 		    !skb->csum_complete_sw)
3854 			netdev_rx_csum_fault(skb->dev, skb);
3855 	}
3856 
3857 	if (!skb_shared(skb)) {
3858 		/* Save full packet checksum */
3859 		skb->csum = csum;
3860 		skb->ip_summed = CHECKSUM_COMPLETE;
3861 		skb->csum_complete_sw = 1;
3862 		skb->csum_valid = !sum;
3863 	}
3864 
3865 	return sum;
3866 }
3867 EXPORT_SYMBOL(__skb_checksum_complete);
3868 
3869  /**
3870  *	skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy()
3871  *	@from: source buffer
3872  *
3873  *	Calculates the amount of linear headroom needed in the 'to' skb passed
3874  *	into skb_zerocopy().
3875  */
3876 unsigned int
3877 skb_zerocopy_headlen(const struct sk_buff *from)
3878 {
3879 	unsigned int hlen = 0;
3880 
3881 	if (!from->head_frag ||
3882 	    skb_headlen(from) < L1_CACHE_BYTES ||
3883 	    skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) {
3884 		hlen = skb_headlen(from);
3885 		if (!hlen)
3886 			hlen = from->len;
3887 	}
3888 
3889 	if (skb_has_frag_list(from))
3890 		hlen = from->len;
3891 
3892 	return hlen;
3893 }
3894 EXPORT_SYMBOL_GPL(skb_zerocopy_headlen);
3895 
3896 /**
3897  *	skb_zerocopy - Zero copy skb to skb
3898  *	@to: destination buffer
3899  *	@from: source buffer
3900  *	@len: number of bytes to copy from source buffer
3901  *	@hlen: size of linear headroom in destination buffer
3902  *
3903  *	Copies up to `len` bytes from `from` to `to` by creating references
3904  *	to the frags in the source buffer.
3905  *
3906  *	The `hlen` as calculated by skb_zerocopy_headlen() specifies the
3907  *	headroom in the `to` buffer.
3908  *
3909  *	Return value:
3910  *	0: everything is OK
3911  *	-ENOMEM: couldn't orphan frags of @from due to lack of memory
3912  *	-EFAULT: skb_copy_bits() found some problem with skb geometry, or readable head
3913  *      payload would be mixed with unreadable frags.
3914  */
3915 int
3916 skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen)
3917 {
3918 	int i, j = 0;
3919 	int plen = 0; /* length of skb->head fragment */
3920 	int ret;
3921 	struct page *page;
3922 	unsigned int offset;
3923 
3924 	if (unlikely(!from->head_frag && !hlen)) {
3925 		DEBUG_NET_WARN_ON_ONCE(1);
3926 		return -EFAULT;
3927 	}
3928 
3929 	/* dont bother with small payloads */
3930 	if (len <= skb_tailroom(to))
3931 		return skb_copy_bits(from, 0, skb_put(to, len), len);
3932 
3933 	to->pp_recycle = from->pp_recycle;
3934 	if (hlen) {
3935 		ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen);
3936 		if (unlikely(ret))
3937 			return ret;
3938 		len -= hlen;
3939 	} else {
3940 		plen = min_t(int, skb_headlen(from), len);
3941 		if (plen) {
3942 			page = virt_to_head_page(from->head);
3943 			offset = from->data - (unsigned char *)page_address(page);
3944 			__skb_fill_netmem_desc(to, 0, page_to_netmem(page),
3945 					       offset, plen);
3946 			skb_frag_ref(to, 0);
3947 			j = 1;
3948 			len -= plen;
3949 		}
3950 	}
3951 
3952 	if (!skb_frags_readable(from) && j > 0 && len) {
3953 		skb_frag_unref(to, 0);
3954 		return -EFAULT;
3955 	}
3956 
3957 	skb_len_add(to, len + plen);
3958 
3959 	if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) {
3960 		if (j > 0)
3961 			skb_frag_unref(to, 0);
3962 		return -ENOMEM;
3963 	}
3964 	skb_zerocopy_clone(to, from, GFP_ATOMIC);
3965 
3966 	for (i = 0; i < skb_shinfo(from)->nr_frags; i++) {
3967 		int size;
3968 
3969 		if (!len)
3970 			break;
3971 		skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i];
3972 		size = min_t(int, skb_frag_size(&skb_shinfo(to)->frags[j]),
3973 					len);
3974 		skb_frag_size_set(&skb_shinfo(to)->frags[j], size);
3975 		len -= size;
3976 		skb_frag_ref(to, j);
3977 		j++;
3978 	}
3979 	skb_shinfo(to)->nr_frags = j;
3980 
3981 	skb_shinfo(to)->flags |= skb_shinfo(from)->flags & SKBFL_SHARED_FRAG;
3982 
3983 	if (i > 0 && from->unreadable)
3984 		to->unreadable = 1;
3985 
3986 	return 0;
3987 }
3988 EXPORT_SYMBOL_GPL(skb_zerocopy);
3989 
3990 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
3991 {
3992 	__wsum csum;
3993 	long csstart;
3994 
3995 	if (skb->ip_summed == CHECKSUM_PARTIAL)
3996 		csstart = skb_checksum_start_offset(skb);
3997 	else
3998 		csstart = skb_headlen(skb);
3999 
4000 	if (unlikely(csstart < 0 || csstart > skb_headlen(skb) ||
4001 		     (skb->ip_summed == CHECKSUM_PARTIAL &&
4002 		      csstart + skb->csum_offset + sizeof(__sum16) >
4003 		      skb->len))) {
4004 		DEBUG_NET_WARN_ON_ONCE(1);
4005 		/* Send the frame without the checksum filled in, or send
4006 		 * zeroes if we can't even copy it.
4007 		 */
4008 		if (skb_copy_bits(skb, 0, to, skb->len))
4009 			memset(to, 0, skb->len);
4010 		return;
4011 	}
4012 
4013 	skb_copy_from_linear_data(skb, to, csstart);
4014 
4015 	csum = 0;
4016 	if (csstart != skb->len)
4017 		csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
4018 					      skb->len - csstart);
4019 
4020 	if (skb->ip_summed == CHECKSUM_PARTIAL) {
4021 		long csstuff = csstart + skb->csum_offset;
4022 
4023 		*((__sum16 *)(to + csstuff)) = csum_fold(csum);
4024 	}
4025 }
4026 EXPORT_SYMBOL(skb_copy_and_csum_dev);
4027 
4028 /**
4029  *	skb_dequeue - remove from the head of the queue
4030  *	@list: list to dequeue from
4031  *
4032  *	Remove the head of the list. The list lock is taken so the function
4033  *	may be used safely with other locking list functions. The head item is
4034  *	returned or %NULL if the list is empty.
4035  */
4036 
4037 struct sk_buff *skb_dequeue(struct sk_buff_head *list)
4038 {
4039 	unsigned long flags;
4040 	struct sk_buff *result;
4041 
4042 	spin_lock_irqsave(&list->lock, flags);
4043 	result = __skb_dequeue(list);
4044 	spin_unlock_irqrestore(&list->lock, flags);
4045 	return result;
4046 }
4047 EXPORT_SYMBOL(skb_dequeue);
4048 
4049 /**
4050  *	skb_dequeue_tail - remove from the tail of the queue
4051  *	@list: list to dequeue from
4052  *
4053  *	Remove the tail of the list. The list lock is taken so the function
4054  *	may be used safely with other locking list functions. The tail item is
4055  *	returned or %NULL if the list is empty.
4056  */
4057 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
4058 {
4059 	unsigned long flags;
4060 	struct sk_buff *result;
4061 
4062 	spin_lock_irqsave(&list->lock, flags);
4063 	result = __skb_dequeue_tail(list);
4064 	spin_unlock_irqrestore(&list->lock, flags);
4065 	return result;
4066 }
4067 EXPORT_SYMBOL(skb_dequeue_tail);
4068 
4069 /**
4070  *	skb_queue_purge_reason - empty a list
4071  *	@list: list to empty
4072  *	@reason: drop reason
4073  *
4074  *	Delete all buffers on an &sk_buff list. Each buffer is removed from
4075  *	the list and one reference dropped. This function takes the list
4076  *	lock and is atomic with respect to other list locking functions.
4077  */
4078 void skb_queue_purge_reason(struct sk_buff_head *list,
4079 			    enum skb_drop_reason reason)
4080 {
4081 	struct sk_buff_head tmp;
4082 	unsigned long flags;
4083 
4084 	if (skb_queue_empty_lockless(list))
4085 		return;
4086 
4087 	__skb_queue_head_init(&tmp);
4088 
4089 	spin_lock_irqsave(&list->lock, flags);
4090 	skb_queue_splice_init(list, &tmp);
4091 	spin_unlock_irqrestore(&list->lock, flags);
4092 
4093 	__skb_queue_purge_reason(&tmp, reason);
4094 }
4095 EXPORT_SYMBOL(skb_queue_purge_reason);
4096 
4097 /**
4098  *	skb_rbtree_purge - empty a skb rbtree
4099  *	@root: root of the rbtree to empty
4100  *	Return value: the sum of truesizes of all purged skbs.
4101  *
4102  *	Delete all buffers on an &sk_buff rbtree. Each buffer is removed from
4103  *	the list and one reference dropped. This function does not take
4104  *	any lock. Synchronization should be handled by the caller (e.g., TCP
4105  *	out-of-order queue is protected by the socket lock).
4106  */
4107 unsigned int skb_rbtree_purge(struct rb_root *root)
4108 {
4109 	struct rb_node *p = rb_first(root);
4110 	unsigned int sum = 0;
4111 
4112 	while (p) {
4113 		struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode);
4114 
4115 		p = rb_next(p);
4116 		rb_erase(&skb->rbnode, root);
4117 		sum += skb->truesize;
4118 		kfree_skb(skb);
4119 	}
4120 	return sum;
4121 }
4122 
4123 void skb_errqueue_purge(struct sk_buff_head *list)
4124 {
4125 	struct sk_buff *skb, *next;
4126 	struct sk_buff_head kill;
4127 	unsigned long flags;
4128 
4129 	__skb_queue_head_init(&kill);
4130 
4131 	spin_lock_irqsave(&list->lock, flags);
4132 	skb_queue_walk_safe(list, skb, next) {
4133 		if (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ZEROCOPY ||
4134 		    SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_TIMESTAMPING)
4135 			continue;
4136 		__skb_unlink(skb, list);
4137 		__skb_queue_tail(&kill, skb);
4138 	}
4139 	spin_unlock_irqrestore(&list->lock, flags);
4140 	__skb_queue_purge(&kill);
4141 }
4142 EXPORT_SYMBOL(skb_errqueue_purge);
4143 
4144 /**
4145  *	skb_queue_head - queue a buffer at the list head
4146  *	@list: list to use
4147  *	@newsk: buffer to queue
4148  *
4149  *	Queue a buffer at the start of the list. This function takes the
4150  *	list lock and can be used safely with other locking &sk_buff functions
4151  *	safely.
4152  *
4153  *	A buffer cannot be placed on two lists at the same time.
4154  */
4155 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
4156 {
4157 	unsigned long flags;
4158 
4159 	spin_lock_irqsave(&list->lock, flags);
4160 	__skb_queue_head(list, newsk);
4161 	spin_unlock_irqrestore(&list->lock, flags);
4162 }
4163 EXPORT_SYMBOL(skb_queue_head);
4164 
4165 /**
4166  *	skb_queue_tail - queue a buffer at the list tail
4167  *	@list: list to use
4168  *	@newsk: buffer to queue
4169  *
4170  *	Queue a buffer at the tail of the list. This function takes the
4171  *	list lock and can be used safely with other locking &sk_buff functions
4172  *	safely.
4173  *
4174  *	A buffer cannot be placed on two lists at the same time.
4175  */
4176 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
4177 {
4178 	unsigned long flags;
4179 
4180 	spin_lock_irqsave(&list->lock, flags);
4181 	__skb_queue_tail(list, newsk);
4182 	spin_unlock_irqrestore(&list->lock, flags);
4183 }
4184 EXPORT_SYMBOL(skb_queue_tail);
4185 
4186 /**
4187  *	skb_unlink	-	remove a buffer from a list
4188  *	@skb: buffer to remove
4189  *	@list: list to use
4190  *
4191  *	Remove a packet from a list. The list locks are taken and this
4192  *	function is atomic with respect to other list locked calls
4193  *
4194  *	You must know what list the SKB is on.
4195  */
4196 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
4197 {
4198 	unsigned long flags;
4199 
4200 	spin_lock_irqsave(&list->lock, flags);
4201 	__skb_unlink(skb, list);
4202 	spin_unlock_irqrestore(&list->lock, flags);
4203 }
4204 EXPORT_SYMBOL(skb_unlink);
4205 
4206 /**
4207  *	skb_append	-	append a buffer
4208  *	@old: buffer to insert after
4209  *	@newsk: buffer to insert
4210  *	@list: list to use
4211  *
4212  *	Place a packet after a given packet in a list. The list locks are taken
4213  *	and this function is atomic with respect to other list locked calls.
4214  *	A buffer cannot be placed on two lists at the same time.
4215  */
4216 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
4217 {
4218 	unsigned long flags;
4219 
4220 	spin_lock_irqsave(&list->lock, flags);
4221 	__skb_queue_after(list, old, newsk);
4222 	spin_unlock_irqrestore(&list->lock, flags);
4223 }
4224 EXPORT_SYMBOL(skb_append);
4225 
4226 static inline void skb_split_inside_header(struct sk_buff *skb,
4227 					   struct sk_buff* skb1,
4228 					   const u32 len, const int pos)
4229 {
4230 	int i;
4231 
4232 	skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
4233 					 pos - len);
4234 	/* And move data appendix as is. */
4235 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
4236 		skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
4237 
4238 	skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
4239 	skb1->unreadable	   = skb->unreadable;
4240 	skb_shinfo(skb)->nr_frags  = 0;
4241 	skb1->data_len		   = skb->data_len;
4242 	skb1->len		   += skb1->data_len;
4243 	skb->data_len		   = 0;
4244 	skb->len		   = len;
4245 	skb_set_tail_pointer(skb, len);
4246 }
4247 
4248 static inline void skb_split_no_header(struct sk_buff *skb,
4249 				       struct sk_buff* skb1,
4250 				       const u32 len, int pos)
4251 {
4252 	int i, k = 0;
4253 	const int nfrags = skb_shinfo(skb)->nr_frags;
4254 
4255 	skb_shinfo(skb)->nr_frags = 0;
4256 	skb1->len		  = skb1->data_len = skb->len - len;
4257 	skb->len		  = len;
4258 	skb->data_len		  = len - pos;
4259 
4260 	for (i = 0; i < nfrags; i++) {
4261 		int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
4262 
4263 		if (pos + size > len) {
4264 			skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
4265 
4266 			if (pos < len) {
4267 				/* Split frag.
4268 				 * We have two variants in this case:
4269 				 * 1. Move all the frag to the second
4270 				 *    part, if it is possible. F.e.
4271 				 *    this approach is mandatory for TUX,
4272 				 *    where splitting is expensive.
4273 				 * 2. Split is accurately. We make this.
4274 				 */
4275 				skb_frag_ref(skb1, k);
4276 				skb_frag_off_add(&skb_shinfo(skb1)->frags[0], len - pos);
4277 				skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
4278 				skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
4279 				skb_shinfo(skb)->nr_frags++;
4280 			}
4281 			k++;
4282 		} else
4283 			skb_shinfo(skb)->nr_frags++;
4284 		pos += size;
4285 	}
4286 	skb_shinfo(skb1)->nr_frags = k;
4287 
4288 	skb1->unreadable = skb->unreadable;
4289 }
4290 
4291 /**
4292  * skb_split - Split fragmented skb to two parts at length len.
4293  * @skb: the buffer to split
4294  * @skb1: the buffer to receive the second part
4295  * @len: new length for skb
4296  */
4297 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
4298 {
4299 	int pos = skb_headlen(skb);
4300 	const int zc_flags = SKBFL_SHARED_FRAG | SKBFL_PURE_ZEROCOPY;
4301 
4302 	skb_zcopy_downgrade_managed(skb);
4303 
4304 	skb_shinfo(skb1)->flags |= skb_shinfo(skb)->flags & zc_flags;
4305 	skb_zerocopy_clone(skb1, skb, 0);
4306 	skb1->pp_recycle = skb->pp_recycle;
4307 	if (len < pos)	/* Split line is inside header. */
4308 		skb_split_inside_header(skb, skb1, len, pos);
4309 	else		/* Second chunk has no header, nothing to copy. */
4310 		skb_split_no_header(skb, skb1, len, pos);
4311 }
4312 EXPORT_SYMBOL(skb_split);
4313 
4314 /* Shifting from/to a cloned skb is a no-go.
4315  *
4316  * Caller cannot keep skb_shinfo related pointers past calling here!
4317  */
4318 static int skb_prepare_for_shift(struct sk_buff *skb)
4319 {
4320 	return skb_unclone_keeptruesize(skb, GFP_ATOMIC);
4321 }
4322 
4323 /**
4324  * skb_shift - Shifts paged data partially from skb to another
4325  * @tgt: buffer into which tail data gets added
4326  * @skb: buffer from which the paged data comes from
4327  * @shiftlen: shift up to this many bytes
4328  *
4329  * Attempts to shift up to shiftlen worth of bytes, which may be less than
4330  * the length of the skb, from skb to tgt. Returns number bytes shifted.
4331  * It's up to caller to free skb if everything was shifted.
4332  *
4333  * If @tgt runs out of frags, the whole operation is aborted.
4334  *
4335  * Skb cannot include anything else but paged data while tgt is allowed
4336  * to have non-paged data as well.
4337  *
4338  * TODO: full sized shift could be optimized but that would need
4339  * specialized skb free'er to handle frags without up-to-date nr_frags.
4340  */
4341 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
4342 {
4343 	int from, to, merge, todo;
4344 	skb_frag_t *fragfrom, *fragto;
4345 
4346 	if (unlikely(shiftlen > skb->len)) {
4347 		DEBUG_NET_WARN_ON_ONCE(1);
4348 		return 0;
4349 	}
4350 
4351 	if (skb_headlen(skb))
4352 		return 0;
4353 	if (skb_zcopy(tgt) || skb_zcopy(skb))
4354 		return 0;
4355 
4356 	DEBUG_NET_WARN_ON_ONCE(tgt->pp_recycle != skb->pp_recycle);
4357 	DEBUG_NET_WARN_ON_ONCE(skb_cmp_decrypted(tgt, skb));
4358 
4359 	todo = shiftlen;
4360 	from = 0;
4361 	to = skb_shinfo(tgt)->nr_frags;
4362 	fragfrom = &skb_shinfo(skb)->frags[from];
4363 
4364 	/* Actual merge is delayed until the point when we know we can
4365 	 * commit all, so that we don't have to undo partial changes
4366 	 */
4367 	if (!skb_can_coalesce_netmem(tgt, to, skb_frag_netmem(fragfrom),
4368 				     skb_frag_off(fragfrom))) {
4369 		merge = -1;
4370 	} else {
4371 		merge = to - 1;
4372 
4373 		todo -= skb_frag_size(fragfrom);
4374 		if (todo < 0) {
4375 			if (skb_prepare_for_shift(skb) ||
4376 			    skb_prepare_for_shift(tgt))
4377 				return 0;
4378 
4379 			/* All previous frag pointers might be stale! */
4380 			fragfrom = &skb_shinfo(skb)->frags[from];
4381 			fragto = &skb_shinfo(tgt)->frags[merge];
4382 
4383 			skb_frag_size_add(fragto, shiftlen);
4384 			skb_frag_size_sub(fragfrom, shiftlen);
4385 			skb_frag_off_add(fragfrom, shiftlen);
4386 
4387 			goto onlymerged;
4388 		}
4389 
4390 		from++;
4391 	}
4392 
4393 	/* Skip full, not-fitting skb to avoid expensive operations */
4394 	if ((shiftlen == skb->len) &&
4395 	    (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
4396 		return 0;
4397 
4398 	if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
4399 		return 0;
4400 
4401 	while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
4402 		if (to == MAX_SKB_FRAGS)
4403 			return 0;
4404 
4405 		fragfrom = &skb_shinfo(skb)->frags[from];
4406 		fragto = &skb_shinfo(tgt)->frags[to];
4407 
4408 		if (todo >= skb_frag_size(fragfrom)) {
4409 			*fragto = *fragfrom;
4410 			todo -= skb_frag_size(fragfrom);
4411 			from++;
4412 			to++;
4413 
4414 		} else {
4415 			skb_frag_page_copy(fragto, fragfrom);
4416 			skb_frag_ref(tgt, to);
4417 			skb_frag_off_copy(fragto, fragfrom);
4418 			skb_frag_size_set(fragto, todo);
4419 
4420 			skb_frag_off_add(fragfrom, todo);
4421 			skb_frag_size_sub(fragfrom, todo);
4422 			todo = 0;
4423 
4424 			to++;
4425 			break;
4426 		}
4427 	}
4428 
4429 	/* The frags ran out before shiftlen did.  Nothing has been committed
4430 	 * yet, so back out.
4431 	 */
4432 	if (unlikely(todo > 0)) {
4433 		DEBUG_NET_WARN_ON_ONCE(1);
4434 		return 0;
4435 	}
4436 
4437 	/* Ready to "commit" this state change to tgt */
4438 	skb_shinfo(tgt)->nr_frags = to;
4439 
4440 	if (merge >= 0) {
4441 		fragfrom = &skb_shinfo(skb)->frags[0];
4442 		fragto = &skb_shinfo(tgt)->frags[merge];
4443 
4444 		skb_frag_size_add(fragto, skb_frag_size(fragfrom));
4445 		skb_frag_unref(skb, 0);
4446 	}
4447 
4448 	/* Reposition in the original skb */
4449 	to = 0;
4450 	while (from < skb_shinfo(skb)->nr_frags)
4451 		skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
4452 	skb_shinfo(skb)->nr_frags = to;
4453 
4454 onlymerged:
4455 	/* Most likely the tgt won't ever need its checksum anymore, skb on
4456 	 * the other hand might need it if it needs to be resent
4457 	 */
4458 	tgt->ip_summed = CHECKSUM_PARTIAL;
4459 	skb->ip_summed = CHECKSUM_PARTIAL;
4460 
4461 	skb_shinfo(tgt)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG;
4462 
4463 	skb_len_add(skb, -shiftlen);
4464 	skb_len_add(tgt, shiftlen);
4465 
4466 	return shiftlen;
4467 }
4468 EXPORT_SYMBOL_IF_KUNIT(skb_shift);
4469 
4470 /**
4471  * skb_prepare_seq_read - Prepare a sequential read of skb data
4472  * @skb: the buffer to read
4473  * @from: lower offset of data to be read
4474  * @to: upper offset of data to be read
4475  * @st: state variable
4476  *
4477  * Initializes the specified state variable. Must be called before
4478  * invoking skb_seq_read() for the first time.
4479  */
4480 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
4481 			  unsigned int to, struct skb_seq_state *st)
4482 {
4483 	st->lower_offset = from;
4484 	st->upper_offset = to;
4485 	st->root_skb = st->cur_skb = skb;
4486 	st->frag_idx = st->stepped_offset = 0;
4487 	st->frag_data = NULL;
4488 	st->frag_off = 0;
4489 }
4490 EXPORT_SYMBOL(skb_prepare_seq_read);
4491 
4492 /**
4493  * skb_seq_read - Sequentially read skb data
4494  * @consumed: number of bytes consumed by the caller so far
4495  * @data: destination pointer for data to be returned
4496  * @st: state variable
4497  *
4498  * Reads a block of skb data at @consumed relative to the
4499  * lower offset specified to skb_prepare_seq_read(). Assigns
4500  * the head of the data block to @data and returns the length
4501  * of the block or 0 if the end of the skb data or the upper
4502  * offset has been reached.
4503  *
4504  * The caller is not required to consume all of the data
4505  * returned, i.e. @consumed is typically set to the number
4506  * of bytes already consumed and the next call to
4507  * skb_seq_read() will return the remaining part of the block.
4508  *
4509  * Note 1: The size of each block of data returned can be arbitrary,
4510  *       this limitation is the cost for zerocopy sequential
4511  *       reads of potentially non linear data.
4512  *
4513  * Note 2: Fragment lists within fragments are not implemented
4514  *       at the moment, state->root_skb could be replaced with
4515  *       a stack for this purpose.
4516  */
4517 unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
4518 			  struct skb_seq_state *st)
4519 {
4520 	unsigned int block_limit, abs_offset = consumed + st->lower_offset;
4521 	skb_frag_t *frag;
4522 
4523 	if (unlikely(abs_offset >= st->upper_offset)) {
4524 		if (st->frag_data) {
4525 			kunmap_atomic(st->frag_data);
4526 			st->frag_data = NULL;
4527 		}
4528 		return 0;
4529 	}
4530 
4531 next_skb:
4532 	block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
4533 
4534 	if (abs_offset < block_limit && !st->frag_data) {
4535 		*data = st->cur_skb->data + (abs_offset - st->stepped_offset);
4536 		return block_limit - abs_offset;
4537 	}
4538 
4539 	if (!skb_frags_readable(st->cur_skb))
4540 		return 0;
4541 
4542 	if (st->frag_idx == 0 && !st->frag_data)
4543 		st->stepped_offset += skb_headlen(st->cur_skb);
4544 
4545 	while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
4546 		unsigned int pg_idx, pg_off, pg_sz;
4547 
4548 		frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
4549 
4550 		pg_idx = 0;
4551 		pg_off = skb_frag_off(frag);
4552 		pg_sz = skb_frag_size(frag);
4553 
4554 		if (skb_frag_must_loop(skb_frag_page(frag))) {
4555 			pg_idx = (pg_off + st->frag_off) >> PAGE_SHIFT;
4556 			pg_off = offset_in_page(pg_off + st->frag_off);
4557 			pg_sz = min_t(unsigned int, pg_sz - st->frag_off,
4558 						    PAGE_SIZE - pg_off);
4559 		}
4560 
4561 		block_limit = pg_sz + st->stepped_offset;
4562 		if (abs_offset < block_limit) {
4563 			if (!st->frag_data)
4564 				st->frag_data = kmap_atomic(skb_frag_page(frag) + pg_idx);
4565 
4566 			*data = (u8 *)st->frag_data + pg_off +
4567 				(abs_offset - st->stepped_offset);
4568 
4569 			return block_limit - abs_offset;
4570 		}
4571 
4572 		if (st->frag_data) {
4573 			kunmap_atomic(st->frag_data);
4574 			st->frag_data = NULL;
4575 		}
4576 
4577 		st->stepped_offset += pg_sz;
4578 		st->frag_off += pg_sz;
4579 		if (st->frag_off == skb_frag_size(frag)) {
4580 			st->frag_off = 0;
4581 			st->frag_idx++;
4582 		}
4583 	}
4584 
4585 	if (st->frag_data) {
4586 		kunmap_atomic(st->frag_data);
4587 		st->frag_data = NULL;
4588 	}
4589 
4590 	if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
4591 		st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
4592 		st->frag_idx = 0;
4593 		goto next_skb;
4594 	} else if (st->cur_skb->next) {
4595 		st->cur_skb = st->cur_skb->next;
4596 		st->frag_idx = 0;
4597 		goto next_skb;
4598 	}
4599 
4600 	return 0;
4601 }
4602 EXPORT_SYMBOL(skb_seq_read);
4603 
4604 /**
4605  * skb_abort_seq_read - Abort a sequential read of skb data
4606  * @st: state variable
4607  *
4608  * Must be called if skb_seq_read() was not called until it
4609  * returned 0.
4610  */
4611 void skb_abort_seq_read(struct skb_seq_state *st)
4612 {
4613 	if (st->frag_data)
4614 		kunmap_atomic(st->frag_data);
4615 }
4616 EXPORT_SYMBOL(skb_abort_seq_read);
4617 
4618 /**
4619  * skb_copy_seq_read() - copy from a skb_seq_state to a buffer
4620  * @st: source skb_seq_state
4621  * @offset: offset in source
4622  * @to: destination buffer
4623  * @len: number of bytes to copy
4624  *
4625  * Copy @len bytes from @offset bytes into the source @st to the destination
4626  * buffer @to. `offset` should increase (or be unchanged) with each subsequent
4627  * call to this function. If offset needs to decrease from the previous use `st`
4628  * should be reset first.
4629  *
4630  * Return: 0 on success or -EINVAL if the copy ended early
4631  */
4632 int skb_copy_seq_read(struct skb_seq_state *st, int offset, void *to, int len)
4633 {
4634 	const u8 *data;
4635 	u32 sqlen;
4636 
4637 	for (;;) {
4638 		sqlen = skb_seq_read(offset, &data, st);
4639 		if (sqlen == 0)
4640 			return -EINVAL;
4641 		if (sqlen >= len) {
4642 			memcpy(to, data, len);
4643 			return 0;
4644 		}
4645 		memcpy(to, data, sqlen);
4646 		to += sqlen;
4647 		offset += sqlen;
4648 		len -= sqlen;
4649 	}
4650 }
4651 EXPORT_SYMBOL(skb_copy_seq_read);
4652 
4653 #define TS_SKB_CB(state)	((struct skb_seq_state *) &((state)->cb))
4654 
4655 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
4656 					  struct ts_config *conf,
4657 					  struct ts_state *state)
4658 {
4659 	return skb_seq_read(offset, text, TS_SKB_CB(state));
4660 }
4661 
4662 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
4663 {
4664 	skb_abort_seq_read(TS_SKB_CB(state));
4665 }
4666 
4667 /**
4668  * skb_find_text - Find a text pattern in skb data
4669  * @skb: the buffer to look in
4670  * @from: search offset
4671  * @to: search limit
4672  * @config: textsearch configuration
4673  *
4674  * Finds a pattern in the skb data according to the specified
4675  * textsearch configuration. Use textsearch_next() to retrieve
4676  * subsequent occurrences of the pattern. Returns the offset
4677  * to the first occurrence or UINT_MAX if no match was found.
4678  */
4679 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
4680 			   unsigned int to, struct ts_config *config)
4681 {
4682 	unsigned int patlen = config->ops->get_pattern_len(config);
4683 	struct ts_state state;
4684 	unsigned int ret;
4685 
4686 	BUILD_BUG_ON(sizeof(struct skb_seq_state) > sizeof(state.cb));
4687 
4688 	config->get_next_block = skb_ts_get_next_block;
4689 	config->finish = skb_ts_finish;
4690 
4691 	skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state));
4692 
4693 	ret = textsearch_find(config, &state);
4694 	return (ret + patlen <= to - from ? ret : UINT_MAX);
4695 }
4696 EXPORT_SYMBOL(skb_find_text);
4697 
4698 int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
4699 			 int offset, size_t size, size_t max_frags)
4700 {
4701 	int i = skb_shinfo(skb)->nr_frags;
4702 
4703 	if (skb_can_coalesce(skb, i, page, offset)) {
4704 		skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size);
4705 	} else if (i < max_frags) {
4706 		skb_zcopy_downgrade_managed(skb);
4707 		get_page(page);
4708 		skb_fill_page_desc_noacc(skb, i, page, offset, size);
4709 	} else {
4710 		return -EMSGSIZE;
4711 	}
4712 
4713 	return 0;
4714 }
4715 EXPORT_SYMBOL_GPL(skb_append_pagefrags);
4716 
4717 /**
4718  *	skb_pull_rcsum - pull skb and update receive checksum
4719  *	@skb: buffer to update
4720  *	@len: length of data pulled
4721  *
4722  *	This function performs an skb_pull on the packet and updates
4723  *	the CHECKSUM_COMPLETE checksum.  It should be used on
4724  *	receive path processing instead of skb_pull unless you know
4725  *	that the checksum difference is zero (e.g., a valid IP header)
4726  *	or you are setting ip_summed to CHECKSUM_NONE.
4727  */
4728 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
4729 {
4730 	unsigned char *data = skb->data;
4731 
4732 	BUG_ON(len > skb->len);
4733 	__skb_pull(skb, len);
4734 	skb_postpull_rcsum(skb, data, len);
4735 	return skb->data;
4736 }
4737 EXPORT_SYMBOL_GPL(skb_pull_rcsum);
4738 
4739 static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb)
4740 {
4741 	skb_frag_t head_frag;
4742 	struct page *page;
4743 
4744 	page = virt_to_head_page(frag_skb->head);
4745 	skb_frag_fill_page_desc(&head_frag, page, frag_skb->data -
4746 				(unsigned char *)page_address(page),
4747 				skb_headlen(frag_skb));
4748 	return head_frag;
4749 }
4750 
4751 struct sk_buff *skb_segment_list(struct sk_buff *skb,
4752 				 netdev_features_t features,
4753 				 unsigned int offset)
4754 {
4755 	struct sk_buff *list_skb = skb_shinfo(skb)->frag_list;
4756 	unsigned int tnl_hlen = skb_tnl_header_len(skb);
4757 	unsigned int delta_len = 0;
4758 	struct sk_buff *tail = NULL;
4759 	struct sk_buff *nskb, *tmp;
4760 	int len_diff, err;
4761 
4762 	/* Only skb_gro_receive_list generated skbs arrive here */
4763 	DEBUG_NET_WARN_ON_ONCE(!(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST));
4764 
4765 	skb_push(skb, -skb_network_offset(skb) + offset);
4766 
4767 	/* Ensure the head is writeable before touching the shared info */
4768 	err = skb_unclone(skb, GFP_ATOMIC);
4769 	if (err)
4770 		goto err_linearize;
4771 
4772 	skb_shinfo(skb)->frag_list = NULL;
4773 
4774 	while (list_skb) {
4775 		nskb = list_skb;
4776 		list_skb = list_skb->next;
4777 
4778 		DEBUG_NET_WARN_ON_ONCE(nskb->sk);
4779 
4780 		err = 0;
4781 		if (skb_shared(nskb)) {
4782 			tmp = skb_clone(nskb, GFP_ATOMIC);
4783 			if (tmp) {
4784 				consume_skb(nskb);
4785 				nskb = tmp;
4786 				err = skb_unclone(nskb, GFP_ATOMIC);
4787 			} else {
4788 				err = -ENOMEM;
4789 			}
4790 		}
4791 
4792 		if (!err)
4793 			err = skb_cow_head(nskb, offset + tnl_hlen -
4794 					   skb_network_offset(nskb));
4795 
4796 		if (!tail)
4797 			skb->next = nskb;
4798 		else
4799 			tail->next = nskb;
4800 
4801 		/* nskb may be a clone with a NULL ->next: keep the remaining
4802 		 * members linked so that err_linearize frees them.
4803 		 */
4804 		nskb->next = list_skb;
4805 
4806 		if (unlikely(err))
4807 			goto err_linearize;
4808 
4809 		tail = nskb;
4810 
4811 		delta_len += nskb->len;
4812 
4813 		skb_push(nskb, -skb_network_offset(nskb) + offset);
4814 
4815 		skb_release_head_state(nskb);
4816 		len_diff = skb_network_header_len(nskb) - skb_network_header_len(skb);
4817 		__copy_skb_header(nskb, skb);
4818 
4819 		skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb));
4820 		nskb->transport_header += len_diff;
4821 		skb_copy_from_linear_data_offset(skb, -tnl_hlen,
4822 						 nskb->data - tnl_hlen,
4823 						 offset + tnl_hlen);
4824 
4825 		if (skb_needs_linearize(nskb, features) &&
4826 		    __skb_linearize(nskb))
4827 			goto err_linearize;
4828 	}
4829 
4830 	skb->data_len = skb->data_len - delta_len;
4831 	skb->len = skb->len - delta_len;
4832 
4833 	skb_gso_reset(skb);
4834 
4835 	skb->prev = tail;
4836 
4837 	if (skb_needs_linearize(skb, features) &&
4838 	    __skb_linearize(skb))
4839 		goto err_linearize;
4840 
4841 	skb_get(skb);
4842 
4843 	return skb;
4844 
4845 err_linearize:
4846 	kfree_skb_list(skb->next);
4847 	skb->next = NULL;
4848 	return ERR_PTR(-ENOMEM);
4849 }
4850 EXPORT_SYMBOL_GPL(skb_segment_list);
4851 
4852 /**
4853  *	skb_segment - Perform protocol segmentation on skb.
4854  *	@head_skb: buffer to segment
4855  *	@features: features for the output path (see dev->features)
4856  *
4857  *	This function performs segmentation on the given skb.  It returns
4858  *	a pointer to the first in a list of new skbs for the segments.
4859  *	In case of error it returns ERR_PTR(err).
4860  */
4861 struct sk_buff *skb_segment(struct sk_buff *head_skb,
4862 			    netdev_features_t features)
4863 {
4864 	struct sk_buff *segs = NULL;
4865 	struct sk_buff *tail = NULL;
4866 	struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list;
4867 	unsigned int max_segs = SKB_GSO_CB(head_skb)->max_segs;
4868 	unsigned int mss = skb_shinfo(head_skb)->gso_size;
4869 	bool gso_by_frags = mss == GSO_BY_FRAGS;
4870 	unsigned int doffset = head_skb->data - skb_mac_header(head_skb);
4871 	unsigned int offset = doffset;
4872 	unsigned int tnl_hlen = skb_tnl_header_len(head_skb);
4873 	unsigned int partial_segs = 0;
4874 	unsigned int headroom;
4875 	unsigned int len = head_skb->len;
4876 	struct sk_buff *frag_skb;
4877 	skb_frag_t *frag;
4878 	__be16 proto;
4879 	bool csum, sg;
4880 	int err = -ENOMEM;
4881 	int i = 0;
4882 	int nfrags, pos;
4883 
4884 	if ((skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY) &&
4885 	    !gso_by_frags && mss != skb_headlen(head_skb)) {
4886 		struct sk_buff *check_skb;
4887 
4888 		for (check_skb = list_skb; check_skb; check_skb = check_skb->next) {
4889 			if (skb_headlen(check_skb) && !check_skb->head_frag) {
4890 				/* gso_size is untrusted, and we have a frag_list with
4891 				 * a linear non head_frag item.
4892 				 *
4893 				 * If head_skb's headlen does not fit requested gso_size,
4894 				 * it means that the frag_list members do NOT terminate
4895 				 * on exact gso_size boundaries. Hence we cannot perform
4896 				 * skb_frag_t page sharing. Therefore we must fallback to
4897 				 * copying the frag_list skbs; we do so by disabling SG.
4898 				 */
4899 				features &= ~NETIF_F_SG;
4900 				break;
4901 			}
4902 		}
4903 	}
4904 
4905 	__skb_push(head_skb, doffset);
4906 	proto = skb_network_protocol(head_skb, NULL);
4907 	if (unlikely(!proto))
4908 		return ERR_PTR(-EINVAL);
4909 
4910 	sg = !!(features & NETIF_F_SG);
4911 	csum = !!can_checksum_protocol(features, proto);
4912 
4913 	if (sg && csum && !gso_by_frags)  {
4914 		if (!max_segs && !(features & NETIF_F_GSO_PARTIAL)) {
4915 			struct sk_buff *iter;
4916 			unsigned int frag_len;
4917 
4918 			if (!list_skb ||
4919 			    !net_gso_ok(features, skb_shinfo(head_skb)->gso_type))
4920 				goto normal;
4921 
4922 			/* If we get here then all the required
4923 			 * GSO features except frag_list are supported.
4924 			 * Try to split the SKB to multiple GSO SKBs
4925 			 * with no frag_list.
4926 			 * Currently we can do that only when the buffers don't
4927 			 * have a linear part and all the buffers except
4928 			 * the last are of the same length.
4929 			 */
4930 			frag_len = list_skb->len;
4931 			skb_walk_frags(head_skb, iter) {
4932 				if (frag_len != iter->len && iter->next)
4933 					goto normal;
4934 				if (skb_headlen(iter) && !iter->head_frag)
4935 					goto normal;
4936 
4937 				len -= iter->len;
4938 			}
4939 
4940 			if (len != frag_len)
4941 				goto normal;
4942 		}
4943 
4944 		/* GSO partial only requires that we trim off any excess that
4945 		 * doesn't fit into an MSS sized block, so take care of that
4946 		 * now.
4947 		 */
4948 		DEBUG_NET_WARN_ON_ONCE(!max_segs && len / mss > GSO_MAX_SEGS);
4949 		if (max_segs)
4950 			partial_segs = min(len / mss, max_segs);
4951 		else
4952 			partial_segs = min(len / mss, GSO_MAX_SEGS);
4953 		if (partial_segs > 1)
4954 			mss *= partial_segs;
4955 		else
4956 			partial_segs = 0;
4957 	}
4958 
4959 normal:
4960 	headroom = skb_headroom(head_skb);
4961 	pos = skb_headlen(head_skb);
4962 
4963 	if (skb_orphan_frags(head_skb, GFP_ATOMIC))
4964 		return ERR_PTR(-ENOMEM);
4965 
4966 	nfrags = skb_shinfo(head_skb)->nr_frags;
4967 	frag = skb_shinfo(head_skb)->frags;
4968 	frag_skb = head_skb;
4969 
4970 	do {
4971 		struct sk_buff *nskb;
4972 		skb_frag_t *nskb_frag;
4973 		int hsize;
4974 		int size;
4975 
4976 		if (unlikely(gso_by_frags)) {
4977 			len = list_skb->len;
4978 		} else {
4979 			len = head_skb->len - offset;
4980 			if (len > mss)
4981 				len = mss;
4982 		}
4983 
4984 		hsize = skb_headlen(head_skb) - offset;
4985 
4986 		if (hsize <= 0 && i >= nfrags && skb_headlen(list_skb) &&
4987 		    (skb_headlen(list_skb) == len || sg)) {
4988 			if (unlikely(skb_headlen(list_skb) > len)) {
4989 				DEBUG_NET_WARN_ON_ONCE(1);
4990 				err = -EINVAL;
4991 				goto err;
4992 			}
4993 
4994 			nskb = skb_clone(list_skb, GFP_ATOMIC);
4995 			if (unlikely(!nskb))
4996 				goto err;
4997 
4998 			i = 0;
4999 			nfrags = skb_shinfo(list_skb)->nr_frags;
5000 			frag = skb_shinfo(list_skb)->frags;
5001 			frag_skb = list_skb;
5002 			pos += skb_headlen(list_skb);
5003 
5004 			while (pos < offset + len) {
5005 				if (unlikely(i >= nfrags)) {
5006 					DEBUG_NET_WARN_ON_ONCE(1);
5007 					kfree_skb(nskb);
5008 					err = -EINVAL;
5009 					goto err;
5010 				}
5011 
5012 				size = skb_frag_size(frag);
5013 				if (pos + size > offset + len)
5014 					break;
5015 
5016 				i++;
5017 				pos += size;
5018 				frag++;
5019 			}
5020 
5021 			list_skb = list_skb->next;
5022 
5023 			if (unlikely(pskb_trim(nskb, len))) {
5024 				kfree_skb(nskb);
5025 				goto err;
5026 			}
5027 
5028 			hsize = skb_end_offset(nskb);
5029 			if (skb_cow_head(nskb, doffset + headroom)) {
5030 				kfree_skb(nskb);
5031 				goto err;
5032 			}
5033 
5034 			nskb->truesize += skb_end_offset(nskb) - hsize;
5035 			skb_release_head_state(nskb);
5036 			__skb_push(nskb, doffset);
5037 		} else {
5038 			if (hsize < 0)
5039 				hsize = 0;
5040 			if (hsize > len || !sg)
5041 				hsize = len;
5042 
5043 			nskb = __alloc_skb(hsize + doffset + headroom,
5044 					   GFP_ATOMIC, skb_alloc_rx_flag(head_skb),
5045 					   NUMA_NO_NODE);
5046 
5047 			if (unlikely(!nskb))
5048 				goto err;
5049 
5050 			skb_reserve(nskb, headroom);
5051 			__skb_put(nskb, doffset);
5052 		}
5053 
5054 		if (segs)
5055 			tail->next = nskb;
5056 		else
5057 			segs = nskb;
5058 		tail = nskb;
5059 
5060 		__copy_skb_header(nskb, head_skb);
5061 
5062 		skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom);
5063 		skb_reset_mac_len(nskb);
5064 
5065 		skb_copy_from_linear_data_offset(head_skb, -tnl_hlen,
5066 						 nskb->data - tnl_hlen,
5067 						 doffset + tnl_hlen);
5068 
5069 		if (nskb->len == len + doffset)
5070 			goto perform_csum_check;
5071 
5072 		if (!sg) {
5073 			if (!csum) {
5074 				if (!nskb->remcsum_offload)
5075 					nskb->ip_summed = CHECKSUM_NONE;
5076 				SKB_GSO_CB(nskb)->csum =
5077 					skb_copy_and_csum_bits(head_skb, offset,
5078 							       skb_put(nskb,
5079 								       len),
5080 							       len);
5081 				SKB_GSO_CB(nskb)->csum_start =
5082 					skb_headroom(nskb) + doffset;
5083 			} else {
5084 				if (skb_copy_bits(head_skb, offset, skb_put(nskb, len), len))
5085 					goto err;
5086 			}
5087 			continue;
5088 		}
5089 
5090 		nskb_frag = skb_shinfo(nskb)->frags;
5091 
5092 		skb_copy_from_linear_data_offset(head_skb, offset,
5093 						 skb_put(nskb, hsize), hsize);
5094 
5095 		skb_shinfo(nskb)->flags |= (skb_shinfo(head_skb)->flags |
5096 					    skb_shinfo(frag_skb)->flags) &
5097 					   SKBFL_SHARED_FRAG;
5098 
5099 		if (skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC))
5100 			goto err;
5101 
5102 		while (pos < offset + len) {
5103 			if (i >= nfrags) {
5104 				if (skb_shinfo(nskb)->nr_frags &&
5105 				    nskb->pp_recycle != list_skb->pp_recycle) {
5106 					nskb->data_len = pos - offset - hsize;
5107 					nskb->len += nskb->data_len;
5108 					nskb->truesize += nskb->data_len;
5109 					if (__skb_linearize(nskb))
5110 						goto err;
5111 					hsize = pos - offset;
5112 					nskb_frag = skb_shinfo(nskb)->frags;
5113 				}
5114 				if (skb_orphan_frags(list_skb, GFP_ATOMIC) ||
5115 				    skb_zerocopy_clone(nskb, list_skb,
5116 						       GFP_ATOMIC))
5117 					goto err;
5118 
5119 				i = 0;
5120 				nfrags = skb_shinfo(list_skb)->nr_frags;
5121 				frag = skb_shinfo(list_skb)->frags;
5122 				frag_skb = list_skb;
5123 
5124 				skb_shinfo(nskb)->flags |= skb_shinfo(frag_skb)->flags & SKBFL_SHARED_FRAG;
5125 
5126 				if (!skb_headlen(list_skb)) {
5127 					if (unlikely(!nfrags)) {
5128 						DEBUG_NET_WARN_ON_ONCE(1);
5129 						err = -EINVAL;
5130 						goto err;
5131 					}
5132 				} else {
5133 					if (unlikely(!list_skb->head_frag)) {
5134 						DEBUG_NET_WARN_ON_ONCE(1);
5135 						err = -EINVAL;
5136 						goto err;
5137 					}
5138 
5139 					/* to make room for head_frag. */
5140 					i--;
5141 					frag--;
5142 				}
5143 
5144 				list_skb = list_skb->next;
5145 			}
5146 
5147 			if (unlikely(skb_shinfo(nskb)->nr_frags >=
5148 				     MAX_SKB_FRAGS)) {
5149 				net_warn_ratelimited(
5150 					"skb_segment: too many frags: %u %u\n",
5151 					pos, mss);
5152 				err = -EINVAL;
5153 				goto err;
5154 			}
5155 
5156 			if (!skb_shinfo(nskb)->nr_frags)
5157 				nskb->pp_recycle = frag_skb->pp_recycle;
5158 			*nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag;
5159 			skb_frag_ref(nskb, skb_shinfo(nskb)->nr_frags);
5160 			size = skb_frag_size(nskb_frag);
5161 
5162 			if (pos < offset) {
5163 				skb_frag_off_add(nskb_frag, offset - pos);
5164 				skb_frag_size_sub(nskb_frag, offset - pos);
5165 			}
5166 
5167 			skb_shinfo(nskb)->nr_frags++;
5168 
5169 			if (pos + size <= offset + len) {
5170 				i++;
5171 				frag++;
5172 				pos += size;
5173 			} else {
5174 				skb_frag_size_sub(nskb_frag, pos + size - (offset + len));
5175 				goto skip_fraglist;
5176 			}
5177 
5178 			nskb_frag++;
5179 		}
5180 
5181 skip_fraglist:
5182 		nskb->data_len = len - hsize;
5183 		nskb->len += nskb->data_len;
5184 		nskb->truesize += nskb->data_len;
5185 
5186 perform_csum_check:
5187 		if (!csum) {
5188 			if (skb_has_shared_frag(nskb) &&
5189 			    __skb_linearize(nskb))
5190 				goto err;
5191 
5192 			if (!nskb->remcsum_offload)
5193 				nskb->ip_summed = CHECKSUM_NONE;
5194 			SKB_GSO_CB(nskb)->csum =
5195 				skb_checksum(nskb, doffset,
5196 					     nskb->len - doffset, 0);
5197 			SKB_GSO_CB(nskb)->csum_start =
5198 				skb_headroom(nskb) + doffset;
5199 		}
5200 	} while ((offset += len) < head_skb->len);
5201 
5202 	/* Some callers want to get the end of the list.
5203 	 * Put it in segs->prev to avoid walking the list.
5204 	 * (see validate_xmit_skb_list() for example)
5205 	 */
5206 	segs->prev = tail;
5207 
5208 	if (partial_segs) {
5209 		struct sk_buff *iter;
5210 		int type = skb_shinfo(head_skb)->gso_type;
5211 		unsigned short gso_size = skb_shinfo(head_skb)->gso_size;
5212 
5213 		/* Update type to add partial and then remove dodgy if set */
5214 		type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL;
5215 		type &= ~SKB_GSO_DODGY;
5216 
5217 		/* Update GSO info and prepare to start updating headers on
5218 		 * our way back down the stack of protocols.
5219 		 */
5220 		for (iter = segs; iter; iter = iter->next) {
5221 			skb_shinfo(iter)->gso_size = gso_size;
5222 			skb_shinfo(iter)->gso_segs = partial_segs;
5223 			skb_shinfo(iter)->gso_type = type;
5224 			SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset;
5225 		}
5226 
5227 		if (tail->len - doffset <= gso_size)
5228 			skb_gso_reset(tail);
5229 		else if (tail != segs)
5230 			skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size);
5231 	}
5232 
5233 	/* Following permits correct backpressure, for protocols
5234 	 * using skb_set_owner_w().
5235 	 * Idea is to tranfert ownership from head_skb to last segment.
5236 	 */
5237 	if (head_skb->destructor == sock_wfree) {
5238 		swap(tail->truesize, head_skb->truesize);
5239 		swap(tail->destructor, head_skb->destructor);
5240 		swap(tail->sk, head_skb->sk);
5241 	}
5242 	return segs;
5243 
5244 err:
5245 	kfree_skb_list(segs);
5246 	return ERR_PTR(err);
5247 }
5248 EXPORT_SYMBOL_GPL(skb_segment);
5249 
5250 #ifdef CONFIG_SKB_EXTENSIONS
5251 #define SKB_EXT_ALIGN_VALUE	8
5252 #define SKB_EXT_CHUNKSIZEOF(x)	(ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE)
5253 
5254 static const u8 skb_ext_type_len[] = {
5255 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
5256 	[SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info),
5257 #endif
5258 #ifdef CONFIG_XFRM
5259 	[SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path),
5260 #endif
5261 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
5262 	[TC_SKB_EXT] = SKB_EXT_CHUNKSIZEOF(struct tc_skb_ext),
5263 #endif
5264 #if IS_ENABLED(CONFIG_MPTCP)
5265 	[SKB_EXT_MPTCP] = SKB_EXT_CHUNKSIZEOF(struct mptcp_ext),
5266 #endif
5267 #if IS_ENABLED(CONFIG_MCTP_FLOWS)
5268 	[SKB_EXT_MCTP] = SKB_EXT_CHUNKSIZEOF(struct mctp_flow),
5269 #endif
5270 #if IS_ENABLED(CONFIG_INET_PSP)
5271 	[SKB_EXT_PSP] = SKB_EXT_CHUNKSIZEOF(struct psp_skb_ext),
5272 #endif
5273 #if IS_ENABLED(CONFIG_CAN)
5274 	[SKB_EXT_CAN] = SKB_EXT_CHUNKSIZEOF(struct can_skb_ext),
5275 #endif
5276 };
5277 
5278 static __always_inline __no_profile unsigned int skb_ext_total_length(void)
5279 {
5280 	unsigned int l = SKB_EXT_CHUNKSIZEOF(struct skb_ext);
5281 	int i;
5282 
5283 	for (i = 0; i < ARRAY_SIZE(skb_ext_type_len); i++)
5284 		l += skb_ext_type_len[i];
5285 
5286 	return l;
5287 }
5288 
5289 static noinline void __init __no_profile skb_extensions_init(void)
5290 {
5291 	BUILD_BUG_ON(SKB_EXT_NUM > 8);
5292 	BUILD_BUG_ON(skb_ext_total_length() > 255);
5293 
5294 	skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache",
5295 					     SKB_EXT_ALIGN_VALUE * skb_ext_total_length(),
5296 					     0,
5297 					     SLAB_HWCACHE_ALIGN|SLAB_PANIC,
5298 					     NULL);
5299 }
5300 #else
5301 static void skb_extensions_init(void) {}
5302 #endif
5303 
5304 /* The SKB kmem_cache slab is critical for network performance.  Never
5305  * merge/alias the slab with similar sized objects.  This avoids fragmentation
5306  * that hurts performance of kmem_cache_{alloc,free}_bulk APIs.
5307  */
5308 #ifndef CONFIG_SLUB_TINY
5309 #define FLAG_SKB_NO_MERGE	SLAB_NO_MERGE
5310 #else /* CONFIG_SLUB_TINY - simple loop in kmem_cache_alloc_bulk */
5311 #define FLAG_SKB_NO_MERGE	0
5312 #endif
5313 
5314 void __init skb_init(void)
5315 {
5316 	net_hotdata.skbuff_cache = kmem_cache_create_usercopy("skbuff_head_cache",
5317 					      sizeof(struct sk_buff),
5318 					      0,
5319 					      SLAB_HWCACHE_ALIGN|SLAB_PANIC|
5320 						FLAG_SKB_NO_MERGE,
5321 					      offsetof(struct sk_buff, cb),
5322 					      sizeof_field(struct sk_buff, cb),
5323 					      NULL);
5324 	skbuff_cache_size = kmem_cache_size(net_hotdata.skbuff_cache);
5325 
5326 	net_hotdata.skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
5327 						sizeof(struct sk_buff_fclones),
5328 						0,
5329 						SLAB_HWCACHE_ALIGN|SLAB_PANIC,
5330 						NULL);
5331 	/* usercopy should only access first SKB_SMALL_HEAD_HEADROOM bytes.
5332 	 * struct skb_shared_info is located at the end of skb->head,
5333 	 * and should not be copied to/from user.
5334 	 */
5335 	net_hotdata.skb_small_head_cache = kmem_cache_create_usercopy("skbuff_small_head",
5336 						SKB_SMALL_HEAD_CACHE_SIZE,
5337 						0,
5338 						SLAB_HWCACHE_ALIGN | SLAB_PANIC,
5339 						0,
5340 						SKB_SMALL_HEAD_HEADROOM,
5341 						NULL);
5342 	skb_extensions_init();
5343 }
5344 
5345 static int
5346 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len,
5347 	       unsigned int recursion_level)
5348 {
5349 	int start = skb_headlen(skb);
5350 	int i, copy = start - offset;
5351 	struct sk_buff *frag_iter;
5352 	int elt = 0;
5353 
5354 	if (unlikely(recursion_level >= 24))
5355 		return -EMSGSIZE;
5356 
5357 	if (copy > 0) {
5358 		if (copy > len)
5359 			copy = len;
5360 		sg_set_buf(sg, skb->data + offset, copy);
5361 		elt++;
5362 		if ((len -= copy) == 0)
5363 			return elt;
5364 		offset += copy;
5365 	}
5366 
5367 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
5368 		int end;
5369 
5370 		WARN_ON(start > offset + len);
5371 
5372 		end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
5373 		if ((copy = end - offset) > 0) {
5374 			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
5375 			if (unlikely(elt && sg_is_last(&sg[elt - 1])))
5376 				return -EMSGSIZE;
5377 
5378 			if (copy > len)
5379 				copy = len;
5380 			sg_set_page(&sg[elt], skb_frag_page(frag), copy,
5381 				    skb_frag_off(frag) + offset - start);
5382 			elt++;
5383 			if (!(len -= copy))
5384 				return elt;
5385 			offset += copy;
5386 		}
5387 		start = end;
5388 	}
5389 
5390 	skb_walk_frags(skb, frag_iter) {
5391 		int end, ret;
5392 
5393 		WARN_ON(start > offset + len);
5394 
5395 		end = start + frag_iter->len;
5396 		if ((copy = end - offset) > 0) {
5397 			if (unlikely(elt && sg_is_last(&sg[elt - 1])))
5398 				return -EMSGSIZE;
5399 
5400 			if (copy > len)
5401 				copy = len;
5402 			ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start,
5403 					      copy, recursion_level + 1);
5404 			if (unlikely(ret < 0))
5405 				return ret;
5406 			elt += ret;
5407 			if ((len -= copy) == 0)
5408 				return elt;
5409 			offset += copy;
5410 		}
5411 		start = end;
5412 	}
5413 	if (unlikely(len)) {
5414 		DEBUG_NET_WARN_ON_ONCE(1);
5415 		return -EINVAL;
5416 	}
5417 	return elt;
5418 }
5419 
5420 /**
5421  *	skb_to_sgvec - Fill a scatter-gather list from a socket buffer
5422  *	@skb: Socket buffer containing the buffers to be mapped
5423  *	@sg: The scatter-gather list to map into
5424  *	@offset: The offset into the buffer's contents to start mapping
5425  *	@len: Length of buffer space to be mapped
5426  *
5427  *	Fill the specified scatter-gather list with mappings/pointers into a
5428  *	region of the buffer space attached to a socket buffer. Returns either
5429  *	the number of scatterlist items used, or -EMSGSIZE if the contents
5430  *	could not fit.
5431  */
5432 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
5433 {
5434 	int nsg = __skb_to_sgvec(skb, sg, offset, len, 0);
5435 
5436 	if (nsg <= 0)
5437 		return nsg;
5438 
5439 	sg_mark_end(&sg[nsg - 1]);
5440 
5441 	return nsg;
5442 }
5443 EXPORT_SYMBOL_GPL(skb_to_sgvec);
5444 
5445 /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given
5446  * sglist without mark the sg which contain last skb data as the end.
5447  * So the caller can mannipulate sg list as will when padding new data after
5448  * the first call without calling sg_unmark_end to expend sg list.
5449  *
5450  * Scenario to use skb_to_sgvec_nomark:
5451  * 1. sg_init_table
5452  * 2. skb_to_sgvec_nomark(payload1)
5453  * 3. skb_to_sgvec_nomark(payload2)
5454  *
5455  * This is equivalent to:
5456  * 1. sg_init_table
5457  * 2. skb_to_sgvec(payload1)
5458  * 3. sg_unmark_end
5459  * 4. skb_to_sgvec(payload2)
5460  *
5461  * When mapping multiple payload conditionally, skb_to_sgvec_nomark
5462  * is more preferable.
5463  */
5464 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
5465 			int offset, int len)
5466 {
5467 	return __skb_to_sgvec(skb, sg, offset, len, 0);
5468 }
5469 EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark);
5470 
5471 
5472 
5473 /**
5474  *	skb_cow_data - Check that a socket buffer's data buffers are writable
5475  *	@skb: The socket buffer to check.
5476  *	@tailbits: Amount of trailing space to be added
5477  *	@trailer: Returned pointer to the skb where the @tailbits space begins
5478  *
5479  *	Make sure that the data buffers attached to a socket buffer are
5480  *	writable. If they are not, private copies are made of the data buffers
5481  *	and the socket buffer is set to use these instead.
5482  *
5483  *	If @tailbits is given, make sure that there is space to write @tailbits
5484  *	bytes of data beyond current end of socket buffer.  @trailer will be
5485  *	set to point to the skb in which this space begins.
5486  *
5487  *	The number of scatterlist elements required to completely map the
5488  *	COW'd and extended socket buffer will be returned.
5489  */
5490 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
5491 {
5492 	int copyflag;
5493 	int elt;
5494 	struct sk_buff *skb1, **skb_p;
5495 
5496 	/* If skb is cloned or its head is paged, reallocate
5497 	 * head pulling out all the pages (pages are considered not writable
5498 	 * at the moment even if they are anonymous).
5499 	 */
5500 	if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
5501 	    !__pskb_pull_tail(skb, __skb_pagelen(skb)))
5502 		return -ENOMEM;
5503 
5504 	/* Easy case. Most of packets will go this way. */
5505 	if (!skb_has_frag_list(skb)) {
5506 		/* A little of trouble, not enough of space for trailer.
5507 		 * This should not happen, when stack is tuned to generate
5508 		 * good frames. OK, on miss we reallocate and reserve even more
5509 		 * space, 128 bytes is fair. */
5510 
5511 		if (skb_tailroom(skb) < tailbits &&
5512 		    pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
5513 			return -ENOMEM;
5514 
5515 		/* Voila! */
5516 		*trailer = skb;
5517 		return 1;
5518 	}
5519 
5520 	/* Misery. We are in troubles, going to mincer fragments... */
5521 
5522 	elt = 1;
5523 	skb_p = &skb_shinfo(skb)->frag_list;
5524 	copyflag = 0;
5525 
5526 	while ((skb1 = *skb_p) != NULL) {
5527 		int ntail = 0;
5528 
5529 		/* The fragment is partially pulled by someone,
5530 		 * this can happen on input. Copy it and everything
5531 		 * after it. */
5532 
5533 		if (skb_shared(skb1))
5534 			copyflag = 1;
5535 
5536 		/* If the skb is the last, worry about trailer. */
5537 
5538 		if (skb1->next == NULL && tailbits) {
5539 			if (skb_shinfo(skb1)->nr_frags ||
5540 			    skb_has_frag_list(skb1) ||
5541 			    skb_tailroom(skb1) < tailbits)
5542 				ntail = tailbits + 128;
5543 		}
5544 
5545 		if (copyflag ||
5546 		    skb_cloned(skb1) ||
5547 		    ntail ||
5548 		    skb_shinfo(skb1)->nr_frags ||
5549 		    skb_has_frag_list(skb1)) {
5550 			struct sk_buff *skb2;
5551 
5552 			/* Fuck, we are miserable poor guys... */
5553 			if (ntail == 0)
5554 				skb2 = skb_copy(skb1, GFP_ATOMIC);
5555 			else
5556 				skb2 = skb_copy_expand(skb1,
5557 						       skb_headroom(skb1),
5558 						       ntail,
5559 						       GFP_ATOMIC);
5560 			if (unlikely(skb2 == NULL))
5561 				return -ENOMEM;
5562 
5563 			if (skb1->sk)
5564 				skb_set_owner_w(skb2, skb1->sk);
5565 
5566 			/* Looking around. Are we still alive?
5567 			 * OK, link new skb, drop old one */
5568 
5569 			skb2->next = skb1->next;
5570 			*skb_p = skb2;
5571 			kfree_skb(skb1);
5572 			skb1 = skb2;
5573 		}
5574 		elt++;
5575 		*trailer = skb1;
5576 		skb_p = &skb1->next;
5577 	}
5578 
5579 	return elt;
5580 }
5581 EXPORT_SYMBOL_GPL(skb_cow_data);
5582 
5583 void sock_rmem_free(struct sk_buff *skb)
5584 {
5585 	struct sock *sk = skb->sk;
5586 
5587 	atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
5588 }
5589 
5590 static void skb_set_err_queue(struct sk_buff *skb)
5591 {
5592 	/* The error-queue test in skb_is_err_queue() matches this marker
5593 	 * with the sock_rmem_free destructor installed by sock_queue_err_skb().
5594 	 */
5595 	skb->pkt_type = PACKET_OUTGOING;
5596 	BUILD_BUG_ON(PACKET_OUTGOING == 0);
5597 }
5598 
5599 /*
5600  * Note: We dont mem charge error packets (no sk_forward_alloc changes)
5601  */
5602 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
5603 {
5604 	if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
5605 	    (unsigned int)READ_ONCE(sk->sk_rcvbuf))
5606 		return -ENOMEM;
5607 
5608 	skb_orphan(skb);
5609 	skb->sk = sk;
5610 	skb->destructor = sock_rmem_free;
5611 	atomic_add(skb->truesize, &sk->sk_rmem_alloc);
5612 	skb_set_err_queue(skb);
5613 
5614 	/* before exiting rcu section, make sure dst is refcounted */
5615 	skb_dst_force(skb);
5616 
5617 	skb_queue_tail(&sk->sk_error_queue, skb);
5618 	if (!sock_flag(sk, SOCK_DEAD))
5619 		sk_error_report(sk);
5620 	return 0;
5621 }
5622 EXPORT_SYMBOL(sock_queue_err_skb);
5623 
5624 static bool is_icmp_err_skb(const struct sk_buff *skb)
5625 {
5626 	return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP ||
5627 		       SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6);
5628 }
5629 
5630 struct sk_buff *sock_dequeue_err_skb(struct sock *sk)
5631 {
5632 	struct sk_buff_head *q = &sk->sk_error_queue;
5633 	struct sk_buff *skb, *skb_next = NULL;
5634 	bool icmp_next = false;
5635 	unsigned long flags;
5636 
5637 	if (skb_queue_empty_lockless(q))
5638 		return NULL;
5639 
5640 	spin_lock_irqsave(&q->lock, flags);
5641 	skb = __skb_dequeue(q);
5642 	if (skb && (skb_next = skb_peek(q))) {
5643 		icmp_next = is_icmp_err_skb(skb_next);
5644 		if (icmp_next)
5645 			sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_errno;
5646 	}
5647 	spin_unlock_irqrestore(&q->lock, flags);
5648 
5649 	if (is_icmp_err_skb(skb) && !icmp_next)
5650 		sk->sk_err = 0;
5651 
5652 	if (skb_next)
5653 		sk_error_report(sk);
5654 
5655 	return skb;
5656 }
5657 EXPORT_SYMBOL(sock_dequeue_err_skb);
5658 
5659 /**
5660  * skb_clone_sk - create clone of skb, and take reference to socket
5661  * @skb: the skb to clone
5662  *
5663  * This function creates a clone of a buffer that holds a reference on
5664  * sk_refcnt.  Buffers created via this function are meant to be
5665  * returned using sock_queue_err_skb, or free via kfree_skb.
5666  *
5667  * When passing buffers allocated with this function to sock_queue_err_skb
5668  * it is necessary to wrap the call with sock_hold/sock_put in order to
5669  * prevent the socket from being released prior to being enqueued on
5670  * the sk_error_queue.
5671  */
5672 struct sk_buff *skb_clone_sk(struct sk_buff *skb)
5673 {
5674 	struct sock *sk = skb->sk;
5675 	struct sk_buff *clone;
5676 
5677 	if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt))
5678 		return NULL;
5679 
5680 	clone = skb_clone(skb, GFP_ATOMIC);
5681 	if (!clone) {
5682 		sock_put(sk);
5683 		return NULL;
5684 	}
5685 
5686 	clone->sk = sk;
5687 	clone->destructor = sock_efree;
5688 
5689 	return clone;
5690 }
5691 EXPORT_SYMBOL(skb_clone_sk);
5692 
5693 static void __skb_complete_tx_timestamp(struct sk_buff *skb,
5694 					struct sock *sk,
5695 					int tstype,
5696 					bool opt_stats)
5697 {
5698 	struct sock_exterr_skb *serr;
5699 	int err;
5700 
5701 	BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb));
5702 
5703 	serr = SKB_EXT_ERR(skb);
5704 	memset(serr, 0, sizeof(*serr));
5705 	serr->ee.ee_errno = ENOMSG;
5706 	serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
5707 	serr->ee.ee_info = tstype;
5708 	serr->opt_stats = opt_stats;
5709 	serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0;
5710 	if (READ_ONCE(sk->sk_tsflags) & SOF_TIMESTAMPING_OPT_ID) {
5711 		serr->ee.ee_data = skb_shinfo(skb)->tskey;
5712 		if (sk_is_tcp(sk))
5713 			serr->ee.ee_data -= atomic_read(&sk->sk_tskey);
5714 	}
5715 
5716 	err = sock_queue_err_skb(sk, skb);
5717 
5718 	if (err)
5719 		kfree_skb(skb);
5720 }
5721 
5722 static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly)
5723 {
5724 	struct socket *sock;
5725 	struct file *file;
5726 	bool ret = false;
5727 
5728 	if (likely(tsonly || READ_ONCE(sock_net(sk)->core.sysctl_tstamp_allow_data)))
5729 		return true;
5730 
5731 	/* The sk pointer remains valid as long as the skb is. The sk_socket and
5732 	 * file pointer may become NULL if the socket is closed. Both structures
5733 	 * (including file->cred) are RCU freed which means they can be accessed
5734 	 * within a RCU read section.
5735 	 */
5736 	rcu_read_lock();
5737 	sock = READ_ONCE(sk->sk_socket);
5738 	if (!sock)
5739 		goto out;
5740 	file = READ_ONCE(sock->file);
5741 	if (!file)
5742 		goto out;
5743 	ret = file_ns_capable(file, &init_user_ns, CAP_NET_RAW);
5744 out:
5745 	rcu_read_unlock();
5746 	return ret;
5747 }
5748 
5749 void skb_complete_tx_timestamp(struct sk_buff *skb,
5750 			       struct skb_shared_hwtstamps *hwtstamps)
5751 {
5752 	struct sock *sk = skb->sk;
5753 
5754 	if (!skb_may_tx_timestamp(sk, false))
5755 		goto err;
5756 
5757 	/* Take a reference to prevent skb_orphan() from freeing the socket,
5758 	 * but only if the socket refcount is not zero.
5759 	 */
5760 	if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
5761 		*skb_hwtstamps(skb) = *hwtstamps;
5762 		__skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false);
5763 		sock_put(sk);
5764 		return;
5765 	}
5766 
5767 err:
5768 	kfree_skb(skb);
5769 }
5770 EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp);
5771 
5772 static bool skb_tstamp_tx_report_so_timestamping(struct sk_buff *skb,
5773 						 struct skb_shared_hwtstamps *hwtstamps,
5774 						 int tstype)
5775 {
5776 	switch (tstype) {
5777 	case SCM_TSTAMP_SCHED:
5778 		return skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP;
5779 	case SCM_TSTAMP_SND:
5780 		return skb_shinfo(skb)->tx_flags & (hwtstamps ? SKBTX_HW_TSTAMP_NOBPF :
5781 						    SKBTX_SW_TSTAMP);
5782 	case SCM_TSTAMP_ACK:
5783 		return TCP_SKB_CB(skb)->txstamp_ack & TSTAMP_ACK_SK;
5784 	case SCM_TSTAMP_COMPLETION:
5785 		return skb_shinfo(skb)->tx_flags & SKBTX_COMPLETION_TSTAMP;
5786 	}
5787 
5788 	return false;
5789 }
5790 
5791 static void skb_tstamp_tx_report_bpf_timestamping(struct sk_buff *skb,
5792 						  struct skb_shared_hwtstamps *hwtstamps,
5793 						  struct sock *sk,
5794 						  int tstype)
5795 {
5796 	int op;
5797 
5798 	switch (tstype) {
5799 	case SCM_TSTAMP_SCHED:
5800 		op = BPF_SOCK_OPS_TSTAMP_SCHED_CB;
5801 		break;
5802 	case SCM_TSTAMP_SND:
5803 		if (hwtstamps) {
5804 			op = BPF_SOCK_OPS_TSTAMP_SND_HW_CB;
5805 			*skb_hwtstamps(skb) = *hwtstamps;
5806 		} else {
5807 			op = BPF_SOCK_OPS_TSTAMP_SND_SW_CB;
5808 		}
5809 		break;
5810 	case SCM_TSTAMP_ACK:
5811 		op = BPF_SOCK_OPS_TSTAMP_ACK_CB;
5812 		break;
5813 	default:
5814 		return;
5815 	}
5816 
5817 	bpf_skops_tx_timestamping(sk, skb, op);
5818 }
5819 
5820 void __skb_tstamp_tx(struct sk_buff *orig_skb,
5821 		     const struct sk_buff *ack_skb,
5822 		     struct skb_shared_hwtstamps *hwtstamps,
5823 		     struct sock *sk, int tstype)
5824 {
5825 	struct sk_buff *skb;
5826 	bool tsonly, opt_stats = false;
5827 	u32 tsflags;
5828 
5829 	if (!sk)
5830 		return;
5831 
5832 	if (skb_shinfo(orig_skb)->tx_flags & SKBTX_BPF)
5833 		skb_tstamp_tx_report_bpf_timestamping(orig_skb, hwtstamps,
5834 						      sk, tstype);
5835 
5836 	if (!skb_tstamp_tx_report_so_timestamping(orig_skb, hwtstamps, tstype))
5837 		return;
5838 
5839 	tsflags = READ_ONCE(sk->sk_tsflags);
5840 	if (!hwtstamps && !(tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) &&
5841 	    skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS)
5842 		return;
5843 
5844 	tsonly = tsflags & SOF_TIMESTAMPING_OPT_TSONLY;
5845 	if (!skb_may_tx_timestamp(sk, tsonly))
5846 		return;
5847 
5848 	if (tsonly) {
5849 #ifdef CONFIG_INET
5850 		if ((tsflags & SOF_TIMESTAMPING_OPT_STATS) &&
5851 		    sk_is_tcp(sk)) {
5852 			skb = tcp_get_timestamping_opt_stats(sk, orig_skb,
5853 							     ack_skb);
5854 			opt_stats = true;
5855 		} else
5856 #endif
5857 			skb = alloc_skb(0, GFP_ATOMIC);
5858 	} else {
5859 		skb = skb_clone(orig_skb, GFP_ATOMIC);
5860 
5861 		if (skb_orphan_frags_rx(skb, GFP_ATOMIC)) {
5862 			kfree_skb(skb);
5863 			return;
5864 		}
5865 	}
5866 	if (!skb)
5867 		return;
5868 
5869 	if (tsonly) {
5870 		skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags &
5871 					     SKBTX_ANY_TSTAMP;
5872 		skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey;
5873 	}
5874 
5875 	if (hwtstamps)
5876 		*skb_hwtstamps(skb) = *hwtstamps;
5877 	else
5878 		__net_timestamp(skb);
5879 
5880 	__skb_complete_tx_timestamp(skb, sk, tstype, opt_stats);
5881 }
5882 EXPORT_SYMBOL_GPL(__skb_tstamp_tx);
5883 
5884 void skb_tstamp_tx(struct sk_buff *orig_skb,
5885 		   struct skb_shared_hwtstamps *hwtstamps)
5886 {
5887 	return __skb_tstamp_tx(orig_skb, NULL, hwtstamps, orig_skb->sk,
5888 			       SCM_TSTAMP_SND);
5889 }
5890 EXPORT_SYMBOL_GPL(skb_tstamp_tx);
5891 
5892 #ifdef CONFIG_WIRELESS
5893 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
5894 {
5895 	struct sock *sk = skb->sk;
5896 	struct sock_exterr_skb *serr;
5897 	int err = 1;
5898 
5899 	skb->wifi_acked_valid = 1;
5900 	skb->wifi_acked = acked;
5901 
5902 	serr = SKB_EXT_ERR(skb);
5903 	memset(serr, 0, sizeof(*serr));
5904 	serr->ee.ee_errno = ENOMSG;
5905 	serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;
5906 
5907 	/* Take a reference to prevent skb_orphan() from freeing the socket,
5908 	 * but only if the socket refcount is not zero.
5909 	 */
5910 	if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
5911 		err = sock_queue_err_skb(sk, skb);
5912 		sock_put(sk);
5913 	}
5914 	if (err)
5915 		kfree_skb(skb);
5916 }
5917 EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);
5918 #endif /* CONFIG_WIRELESS */
5919 
5920 /**
5921  * skb_partial_csum_set - set up and verify partial csum values for packet
5922  * @skb: the skb to set
5923  * @start: the number of bytes after skb->data to start checksumming.
5924  * @off: the offset from start to place the checksum.
5925  *
5926  * For untrusted partially-checksummed packets, we need to make sure the values
5927  * for skb->csum_start and skb->csum_offset are valid so we don't oops.
5928  *
5929  * This function checks and sets those values and skb->ip_summed: if this
5930  * returns false you should drop the packet.
5931  */
5932 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
5933 {
5934 	u32 csum_end = (u32)start + (u32)off + sizeof(__sum16);
5935 	u32 csum_start = skb_headroom(skb) + (u32)start;
5936 
5937 	if (unlikely(csum_start >= U16_MAX || csum_end > skb_headlen(skb))) {
5938 		net_warn_ratelimited("bad partial csum: csum=%u/%u headroom=%u headlen=%u\n",
5939 				     start, off, skb_headroom(skb), skb_headlen(skb));
5940 		return false;
5941 	}
5942 	skb->ip_summed = CHECKSUM_PARTIAL;
5943 	skb->csum_start = csum_start;
5944 	skb->csum_offset = off;
5945 	skb->transport_header = csum_start;
5946 	return true;
5947 }
5948 EXPORT_SYMBOL_GPL(skb_partial_csum_set);
5949 
5950 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len,
5951 			       unsigned int max)
5952 {
5953 	if (skb_headlen(skb) >= len)
5954 		return 0;
5955 
5956 	/* If we need to pullup then pullup to the max, so we
5957 	 * won't need to do it again.
5958 	 */
5959 	if (max > skb->len)
5960 		max = skb->len;
5961 
5962 	if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL)
5963 		return -ENOMEM;
5964 
5965 	if (skb_headlen(skb) < len)
5966 		return -EPROTO;
5967 
5968 	return 0;
5969 }
5970 
5971 #define MAX_TCP_HDR_LEN (15 * 4)
5972 
5973 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb,
5974 				      typeof(IPPROTO_IP) proto,
5975 				      unsigned int off)
5976 {
5977 	int err;
5978 
5979 	switch (proto) {
5980 	case IPPROTO_TCP:
5981 		err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr),
5982 					  off + MAX_TCP_HDR_LEN);
5983 		if (!err && !skb_partial_csum_set(skb, off,
5984 						  offsetof(struct tcphdr,
5985 							   check)))
5986 			err = -EPROTO;
5987 		return err ? ERR_PTR(err) : &tcp_hdr(skb)->check;
5988 
5989 	case IPPROTO_UDP:
5990 		err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr),
5991 					  off + sizeof(struct udphdr));
5992 		if (!err && !skb_partial_csum_set(skb, off,
5993 						  offsetof(struct udphdr,
5994 							   check)))
5995 			err = -EPROTO;
5996 		return err ? ERR_PTR(err) : &udp_hdr(skb)->check;
5997 	}
5998 
5999 	return ERR_PTR(-EPROTO);
6000 }
6001 
6002 /* This value should be large enough to cover a tagged ethernet header plus
6003  * maximally sized IP and TCP or UDP headers.
6004  */
6005 #define MAX_IP_HDR_LEN 128
6006 
6007 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate)
6008 {
6009 	unsigned int off;
6010 	bool fragment;
6011 	__sum16 *csum;
6012 	int err;
6013 
6014 	fragment = false;
6015 
6016 	err = skb_maybe_pull_tail(skb,
6017 				  sizeof(struct iphdr),
6018 				  MAX_IP_HDR_LEN);
6019 	if (err < 0)
6020 		goto out;
6021 
6022 	if (ip_is_fragment(ip_hdr(skb)))
6023 		fragment = true;
6024 
6025 	off = ip_hdrlen(skb);
6026 
6027 	err = -EPROTO;
6028 
6029 	if (fragment)
6030 		goto out;
6031 
6032 	csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off);
6033 	if (IS_ERR(csum))
6034 		return PTR_ERR(csum);
6035 
6036 	if (recalculate)
6037 		*csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
6038 					   ip_hdr(skb)->daddr,
6039 					   skb->len - off,
6040 					   ip_hdr(skb)->protocol, 0);
6041 	err = 0;
6042 
6043 out:
6044 	return err;
6045 }
6046 
6047 /* This value should be large enough to cover a tagged ethernet header plus
6048  * an IPv6 header, all options, and a maximal TCP or UDP header.
6049  */
6050 #define MAX_IPV6_HDR_LEN 256
6051 
6052 #define OPT_HDR(type, skb, off) \
6053 	(type *)(skb_network_header(skb) + (off))
6054 
6055 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate)
6056 {
6057 	int err;
6058 	u8 nexthdr;
6059 	unsigned int off;
6060 	unsigned int len;
6061 	bool fragment;
6062 	bool done;
6063 	__sum16 *csum;
6064 
6065 	fragment = false;
6066 	done = false;
6067 
6068 	off = sizeof(struct ipv6hdr);
6069 
6070 	err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN);
6071 	if (err < 0)
6072 		goto out;
6073 
6074 	nexthdr = ipv6_hdr(skb)->nexthdr;
6075 
6076 	len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len);
6077 	while (off <= len && !done) {
6078 		switch (nexthdr) {
6079 		case IPPROTO_DSTOPTS:
6080 		case IPPROTO_HOPOPTS:
6081 		case IPPROTO_ROUTING: {
6082 			struct ipv6_opt_hdr *hp;
6083 
6084 			err = skb_maybe_pull_tail(skb,
6085 						  off +
6086 						  sizeof(struct ipv6_opt_hdr),
6087 						  off +
6088 						  sizeof(struct ipv6_opt_hdr));
6089 			if (err < 0)
6090 				goto out;
6091 
6092 			hp = OPT_HDR(struct ipv6_opt_hdr, skb, off);
6093 			nexthdr = hp->nexthdr;
6094 			off += ipv6_optlen(hp);
6095 			break;
6096 		}
6097 		case IPPROTO_AH: {
6098 			struct ip_auth_hdr *hp;
6099 
6100 			err = skb_maybe_pull_tail(skb,
6101 						  off +
6102 						  sizeof(struct ip_auth_hdr),
6103 						  off +
6104 						  sizeof(struct ip_auth_hdr));
6105 			if (err < 0)
6106 				goto out;
6107 
6108 			hp = OPT_HDR(struct ip_auth_hdr, skb, off);
6109 			nexthdr = hp->nexthdr;
6110 			off += ipv6_authlen(hp);
6111 			break;
6112 		}
6113 		case IPPROTO_FRAGMENT: {
6114 			struct frag_hdr *hp;
6115 
6116 			err = skb_maybe_pull_tail(skb,
6117 						  off +
6118 						  sizeof(struct frag_hdr),
6119 						  off +
6120 						  sizeof(struct frag_hdr));
6121 			if (err < 0)
6122 				goto out;
6123 
6124 			hp = OPT_HDR(struct frag_hdr, skb, off);
6125 
6126 			if (hp->frag_off & htons(IP6_OFFSET | IP6_MF))
6127 				fragment = true;
6128 
6129 			nexthdr = hp->nexthdr;
6130 			off += sizeof(struct frag_hdr);
6131 			break;
6132 		}
6133 		default:
6134 			done = true;
6135 			break;
6136 		}
6137 	}
6138 
6139 	err = -EPROTO;
6140 
6141 	if (!done || fragment)
6142 		goto out;
6143 
6144 	csum = skb_checksum_setup_ip(skb, nexthdr, off);
6145 	if (IS_ERR(csum))
6146 		return PTR_ERR(csum);
6147 
6148 	if (recalculate)
6149 		*csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
6150 					 &ipv6_hdr(skb)->daddr,
6151 					 skb->len - off, nexthdr, 0);
6152 	err = 0;
6153 
6154 out:
6155 	return err;
6156 }
6157 
6158 /**
6159  * skb_checksum_setup - set up partial checksum offset
6160  * @skb: the skb to set up
6161  * @recalculate: if true the pseudo-header checksum will be recalculated
6162  */
6163 int skb_checksum_setup(struct sk_buff *skb, bool recalculate)
6164 {
6165 	int err;
6166 
6167 	switch (skb->protocol) {
6168 	case htons(ETH_P_IP):
6169 		err = skb_checksum_setup_ipv4(skb, recalculate);
6170 		break;
6171 
6172 	case htons(ETH_P_IPV6):
6173 		err = skb_checksum_setup_ipv6(skb, recalculate);
6174 		break;
6175 
6176 	default:
6177 		err = -EPROTO;
6178 		break;
6179 	}
6180 
6181 	return err;
6182 }
6183 EXPORT_SYMBOL(skb_checksum_setup);
6184 
6185 /**
6186  * skb_checksum_maybe_trim - maybe trims the given skb
6187  * @skb: the skb to check
6188  * @transport_len: the data length beyond the network header
6189  *
6190  * Checks whether the given skb has data beyond the given transport length.
6191  * If so, returns a cloned skb trimmed to this transport length.
6192  * Otherwise returns the provided skb. Returns NULL in error cases
6193  * (e.g. transport_len exceeds skb length or out-of-memory).
6194  *
6195  * Caller needs to set the skb transport header and free any returned skb if it
6196  * differs from the provided skb.
6197  */
6198 static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb,
6199 					       unsigned int transport_len)
6200 {
6201 	struct sk_buff *skb_chk;
6202 	unsigned int len = skb_transport_offset(skb) + transport_len;
6203 	int ret;
6204 
6205 	if (skb->len < len)
6206 		return NULL;
6207 	else if (skb->len == len)
6208 		return skb;
6209 
6210 	skb_chk = skb_clone(skb, GFP_ATOMIC);
6211 	if (!skb_chk)
6212 		return NULL;
6213 
6214 	ret = pskb_trim_rcsum(skb_chk, len);
6215 	if (ret) {
6216 		kfree_skb(skb_chk);
6217 		return NULL;
6218 	}
6219 
6220 	return skb_chk;
6221 }
6222 
6223 /**
6224  * skb_checksum_trimmed - validate checksum of an skb
6225  * @skb: the skb to check
6226  * @transport_len: the data length beyond the network header
6227  * @skb_chkf: checksum function to use
6228  *
6229  * Applies the given checksum function skb_chkf to the provided skb.
6230  * Returns a checked and maybe trimmed skb. Returns NULL on error.
6231  *
6232  * If the skb has data beyond the given transport length, then a
6233  * trimmed & cloned skb is checked and returned.
6234  *
6235  * Caller needs to set the skb transport header and free any returned skb if it
6236  * differs from the provided skb.
6237  */
6238 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
6239 				     unsigned int transport_len,
6240 				     __sum16(*skb_chkf)(struct sk_buff *skb))
6241 {
6242 	struct sk_buff *skb_chk;
6243 	unsigned int offset = skb_transport_offset(skb);
6244 	__sum16 ret;
6245 
6246 	skb_chk = skb_checksum_maybe_trim(skb, transport_len);
6247 	if (!skb_chk)
6248 		goto err;
6249 
6250 	if (!pskb_may_pull(skb_chk, offset))
6251 		goto err;
6252 
6253 	skb_pull_rcsum(skb_chk, offset);
6254 	ret = skb_chkf(skb_chk);
6255 	skb_push_rcsum(skb_chk, offset);
6256 
6257 	if (ret)
6258 		goto err;
6259 
6260 	return skb_chk;
6261 
6262 err:
6263 	if (skb_chk && skb_chk != skb)
6264 		kfree_skb(skb_chk);
6265 
6266 	return NULL;
6267 
6268 }
6269 EXPORT_SYMBOL(skb_checksum_trimmed);
6270 
6271 void __skb_warn_lro_forwarding(const struct sk_buff *skb)
6272 {
6273 	net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n",
6274 			     skb->dev->name);
6275 }
6276 EXPORT_SYMBOL(__skb_warn_lro_forwarding);
6277 
6278 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
6279 {
6280 	if (head_stolen) {
6281 		skb_release_head_state(skb);
6282 		kfree_skbmem(skb);
6283 	} else {
6284 		__kfree_skb(skb);
6285 	}
6286 }
6287 EXPORT_SYMBOL(kfree_skb_partial);
6288 
6289 /**
6290  * skb_try_coalesce - try to merge skb to prior one
6291  * @to: prior buffer
6292  * @from: buffer to add
6293  * @fragstolen: pointer to boolean
6294  * @delta_truesize: how much more was allocated than was requested
6295  */
6296 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
6297 		      bool *fragstolen, int *delta_truesize)
6298 {
6299 	struct skb_shared_info *to_shinfo, *from_shinfo;
6300 	int i, delta, len = from->len;
6301 
6302 	*fragstolen = false;
6303 
6304 	if (skb_cloned(to))
6305 		return false;
6306 
6307 	/* In general, avoid mixing page_pool and non-page_pool allocated
6308 	 * pages within the same SKB. In theory we could take full
6309 	 * references if @from is cloned and !@to->pp_recycle but its
6310 	 * tricky (due to potential race with the clone disappearing) and
6311 	 * rare, so not worth dealing with.
6312 	 */
6313 	if (to->pp_recycle != from->pp_recycle)
6314 		return false;
6315 
6316 	/* All frags in an skb must either all be struct page-backed or all
6317 	 * belong to the same memory provider instance; do not coalesce skbs
6318 	 * that would mix them.
6319 	 */
6320 	if (skb_frags_readable(from) != skb_frags_readable(to))
6321 		return false;
6322 
6323 	if (len <= skb_tailroom(to) && skb_frags_readable(from)) {
6324 		if (len) {
6325 			if (unlikely(skb_copy_bits(from, 0,
6326 						   skb_tail_pointer(to),
6327 						   len))) {
6328 				DEBUG_NET_WARN_ON_ONCE(1);
6329 				return false;
6330 			}
6331 			skb_put(to, len);
6332 		}
6333 		*delta_truesize = 0;
6334 		return true;
6335 	}
6336 
6337 	to_shinfo = skb_shinfo(to);
6338 	from_shinfo = skb_shinfo(from);
6339 	if (to_shinfo->frag_list || from_shinfo->frag_list)
6340 		return false;
6341 	if (skb_zcopy(to) || skb_zcopy(from))
6342 		return false;
6343 
6344 	if (skb_headlen(from) != 0) {
6345 		struct page *page;
6346 		unsigned int offset;
6347 
6348 		if (to_shinfo->nr_frags +
6349 		    from_shinfo->nr_frags >= MAX_SKB_FRAGS)
6350 			return false;
6351 
6352 		if (skb_head_is_locked(from))
6353 			return false;
6354 
6355 		delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
6356 
6357 		page = virt_to_head_page(from->head);
6358 		offset = from->data - (unsigned char *)page_address(page);
6359 
6360 		skb_fill_page_desc(to, to_shinfo->nr_frags,
6361 				   page, offset, skb_headlen(from));
6362 		*fragstolen = true;
6363 	} else {
6364 		if (to_shinfo->nr_frags +
6365 		    from_shinfo->nr_frags > MAX_SKB_FRAGS)
6366 			return false;
6367 
6368 		delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from));
6369 	}
6370 
6371 	WARN_ON_ONCE(delta < len);
6372 
6373 	memcpy(to_shinfo->frags + to_shinfo->nr_frags,
6374 	       from_shinfo->frags,
6375 	       from_shinfo->nr_frags * sizeof(skb_frag_t));
6376 	to_shinfo->nr_frags += from_shinfo->nr_frags;
6377 	if (from_shinfo->nr_frags)
6378 		to_shinfo->flags |= from_shinfo->flags & SKBFL_SHARED_FRAG;
6379 
6380 	if (!skb_cloned(from))
6381 		from_shinfo->nr_frags = 0;
6382 
6383 	/* if the skb is not cloned this does nothing
6384 	 * since we set nr_frags to 0.
6385 	 */
6386 	for (i = 0; i < from_shinfo->nr_frags; i++)
6387 		skb_frag_ref(from, i);
6388 
6389 	to->truesize += delta;
6390 	to->len += len;
6391 	to->data_len += len;
6392 
6393 	*delta_truesize = delta;
6394 	return true;
6395 }
6396 EXPORT_SYMBOL(skb_try_coalesce);
6397 
6398 static void skb_ext_scrub(struct sk_buff *skb)
6399 {
6400 #if IS_ENABLED(CONFIG_INET_PSP)
6401 	struct psp_skb_ext *pse = skb_ext_find(skb, SKB_EXT_PSP);
6402 
6403 	if (pse) {
6404 		struct skb_ext *ext = skb->extensions;
6405 		struct psp_skb_ext tmp;
6406 
6407 		/* Nothing else to scrub if PSP is the only extension in ext. */
6408 		if (ext->chunks == SKB_EXT_CHUNKSIZEOF(*ext) +
6409 				   skb_ext_type_len[SKB_EXT_PSP])
6410 			return;
6411 
6412 		tmp = *pse;
6413 		skb_ext_reset(skb);
6414 		pse = skb_ext_add(skb, SKB_EXT_PSP);
6415 		if (pse)
6416 			memcpy(pse, &tmp, sizeof(*pse));
6417 		return;
6418 	}
6419 #endif
6420 	skb_ext_reset(skb);
6421 }
6422 
6423 /**
6424  * skb_scrub_packet - scrub an skb
6425  *
6426  * @skb: buffer to clean
6427  * @xnet: packet is crossing netns
6428  *
6429  * skb_scrub_packet can be used after encapsulating or decapsulating a packet
6430  * into/from a tunnel. Some information have to be cleared during these
6431  * operations.
6432  * skb_scrub_packet can also be used to clean a skb before injecting it in
6433  * another namespace (@xnet == true). We have to clear all information in the
6434  * skb that could impact namespace isolation. Note that PSP skb extension is
6435  * preserved across packet scrubbing by design.
6436  */
6437 void skb_scrub_packet(struct sk_buff *skb, bool xnet)
6438 {
6439 	skb->pkt_type = PACKET_HOST;
6440 	skb->skb_iif = 0;
6441 	skb->ignore_df = 0;
6442 	skb_dst_drop(skb);
6443 	skb_ext_scrub(skb);
6444 	nf_reset_ct(skb);
6445 	nf_reset_trace(skb);
6446 
6447 #ifdef CONFIG_NET_SWITCHDEV
6448 	skb->offload_fwd_mark = 0;
6449 	skb->offload_l3_fwd_mark = 0;
6450 #endif
6451 	ipvs_reset(skb);
6452 
6453 	if (!xnet)
6454 		return;
6455 
6456 	skb->mark = 0;
6457 	skb_clear_tstamp(skb);
6458 }
6459 EXPORT_SYMBOL_GPL(skb_scrub_packet);
6460 
6461 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb)
6462 {
6463 	int mac_len, meta_len;
6464 	void *meta;
6465 
6466 	if (skb_cow(skb, skb_headroom(skb)) < 0) {
6467 		kfree_skb(skb);
6468 		return NULL;
6469 	}
6470 
6471 	mac_len = skb->data - skb_mac_header(skb);
6472 	if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) {
6473 		memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb),
6474 			mac_len - VLAN_HLEN - ETH_TLEN);
6475 	}
6476 
6477 	meta_len = skb_metadata_len(skb);
6478 	if (meta_len) {
6479 		meta = skb_metadata_end(skb) - meta_len;
6480 		memmove(meta + VLAN_HLEN, meta, meta_len);
6481 	}
6482 
6483 	skb->mac_header += VLAN_HLEN;
6484 	return skb;
6485 }
6486 
6487 struct sk_buff *skb_vlan_untag(struct sk_buff *skb)
6488 {
6489 	struct vlan_hdr *vhdr;
6490 	u16 vlan_tci;
6491 
6492 	if (unlikely(skb_vlan_tag_present(skb))) {
6493 		/* vlan_tci is already set-up so leave this for another time */
6494 		return skb;
6495 	}
6496 
6497 	skb = skb_share_check(skb, GFP_ATOMIC);
6498 	if (unlikely(!skb))
6499 		goto err_free;
6500 	if (unlikely(!pskb_may_pull(skb, VLAN_HLEN)))
6501 		goto err_free;
6502 
6503 	vhdr = (struct vlan_hdr *)skb->data;
6504 	vlan_tci = ntohs(vhdr->h_vlan_TCI);
6505 	__vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci);
6506 
6507 	skb_pull_rcsum(skb, VLAN_HLEN);
6508 	vlan_set_encap_proto(skb, vhdr);
6509 
6510 	skb = skb_reorder_vlan_header(skb);
6511 	if (unlikely(!skb))
6512 		goto err_free;
6513 
6514 	skb_reset_network_header(skb);
6515 	if (!skb_transport_header_was_set(skb))
6516 		skb_reset_transport_header(skb);
6517 	skb_reset_mac_len(skb);
6518 
6519 	return skb;
6520 
6521 err_free:
6522 	kfree_skb(skb);
6523 	return NULL;
6524 }
6525 EXPORT_SYMBOL(skb_vlan_untag);
6526 
6527 int skb_ensure_writable(struct sk_buff *skb, unsigned int write_len)
6528 {
6529 	if (!pskb_may_pull(skb, write_len))
6530 		return -ENOMEM;
6531 
6532 	if (!skb_cloned(skb) || skb_clone_writable(skb, write_len))
6533 		return 0;
6534 
6535 	return pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
6536 }
6537 EXPORT_SYMBOL(skb_ensure_writable);
6538 
6539 int skb_ensure_writable_head_tail(struct sk_buff *skb, struct net_device *dev)
6540 {
6541 	int needed_headroom = dev->needed_headroom;
6542 	int needed_tailroom = dev->needed_tailroom;
6543 
6544 	/* For tail taggers, we need to pad short frames ourselves, to ensure
6545 	 * that the tail tag does not fail at its role of being at the end of
6546 	 * the packet, once the conduit interface pads the frame. Account for
6547 	 * that pad length here, and pad later.
6548 	 */
6549 	if (unlikely(needed_tailroom && skb->len < ETH_ZLEN))
6550 		needed_tailroom += ETH_ZLEN - skb->len;
6551 	/* skb_headroom() returns unsigned int... */
6552 	needed_headroom = max_t(int, needed_headroom - skb_headroom(skb), 0);
6553 	needed_tailroom = max_t(int, needed_tailroom - skb_tailroom(skb), 0);
6554 
6555 	if (likely(!needed_headroom && !needed_tailroom && !skb_cloned(skb)))
6556 		/* No reallocation needed, yay! */
6557 		return 0;
6558 
6559 	return pskb_expand_head(skb, needed_headroom, needed_tailroom,
6560 				GFP_ATOMIC);
6561 }
6562 EXPORT_SYMBOL(skb_ensure_writable_head_tail);
6563 
6564 /* remove VLAN header from packet and update csum accordingly.
6565  * expects a non skb_vlan_tag_present skb with a vlan tag payload
6566  */
6567 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci)
6568 {
6569 	int offset = skb->data - skb_mac_header(skb);
6570 	int err;
6571 
6572 	if (WARN_ONCE(offset,
6573 		      "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n",
6574 		      offset)) {
6575 		return -EINVAL;
6576 	}
6577 
6578 	err = skb_ensure_writable(skb, VLAN_ETH_HLEN);
6579 	if (unlikely(err))
6580 		return err;
6581 
6582 	skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
6583 
6584 	vlan_remove_tag(skb, vlan_tci);
6585 
6586 	skb->mac_header += VLAN_HLEN;
6587 
6588 	if (skb_network_offset(skb) < ETH_HLEN)
6589 		skb_set_network_header(skb, ETH_HLEN);
6590 
6591 	skb_reset_mac_len(skb);
6592 
6593 	return err;
6594 }
6595 EXPORT_SYMBOL(__skb_vlan_pop);
6596 
6597 /* Pop a vlan tag either from hwaccel or from payload.
6598  * Expects skb->data at mac header.
6599  */
6600 int skb_vlan_pop(struct sk_buff *skb)
6601 {
6602 	u16 vlan_tci;
6603 	__be16 vlan_proto;
6604 	int err;
6605 
6606 	if (likely(skb_vlan_tag_present(skb))) {
6607 		__vlan_hwaccel_clear_tag(skb);
6608 	} else {
6609 		if (unlikely(!eth_type_vlan(skb->protocol)))
6610 			return 0;
6611 
6612 		err = __skb_vlan_pop(skb, &vlan_tci);
6613 		if (err)
6614 			return err;
6615 	}
6616 	/* move next vlan tag to hw accel tag */
6617 	if (likely(!eth_type_vlan(skb->protocol)))
6618 		return 0;
6619 
6620 	vlan_proto = skb->protocol;
6621 	err = __skb_vlan_pop(skb, &vlan_tci);
6622 	if (unlikely(err))
6623 		return err;
6624 
6625 	__vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
6626 	return 0;
6627 }
6628 EXPORT_SYMBOL(skb_vlan_pop);
6629 
6630 /* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present).
6631  * Expects skb->data at mac header.
6632  */
6633 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci)
6634 {
6635 	if (skb_vlan_tag_present(skb)) {
6636 		int offset = skb->data - skb_mac_header(skb);
6637 		int err;
6638 
6639 		if (WARN_ONCE(offset,
6640 			      "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n",
6641 			      offset)) {
6642 			return -EINVAL;
6643 		}
6644 
6645 		err = __vlan_insert_tag(skb, skb->vlan_proto,
6646 					skb_vlan_tag_get(skb));
6647 		if (err)
6648 			return err;
6649 
6650 		skb->protocol = skb->vlan_proto;
6651 		skb->network_header -= VLAN_HLEN;
6652 
6653 		skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
6654 	}
6655 	__vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
6656 	return 0;
6657 }
6658 EXPORT_SYMBOL(skb_vlan_push);
6659 
6660 /**
6661  * skb_eth_pop() - Drop the Ethernet header at the head of a packet
6662  *
6663  * @skb: Socket buffer to modify
6664  *
6665  * Drop the Ethernet header of @skb.
6666  *
6667  * Expects that skb->data points to the mac header and that no VLAN tags are
6668  * present.
6669  *
6670  * Returns 0 on success, -errno otherwise.
6671  */
6672 int skb_eth_pop(struct sk_buff *skb)
6673 {
6674 	if (!pskb_may_pull(skb, ETH_HLEN) || skb_vlan_tagged(skb) ||
6675 	    skb_network_offset(skb) < ETH_HLEN)
6676 		return -EPROTO;
6677 
6678 	skb_pull_rcsum(skb, ETH_HLEN);
6679 	skb_reset_mac_header(skb);
6680 	skb_reset_mac_len(skb);
6681 
6682 	return 0;
6683 }
6684 EXPORT_SYMBOL(skb_eth_pop);
6685 
6686 /**
6687  * skb_eth_push() - Add a new Ethernet header at the head of a packet
6688  *
6689  * @skb: Socket buffer to modify
6690  * @dst: Destination MAC address of the new header
6691  * @src: Source MAC address of the new header
6692  *
6693  * Prepend @skb with a new Ethernet header.
6694  *
6695  * Expects that skb->data points to the mac header, which must be empty.
6696  *
6697  * Returns 0 on success, -errno otherwise.
6698  */
6699 int skb_eth_push(struct sk_buff *skb, const unsigned char *dst,
6700 		 const unsigned char *src)
6701 {
6702 	struct ethhdr *eth;
6703 	int err;
6704 
6705 	if (skb_network_offset(skb) || skb_vlan_tag_present(skb))
6706 		return -EPROTO;
6707 
6708 	err = skb_cow_head(skb, sizeof(*eth));
6709 	if (err < 0)
6710 		return err;
6711 
6712 	skb_push(skb, sizeof(*eth));
6713 	skb_reset_mac_header(skb);
6714 	skb_reset_mac_len(skb);
6715 
6716 	eth = eth_hdr(skb);
6717 	ether_addr_copy(eth->h_dest, dst);
6718 	ether_addr_copy(eth->h_source, src);
6719 	eth->h_proto = skb->protocol;
6720 
6721 	skb_postpush_rcsum(skb, eth, sizeof(*eth));
6722 
6723 	return 0;
6724 }
6725 EXPORT_SYMBOL(skb_eth_push);
6726 
6727 /* Update the ethertype of hdr and the skb csum value if required. */
6728 static void skb_mod_eth_type(struct sk_buff *skb, struct ethhdr *hdr,
6729 			     __be16 ethertype)
6730 {
6731 	if (skb->ip_summed == CHECKSUM_COMPLETE) {
6732 		__be16 diff[] = { ~hdr->h_proto, ethertype };
6733 
6734 		skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
6735 	}
6736 
6737 	hdr->h_proto = ethertype;
6738 }
6739 
6740 /**
6741  * skb_mpls_push() - push a new MPLS header after mac_len bytes from start of
6742  *                   the packet
6743  *
6744  * @skb: buffer
6745  * @mpls_lse: MPLS label stack entry to push
6746  * @mpls_proto: ethertype of the new MPLS header (expects 0x8847 or 0x8848)
6747  * @mac_len: length of the MAC header
6748  * @ethernet: flag to indicate if the resulting packet after skb_mpls_push is
6749  *            ethernet
6750  *
6751  * Expects skb->data at mac header.
6752  *
6753  * Returns 0 on success, -errno otherwise.
6754  */
6755 int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto,
6756 		  int mac_len, bool ethernet)
6757 {
6758 	struct mpls_shim_hdr *lse;
6759 	int err;
6760 
6761 	if (unlikely(!eth_p_mpls(mpls_proto)))
6762 		return -EINVAL;
6763 
6764 	/* Networking stack does not allow simultaneous Tunnel and MPLS GSO. */
6765 	if (skb->encapsulation)
6766 		return -EINVAL;
6767 
6768 	err = skb_cow_head(skb, MPLS_HLEN);
6769 	if (unlikely(err))
6770 		return err;
6771 
6772 	if (!skb->inner_protocol) {
6773 		skb_set_inner_network_header(skb, skb_network_offset(skb));
6774 		skb_set_inner_protocol(skb, skb->protocol);
6775 	}
6776 
6777 	skb_push(skb, MPLS_HLEN);
6778 	memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb),
6779 		mac_len);
6780 	skb_reset_mac_header(skb);
6781 	skb_set_network_header(skb, mac_len);
6782 	skb_reset_mac_len(skb);
6783 
6784 	lse = mpls_hdr(skb);
6785 	lse->label_stack_entry = mpls_lse;
6786 	skb_postpush_rcsum(skb, lse, MPLS_HLEN);
6787 
6788 	if (ethernet && mac_len >= ETH_HLEN)
6789 		skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto);
6790 	skb->protocol = mpls_proto;
6791 
6792 	return 0;
6793 }
6794 EXPORT_SYMBOL_GPL(skb_mpls_push);
6795 
6796 /**
6797  * skb_mpls_pop() - pop the outermost MPLS header
6798  *
6799  * @skb: buffer
6800  * @next_proto: ethertype of header after popped MPLS header
6801  * @mac_len: length of the MAC header
6802  * @ethernet: flag to indicate if the packet is ethernet
6803  *
6804  * Expects skb->data at mac header.
6805  *
6806  * Returns 0 on success, -errno otherwise.
6807  */
6808 int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len,
6809 		 bool ethernet)
6810 {
6811 	int err;
6812 
6813 	if (unlikely(!eth_p_mpls(skb->protocol)))
6814 		return 0;
6815 
6816 	err = skb_ensure_writable(skb, mac_len + MPLS_HLEN);
6817 	if (unlikely(err))
6818 		return err;
6819 
6820 	skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN);
6821 	memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb),
6822 		mac_len);
6823 
6824 	__skb_pull(skb, MPLS_HLEN);
6825 	skb_reset_mac_header(skb);
6826 	skb_set_network_header(skb, mac_len);
6827 
6828 	if (ethernet && mac_len >= ETH_HLEN) {
6829 		struct ethhdr *hdr;
6830 
6831 		/* use mpls_hdr() to get ethertype to account for VLANs. */
6832 		hdr = (struct ethhdr *)((void *)mpls_hdr(skb) - ETH_HLEN);
6833 		skb_mod_eth_type(skb, hdr, next_proto);
6834 	}
6835 	skb->protocol = next_proto;
6836 
6837 	/* The last label is gone, so the inner header recorded by
6838 	 * skb_mpls_push() no longer describes this packet. Drop it, or a
6839 	 * later push keeps the stale offset.
6840 	 */
6841 	if (!eth_p_mpls(next_proto))
6842 		skb->inner_protocol = 0;
6843 
6844 	return 0;
6845 }
6846 EXPORT_SYMBOL_GPL(skb_mpls_pop);
6847 
6848 /**
6849  * skb_mpls_update_lse() - modify outermost MPLS header and update csum
6850  *
6851  * @skb: buffer
6852  * @mpls_lse: new MPLS label stack entry to update to
6853  *
6854  * Expects skb->data at mac header.
6855  *
6856  * Returns 0 on success, -errno otherwise.
6857  */
6858 int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse)
6859 {
6860 	int err;
6861 
6862 	if (unlikely(!eth_p_mpls(skb->protocol)))
6863 		return -EINVAL;
6864 
6865 	err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN);
6866 	if (unlikely(err))
6867 		return err;
6868 
6869 	if (skb->ip_summed == CHECKSUM_COMPLETE) {
6870 		__be32 diff[] = { ~mpls_hdr(skb)->label_stack_entry, mpls_lse };
6871 
6872 		skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
6873 	}
6874 
6875 	mpls_hdr(skb)->label_stack_entry = mpls_lse;
6876 
6877 	return 0;
6878 }
6879 EXPORT_SYMBOL_GPL(skb_mpls_update_lse);
6880 
6881 /**
6882  * skb_mpls_dec_ttl() - decrement the TTL of the outermost MPLS header
6883  *
6884  * @skb: buffer
6885  *
6886  * Expects skb->data at mac header.
6887  *
6888  * Returns 0 on success, -errno otherwise.
6889  */
6890 int skb_mpls_dec_ttl(struct sk_buff *skb)
6891 {
6892 	u32 lse;
6893 	u8 ttl;
6894 
6895 	if (unlikely(!eth_p_mpls(skb->protocol)))
6896 		return -EINVAL;
6897 
6898 	if (!pskb_may_pull(skb, skb_network_offset(skb) + MPLS_HLEN))
6899 		return -ENOMEM;
6900 
6901 	lse = be32_to_cpu(mpls_hdr(skb)->label_stack_entry);
6902 	ttl = (lse & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT;
6903 	if (!ttl || !--ttl)
6904 		return -EINVAL;
6905 
6906 	lse &= ~MPLS_LS_TTL_MASK;
6907 	lse |= ttl << MPLS_LS_TTL_SHIFT;
6908 
6909 	return skb_mpls_update_lse(skb, cpu_to_be32(lse));
6910 }
6911 EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl);
6912 
6913 /**
6914  * alloc_skb_with_frags - allocate skb with page frags
6915  *
6916  * @header_len: size of linear part
6917  * @data_len: needed length in frags
6918  * @order: max page order desired.
6919  * @errcode: pointer to error code if any
6920  * @gfp_mask: allocation mask
6921  *
6922  * This can be used to allocate a paged skb, given a maximal order for frags.
6923  */
6924 struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
6925 				     unsigned long data_len,
6926 				     int order,
6927 				     int *errcode,
6928 				     gfp_t gfp_mask)
6929 {
6930 	unsigned long chunk;
6931 	struct sk_buff *skb;
6932 	struct page *page;
6933 	int nr_frags = 0;
6934 
6935 	*errcode = -EMSGSIZE;
6936 	if (unlikely(data_len > MAX_SKB_FRAGS * (PAGE_SIZE << order)))
6937 		return NULL;
6938 
6939 	*errcode = -ENOBUFS;
6940 	skb = alloc_skb(header_len, gfp_mask);
6941 	if (!skb)
6942 		return NULL;
6943 
6944 	while (data_len) {
6945 		if (nr_frags == MAX_SKB_FRAGS)
6946 			goto failure;
6947 		while (order && PAGE_ALIGN(data_len) < (PAGE_SIZE << order))
6948 			order--;
6949 
6950 		if (order) {
6951 			page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) |
6952 					   __GFP_COMP |
6953 					   __GFP_NOWARN,
6954 					   order);
6955 			if (!page) {
6956 				order--;
6957 				continue;
6958 			}
6959 		} else {
6960 			page = alloc_page(gfp_mask);
6961 			if (!page)
6962 				goto failure;
6963 		}
6964 		chunk = min_t(unsigned long, data_len,
6965 			      PAGE_SIZE << order);
6966 		skb_fill_page_desc(skb, nr_frags, page, 0, chunk);
6967 		nr_frags++;
6968 		skb->truesize += (PAGE_SIZE << order);
6969 		data_len -= chunk;
6970 	}
6971 	return skb;
6972 
6973 failure:
6974 	kfree_skb(skb);
6975 	return NULL;
6976 }
6977 EXPORT_SYMBOL(alloc_skb_with_frags);
6978 
6979 /* pskb_carve_inside_header() and pskb_carve_inside_nonlinear()
6980  * remove the first bytes of a packet and reallocate skb->head.
6981  *
6982  * Whatever headers were present before the operation are gone,
6983  * we must not leave stale offsets, otherwise users of this skb
6984  * (skb_dump(), drop_monitor, taps, ...) would read or pull garbage.
6985  */
6986 static void skb_carve_reset_headers(struct sk_buff *skb)
6987 {
6988 	skb_unset_mac_header(skb);
6989 	skb_unset_transport_header(skb);
6990 	skb_reset_network_header(skb);
6991 	skb->mac_len = 0;
6992 
6993 	/* Inner offsets have no "unset" marker, zero them so that
6994 	 * skb_inner_network_header_was_set() becomes false and no
6995 	 * consumer mistakes them for a real (and long gone) header.
6996 	 */
6997 	skb->inner_mac_header = 0;
6998 	skb->inner_network_header = 0;
6999 	skb->inner_transport_header = 0;
7000 	skb->inner_protocol = 0;
7001 	skb->encapsulation = 0;
7002 
7003 	if (skb->ip_summed == CHECKSUM_PARTIAL)
7004 		skb->ip_summed = CHECKSUM_NONE;
7005 }
7006 
7007 /* carve out the first off bytes from skb when off < headlen */
7008 static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,
7009 				    const int headlen, gfp_t gfp_mask)
7010 {
7011 	int i;
7012 	unsigned int size = skb_end_offset(skb);
7013 	int new_hlen = headlen - off;
7014 	u8 *data;
7015 
7016 	if (skb_pfmemalloc(skb))
7017 		gfp_mask |= __GFP_MEMALLOC;
7018 
7019 	data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL);
7020 	if (!data)
7021 		return -ENOMEM;
7022 	size = SKB_WITH_OVERHEAD(size);
7023 
7024 	/* Copy real data, and all frags */
7025 	skb_copy_from_linear_data_offset(skb, off, data, new_hlen);
7026 	skb->len -= off;
7027 
7028 	/* Remove SKBFL_MANAGED_FRAG_REFS instead of trying to honour it
7029 	 * while refcounting frags below.
7030 	 */
7031 	skb_zcopy_downgrade_managed(skb);
7032 
7033 	memcpy((struct skb_shared_info *)(data + size),
7034 	       skb_shinfo(skb),
7035 	       offsetof(struct skb_shared_info,
7036 			frags[skb_shinfo(skb)->nr_frags]));
7037 	if (skb_cloned(skb)) {
7038 		/* drop the old head gracefully */
7039 		if (skb_orphan_frags(skb, gfp_mask)) {
7040 			skb_kfree_head(data);
7041 			return -ENOMEM;
7042 		}
7043 		if (skb_zcopy(skb))
7044 			net_zcopy_get(skb_uarg(skb));
7045 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
7046 			skb_frag_ref(skb, i);
7047 		if (skb_has_frag_list(skb))
7048 			skb_clone_fraglist(skb);
7049 		skb_release_data(skb, SKB_CONSUMED);
7050 	} else {
7051 		/* we can reuse existing recount- all we did was
7052 		 * relocate values
7053 		 */
7054 		skb_free_head(skb);
7055 	}
7056 
7057 	skb->head = data;
7058 	skb->data = data;
7059 	skb->head_frag = 0;
7060 	skb_set_end_offset(skb, size);
7061 	skb_set_tail_pointer(skb, skb_headlen(skb));
7062 	skb_carve_reset_headers(skb);
7063 	skb->cloned = 0;
7064 	skb->hdr_len = 0;
7065 	skb->nohdr = 0;
7066 	atomic_set(&skb_shinfo(skb)->dataref, 1);
7067 
7068 	return 0;
7069 }
7070 
7071 static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp);
7072 
7073 /* carve out the first eat bytes from skb's frag_list. May recurse into
7074  * pskb_carve()
7075  */
7076 static int pskb_carve_frag_list(struct skb_shared_info *shinfo, int eat,
7077 				gfp_t gfp_mask)
7078 {
7079 	struct sk_buff *list = shinfo->frag_list;
7080 	struct sk_buff *clone = NULL;
7081 	struct sk_buff *insp = NULL;
7082 
7083 	do {
7084 		if (!list) {
7085 			pr_err("Not enough bytes to eat. Want %d\n", eat);
7086 			return -EFAULT;
7087 		}
7088 		if (list->len <= eat) {
7089 			/* Eaten as whole. */
7090 			eat -= list->len;
7091 			list = list->next;
7092 			insp = list;
7093 		} else {
7094 			/* Eaten partially. */
7095 			if (skb_shared(list)) {
7096 				clone = skb_clone(list, gfp_mask);
7097 				if (!clone)
7098 					return -ENOMEM;
7099 				insp = list->next;
7100 				list = clone;
7101 			} else {
7102 				/* This may be pulled without problems. */
7103 				insp = list;
7104 			}
7105 			if (pskb_carve(list, eat, gfp_mask) < 0) {
7106 				kfree_skb(clone);
7107 				return -ENOMEM;
7108 			}
7109 			break;
7110 		}
7111 	} while (eat);
7112 
7113 	/* Free pulled out fragments. */
7114 	while ((list = shinfo->frag_list) != insp) {
7115 		shinfo->frag_list = list->next;
7116 		consume_skb(list);
7117 	}
7118 	/* And insert new clone at head. */
7119 	if (clone) {
7120 		clone->next = list;
7121 		shinfo->frag_list = clone;
7122 	}
7123 	return 0;
7124 }
7125 
7126 /* carve off first len bytes from skb. Split line (off) is in the
7127  * non-linear part of skb
7128  */
7129 static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off,
7130 				       int pos, gfp_t gfp_mask)
7131 {
7132 	int i, k = 0;
7133 	unsigned int size = skb_end_offset(skb);
7134 	u8 *data;
7135 	const int nfrags = skb_shinfo(skb)->nr_frags;
7136 	struct skb_shared_info *shinfo;
7137 
7138 	if (skb_pfmemalloc(skb))
7139 		gfp_mask |= __GFP_MEMALLOC;
7140 
7141 	data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL);
7142 	if (!data)
7143 		return -ENOMEM;
7144 	size = SKB_WITH_OVERHEAD(size);
7145 
7146 	/* Remove SKBFL_MANAGED_FRAG_REFS instead of trying to honour it
7147 	 * while refcounting frags below.
7148 	 */
7149 	skb_zcopy_downgrade_managed(skb);
7150 
7151 	memcpy((struct skb_shared_info *)(data + size),
7152 	       skb_shinfo(skb), offsetof(struct skb_shared_info, frags[0]));
7153 	if (skb_orphan_frags(skb, gfp_mask)) {
7154 		skb_kfree_head(data);
7155 		return -ENOMEM;
7156 	}
7157 	shinfo = (struct skb_shared_info *)(data + size);
7158 	for (i = 0; i < nfrags; i++) {
7159 		int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]);
7160 
7161 		if (pos + fsize > off) {
7162 			shinfo->frags[k] = skb_shinfo(skb)->frags[i];
7163 
7164 			if (pos < off) {
7165 				/* Split frag.
7166 				 * We have two variants in this case:
7167 				 * 1. Move all the frag to the second
7168 				 *    part, if it is possible. F.e.
7169 				 *    this approach is mandatory for TUX,
7170 				 *    where splitting is expensive.
7171 				 * 2. Split is accurately. We make this.
7172 				 */
7173 				skb_frag_off_add(&shinfo->frags[0], off - pos);
7174 				skb_frag_size_sub(&shinfo->frags[0], off - pos);
7175 			}
7176 			skb_frag_ref(skb, i);
7177 			k++;
7178 		}
7179 		pos += fsize;
7180 	}
7181 	shinfo->nr_frags = k;
7182 	if (skb_has_frag_list(skb))
7183 		skb_clone_fraglist(skb);
7184 
7185 	/* split line is in frag list */
7186 	if (k == 0 && pskb_carve_frag_list(shinfo, off - pos, gfp_mask)) {
7187 		/* skb_frag_unref() is not needed here as shinfo->nr_frags = 0. */
7188 		if (skb_has_frag_list(skb))
7189 			kfree_skb_list(skb_shinfo(skb)->frag_list);
7190 		skb_kfree_head(data);
7191 		return -ENOMEM;
7192 	}
7193 	if (skb_zcopy(skb))
7194 		net_zcopy_get(skb_uarg(skb));
7195 	skb_release_data(skb, SKB_CONSUMED);
7196 
7197 	skb->head = data;
7198 	skb->head_frag = 0;
7199 	skb->data = data;
7200 	skb_set_end_offset(skb, size);
7201 	skb_reset_tail_pointer(skb);
7202 	skb_carve_reset_headers(skb);
7203 	skb->cloned   = 0;
7204 	skb->hdr_len  = 0;
7205 	skb->nohdr    = 0;
7206 	skb->len -= off;
7207 	skb->data_len = skb->len;
7208 	atomic_set(&skb_shinfo(skb)->dataref, 1);
7209 	return 0;
7210 }
7211 
7212 /* remove len bytes from the beginning of the skb */
7213 static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp)
7214 {
7215 	int headlen = skb_headlen(skb);
7216 
7217 	if (len < headlen)
7218 		return pskb_carve_inside_header(skb, len, headlen, gfp);
7219 	else
7220 		return pskb_carve_inside_nonlinear(skb, len, headlen, gfp);
7221 }
7222 
7223 /* Extract to_copy bytes starting at off from skb, and return this in
7224  * a new skb
7225  */
7226 struct sk_buff *pskb_extract(struct sk_buff *skb, int off,
7227 			     int to_copy, gfp_t gfp)
7228 {
7229 	struct sk_buff  *clone = skb_clone(skb, gfp);
7230 
7231 	if (!clone)
7232 		return NULL;
7233 
7234 	if (pskb_carve(clone, off, gfp) < 0 ||
7235 	    pskb_trim(clone, to_copy)) {
7236 		kfree_skb(clone);
7237 		return NULL;
7238 	}
7239 	return clone;
7240 }
7241 EXPORT_SYMBOL(pskb_extract);
7242 
7243 /**
7244  * skb_condense - try to get rid of fragments/frag_list if possible
7245  * @skb: buffer
7246  *
7247  * Can be used to save memory before skb is added to a busy queue.
7248  * If packet has bytes in frags and enough tail room in skb->head,
7249  * pull all of them, so that we can free the frags right now and adjust
7250  * truesize.
7251  * Notes:
7252  *	We do not reallocate skb->head thus can not fail.
7253  *	Caller must re-evaluate skb->truesize if needed.
7254  */
7255 void skb_condense(struct sk_buff *skb)
7256 {
7257 	if (skb->data_len) {
7258 		if (skb->data_len > skb->end - skb->tail ||
7259 		    skb_cloned(skb) || !skb_frags_readable(skb))
7260 			return;
7261 
7262 		/* Nice, we can free page frag(s) right now */
7263 		__pskb_pull_tail(skb, skb->data_len);
7264 	}
7265 	/* At this point, skb->truesize might be over estimated,
7266 	 * because skb had a fragment, and fragments do not tell
7267 	 * their truesize.
7268 	 * When we pulled its content into skb->head, fragment
7269 	 * was freed, but __pskb_pull_tail() could not possibly
7270 	 * adjust skb->truesize, not knowing the frag truesize.
7271 	 */
7272 	skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
7273 }
7274 EXPORT_SYMBOL(skb_condense);
7275 
7276 #ifdef CONFIG_SKB_EXTENSIONS
7277 static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id)
7278 {
7279 	return (void *)ext + (ext->offset[id] * SKB_EXT_ALIGN_VALUE);
7280 }
7281 
7282 /**
7283  * __skb_ext_alloc - allocate a new skb extensions storage
7284  *
7285  * @flags: See kmalloc().
7286  *
7287  * Returns the newly allocated pointer. The pointer can later attached to a
7288  * skb via __skb_ext_set().
7289  * Note: caller must handle the skb_ext as an opaque data.
7290  */
7291 struct skb_ext *__skb_ext_alloc(gfp_t flags)
7292 {
7293 	struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, flags);
7294 
7295 	if (new) {
7296 		memset(new->offset, 0, sizeof(new->offset));
7297 		refcount_set(&new->refcnt, 1);
7298 	}
7299 
7300 	return new;
7301 }
7302 
7303 static struct skb_ext *skb_ext_maybe_cow(struct skb_ext *old,
7304 					 unsigned int old_active)
7305 {
7306 	struct skb_ext *new;
7307 
7308 	if (refcount_read(&old->refcnt) == 1)
7309 		return old;
7310 
7311 	new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC);
7312 	if (!new)
7313 		return NULL;
7314 
7315 	memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE);
7316 	refcount_set(&new->refcnt, 1);
7317 
7318 #ifdef CONFIG_XFRM
7319 	if (old_active & (1 << SKB_EXT_SEC_PATH)) {
7320 		struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH);
7321 		unsigned int i;
7322 
7323 		for (i = 0; i < sp->len; i++)
7324 			xfrm_state_hold(sp->xvec[i]);
7325 	}
7326 #endif
7327 #ifdef CONFIG_MCTP_FLOWS
7328 	if (old_active & (1 << SKB_EXT_MCTP)) {
7329 		struct mctp_flow *flow = skb_ext_get_ptr(old, SKB_EXT_MCTP);
7330 
7331 		if (flow->key)
7332 			refcount_inc(&flow->key->refs);
7333 	}
7334 #endif
7335 	__skb_ext_put(old);
7336 	return new;
7337 }
7338 
7339 /**
7340  * __skb_ext_set - attach the specified extension storage to this skb
7341  * @skb: buffer
7342  * @id: extension id
7343  * @ext: extension storage previously allocated via __skb_ext_alloc()
7344  *
7345  * Existing extensions, if any, are cleared.
7346  *
7347  * Returns the pointer to the extension.
7348  */
7349 void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id,
7350 		    struct skb_ext *ext)
7351 {
7352 	unsigned int newlen, newoff = SKB_EXT_CHUNKSIZEOF(*ext);
7353 
7354 	skb_ext_put(skb);
7355 	newlen = newoff + skb_ext_type_len[id];
7356 	ext->chunks = newlen;
7357 	ext->offset[id] = newoff;
7358 	skb->extensions = ext;
7359 	skb->active_extensions = 1 << id;
7360 	return skb_ext_get_ptr(ext, id);
7361 }
7362 EXPORT_SYMBOL_NS_GPL(__skb_ext_set, "NETDEV_INTERNAL");
7363 
7364 /**
7365  * skb_ext_add - allocate space for given extension, COW if needed
7366  * @skb: buffer
7367  * @id: extension to allocate space for
7368  *
7369  * Allocates enough space for the given extension.
7370  * If the extension is already present, a pointer to that extension
7371  * is returned.
7372  *
7373  * If the skb was cloned, COW applies and the returned memory can be
7374  * modified without changing the extension space of clones buffers.
7375  *
7376  * Returns pointer to the extension or NULL on allocation failure.
7377  */
7378 void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id)
7379 {
7380 	struct skb_ext *new, *old = NULL;
7381 	unsigned int newlen, newoff;
7382 
7383 	if (skb->active_extensions) {
7384 		old = skb->extensions;
7385 
7386 		new = skb_ext_maybe_cow(old, skb->active_extensions);
7387 		if (!new)
7388 			return NULL;
7389 
7390 		if (__skb_ext_exist(new, id))
7391 			goto set_active;
7392 
7393 		newoff = new->chunks;
7394 	} else {
7395 		newoff = SKB_EXT_CHUNKSIZEOF(*new);
7396 
7397 		new = __skb_ext_alloc(GFP_ATOMIC);
7398 		if (!new)
7399 			return NULL;
7400 	}
7401 
7402 	newlen = newoff + skb_ext_type_len[id];
7403 	new->chunks = newlen;
7404 	new->offset[id] = newoff;
7405 set_active:
7406 	skb->slow_gro = 1;
7407 	skb->extensions = new;
7408 	skb->active_extensions |= 1 << id;
7409 	return skb_ext_get_ptr(new, id);
7410 }
7411 EXPORT_SYMBOL(skb_ext_add);
7412 
7413 #ifdef CONFIG_XFRM
7414 static void skb_ext_put_sp(struct sec_path *sp)
7415 {
7416 	unsigned int i;
7417 
7418 	if (!sp->len)
7419 		return;
7420 
7421 	for (i = 0; i < sp->len; i++)
7422 		xfrm_state_put(sp->xvec[i]);
7423 	sp->len = 0;
7424 }
7425 #endif
7426 
7427 #ifdef CONFIG_MCTP_FLOWS
7428 static void skb_ext_put_mctp(struct mctp_flow *flow)
7429 {
7430 	if (!flow->key)
7431 		return;
7432 
7433 	mctp_key_unref(flow->key);
7434 	flow->key = NULL;
7435 }
7436 #endif
7437 
7438 void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
7439 {
7440 	struct skb_ext *ext = skb->extensions;
7441 
7442 	skb->active_extensions &= ~(1 << id);
7443 	if (skb->active_extensions == 0) {
7444 		skb->extensions = NULL;
7445 		__skb_ext_put(ext);
7446 		return;
7447 	}
7448 
7449 	if (refcount_read(&ext->refcnt) > 1)
7450 		return;
7451 
7452 #ifdef CONFIG_XFRM
7453 	if (id == SKB_EXT_SEC_PATH)
7454 		skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH));
7455 #endif
7456 #ifdef CONFIG_MCTP_FLOWS
7457 	if (id == SKB_EXT_MCTP)
7458 		skb_ext_put_mctp(skb_ext_get_ptr(ext, SKB_EXT_MCTP));
7459 #endif
7460 }
7461 EXPORT_SYMBOL(__skb_ext_del);
7462 
7463 void __skb_ext_put(struct skb_ext *ext)
7464 {
7465 	/* If this is last clone, nothing can increment
7466 	 * it after check passes.  Avoids one atomic op.
7467 	 */
7468 	if (refcount_read(&ext->refcnt) == 1)
7469 		goto free_now;
7470 
7471 	if (!refcount_dec_and_test(&ext->refcnt))
7472 		return;
7473 free_now:
7474 #ifdef CONFIG_XFRM
7475 	if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH))
7476 		skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH));
7477 #endif
7478 #ifdef CONFIG_MCTP_FLOWS
7479 	if (__skb_ext_exist(ext, SKB_EXT_MCTP))
7480 		skb_ext_put_mctp(skb_ext_get_ptr(ext, SKB_EXT_MCTP));
7481 #endif
7482 
7483 	kmem_cache_free(skbuff_ext_cache, ext);
7484 }
7485 EXPORT_SYMBOL(__skb_ext_put);
7486 #endif /* CONFIG_SKB_EXTENSIONS */
7487 
7488 static void kfree_skb_napi_cache(struct sk_buff *skb)
7489 {
7490 	/* if SKB is a clone, don't handle this case */
7491 	if (skb->fclone != SKB_FCLONE_UNAVAILABLE) {
7492 		__kfree_skb(skb);
7493 		return;
7494 	}
7495 
7496 	local_bh_disable();
7497 	__napi_kfree_skb(skb, SKB_CONSUMED);
7498 	local_bh_enable();
7499 }
7500 
7501 DEFINE_STATIC_KEY_FALSE(skb_defer_disable_key);
7502 
7503 /**
7504  * skb_attempt_defer_free - queue skb for remote freeing
7505  * @skb: buffer
7506  *
7507  * Put @skb in a per-cpu list, using the cpu which
7508  * allocated the skb/pages to reduce false sharing
7509  * and memory zone spinlock contention.
7510  */
7511 void skb_attempt_defer_free(struct sk_buff *skb)
7512 {
7513 	struct skb_defer_node *sdn;
7514 	unsigned long defer_count;
7515 	unsigned int defer_max;
7516 	int cpu, my_cpu;
7517 	bool kick;
7518 
7519 	if (static_branch_unlikely(&skb_defer_disable_key))
7520 		goto nodefer;
7521 
7522 	/* zero copy notifications should not be delayed. */
7523 	if (skb_zcopy(skb))
7524 		goto nodefer;
7525 
7526 	cpu = skb->alloc_cpu;
7527 	my_cpu = raw_smp_processor_id();
7528 	if (cpu == my_cpu ||
7529 	    WARN_ON_ONCE(cpu >= nr_cpu_ids) ||
7530 	    !cpu_online(cpu)) {
7531 nodefer:	kfree_skb_napi_cache(skb);
7532 		return;
7533 	}
7534 
7535 	DEBUG_NET_WARN_ON_ONCE(skb_dst(skb));
7536 	DEBUG_NET_WARN_ON_ONCE(skb->destructor);
7537 	DEBUG_NET_WARN_ON_ONCE(skb_nfct(skb));
7538 
7539 	sdn = per_cpu_ptr(net_hotdata.skb_defer_nodes, cpu) + cpu_to_node(my_cpu);
7540 
7541 	defer_max = READ_ONCE(net_hotdata.sysctl_skb_defer_max);
7542 	defer_count = atomic_long_inc_return(&sdn->defer_count);
7543 
7544 	if (defer_count > defer_max) {
7545 		/* Undo the reservation. An llist_add() does not release one --
7546 		 * the skb stays charged until a flush subtracts the batch that
7547 		 * contains it -- so a rejected free must undo its own.
7548 		 */
7549 		atomic_long_dec(&sdn->defer_count);
7550 		goto nodefer;
7551 	}
7552 
7553 	llist_add(&skb->ll_node, &sdn->defer_list);
7554 
7555 	if (unlikely(!cpu_online(cpu) || my_cpu != raw_smp_processor_id())) {
7556 		skb_defer_node_flush(sdn);
7557 		return;
7558 	}
7559 
7560 	/* Half capacity, plus the skb that fills the queue. Equality alone can
7561 	 * be consumed early -- if the producer holding the kick value publishes
7562 	 * first, its purge drains only that skb and the reservations published
7563 	 * after it match nothing -- which would leave a full queue with nothing
7564 	 * pending.
7565 	 */
7566 	kick = (defer_count - 1) == (defer_max >> 1) || defer_count == defer_max;
7567 
7568 	/* Make sure to trigger NET_RX_SOFTIRQ on the remote CPU
7569 	 * if we are unlucky enough (this seems very unlikely).
7570 	 */
7571 	if (unlikely(kick))
7572 		kick_defer_list_purge(cpu);
7573 }
7574 
7575 static void skb_splice_csum_page(struct sk_buff *skb, struct page *page,
7576 				 size_t offset, size_t len,
7577 				 unsigned int csum_offset)
7578 {
7579 	const char *kaddr;
7580 	__wsum csum;
7581 
7582 	kaddr = kmap_local_page(page);
7583 	csum = csum_partial(kaddr + offset, len, 0);
7584 	kunmap_local(kaddr);
7585 	skb->csum = csum_block_add(skb->csum, csum, csum_offset);
7586 }
7587 
7588 /**
7589  * skb_splice_from_iter - Splice (or copy) pages to skbuff
7590  * @skb: The buffer to add pages to
7591  * @iter: Iterator representing the pages to be added
7592  * @maxsize: Maximum amount of pages to be added
7593  *
7594  * This is a common helper function for supporting MSG_SPLICE_PAGES.  It
7595  * extracts pages from an iterator and adds them to the socket buffer if
7596  * possible, copying them to fragments if not possible (such as if they're slab
7597  * pages).
7598  *
7599  * Returns the amount of data spliced/copied or -EMSGSIZE if there's
7600  * insufficient space in the buffer to transfer anything.
7601  */
7602 ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter,
7603 			     ssize_t maxsize)
7604 {
7605 	size_t frag_limit = READ_ONCE(net_hotdata.sysctl_max_skb_frags);
7606 	struct page *pages[8], **ppages = pages;
7607 	ssize_t spliced = 0, ret = 0;
7608 	unsigned int i;
7609 
7610 	while (iter->count > 0) {
7611 		ssize_t space, nr, len;
7612 		size_t off;
7613 
7614 		ret = -EMSGSIZE;
7615 		space = frag_limit - skb_shinfo(skb)->nr_frags;
7616 		if (space < 0)
7617 			break;
7618 
7619 		/* We might be able to coalesce without increasing nr_frags */
7620 		nr = clamp_t(size_t, space, 1, ARRAY_SIZE(pages));
7621 
7622 		len = iov_iter_extract_pages(iter, &ppages, maxsize, nr, 0, &off);
7623 		if (len <= 0) {
7624 			ret = len ?: -EIO;
7625 			break;
7626 		}
7627 
7628 		i = 0;
7629 		do {
7630 			struct page *page = pages[i++];
7631 			size_t part = min_t(size_t, PAGE_SIZE - off, len);
7632 
7633 			ret = -EIO;
7634 			if (WARN_ON_ONCE(!sendpage_ok(page)))
7635 				goto out;
7636 
7637 			ret = skb_append_pagefrags(skb, page, off, part,
7638 						   frag_limit);
7639 			if (ret < 0) {
7640 				iov_iter_revert(iter, len);
7641 				goto out;
7642 			}
7643 
7644 			if (skb->ip_summed == CHECKSUM_NONE)
7645 				skb_splice_csum_page(skb, page, off, part,
7646 						     skb->len + spliced);
7647 
7648 			off = 0;
7649 			spliced += part;
7650 			maxsize -= part;
7651 			len -= part;
7652 		} while (len > 0);
7653 
7654 		if (maxsize <= 0)
7655 			break;
7656 	}
7657 
7658 out:
7659 	skb_len_add(skb, spliced);
7660 	return spliced ?: ret;
7661 }
7662 EXPORT_SYMBOL(skb_splice_from_iter);
7663 
7664 static __always_inline
7665 size_t memcpy_from_iter_csum(void *iter_from, size_t progress,
7666 			     size_t len, void *to, void *priv2)
7667 {
7668 	__wsum *csum = priv2;
7669 	__wsum next = csum_partial_copy_nocheck(iter_from, to + progress, len);
7670 
7671 	*csum = csum_block_add(*csum, next, progress);
7672 	return 0;
7673 }
7674 
7675 static __always_inline
7676 size_t copy_from_user_iter_csum(void __user *iter_from, size_t progress,
7677 				size_t len, void *to, void *priv2)
7678 {
7679 	__wsum next, *csum = priv2;
7680 
7681 	next = csum_and_copy_from_user(iter_from, to + progress, len);
7682 	*csum = csum_block_add(*csum, next, progress);
7683 	return next ? 0 : len;
7684 }
7685 
7686 bool csum_and_copy_from_iter_full(void *addr, size_t bytes,
7687 				  __wsum *csum, struct iov_iter *i)
7688 {
7689 	size_t copied;
7690 
7691 	if (WARN_ON_ONCE(!i->data_source))
7692 		return false;
7693 	copied = iterate_and_advance2(i, bytes, addr, csum,
7694 				      copy_from_user_iter_csum,
7695 				      memcpy_from_iter_csum);
7696 	if (likely(copied == bytes))
7697 		return true;
7698 	iov_iter_revert(i, copied);
7699 	return false;
7700 }
7701 EXPORT_SYMBOL(csum_and_copy_from_iter_full);
7702 
7703 /**
7704  * net_iov_inc_ref - acquire a non-page_pool reference on a net_iov
7705  * @niov: net_iov to reference
7706  *
7707  * Acquires a non-pp backing reference (binding->ref) on @niov regardless of
7708  * page_pool membership, warning if @niov does not support non-pp references.
7709  * Counterpart to net_iov_dec_ref().
7710  */
7711 void net_iov_inc_ref(struct net_iov *niov)
7712 {
7713 	if (net_is_devmem_iov(niov))
7714 		net_devmem_net_iov_inc_ref(niov);
7715 	else
7716 		DEBUG_NET_WARN_ON_ONCE(true);
7717 }
7718 EXPORT_SYMBOL(net_iov_inc_ref);
7719 
7720 /**
7721  * net_iov_dec_ref - release a non-page_pool reference on a net_iov
7722  * @niov: net_iov to unreference
7723  *
7724  * Drops a non-pp backing reference (binding->ref) on @niov regardless of
7725  * page_pool membership, warning if @niov does not support non-pp references.
7726  * Counterpart to net_iov_inc_ref().
7727  */
7728 void net_iov_dec_ref(struct net_iov *niov)
7729 {
7730 	if (net_is_devmem_iov(niov))
7731 		net_devmem_net_iov_dec_ref(niov);
7732 	else
7733 		DEBUG_NET_WARN_ON_ONCE(true);
7734 }
7735 EXPORT_SYMBOL(net_iov_dec_ref);
7736 
7737 struct vlan_type_depth __vlan_get_protocol_offset(const struct sk_buff *skb,
7738 						  __be16 type,
7739 						  int mac_offset)
7740 {
7741 	unsigned int vlan_depth = skb->mac_len, parse_depth = VLAN_MAX_DEPTH;
7742 
7743 	/* if type is 802.1Q/AD then the header should already be
7744 	 * present at mac_len - VLAN_HLEN (if mac_len > 0), or at
7745 	 * ETH_HLEN otherwise
7746 	 */
7747 	if (vlan_depth) {
7748 		if (WARN_ON_ONCE(vlan_depth < VLAN_HLEN))
7749 			return (struct vlan_type_depth) { 0 };
7750 		vlan_depth -= VLAN_HLEN;
7751 	} else {
7752 		vlan_depth = ETH_HLEN;
7753 	}
7754 	do {
7755 		struct vlan_hdr vhdr, *vh;
7756 
7757 		vh = skb_header_pointer(skb, mac_offset + vlan_depth,
7758 					sizeof(vhdr), &vhdr);
7759 		if (unlikely(!vh || !--parse_depth))
7760 			return (struct vlan_type_depth) { 0 };
7761 
7762 		type = vh->h_vlan_encapsulated_proto;
7763 		vlan_depth += VLAN_HLEN;
7764 	} while (eth_type_vlan(type));
7765 
7766 	return (struct vlan_type_depth) {
7767 		.type = type,
7768 		.depth = vlan_depth
7769 	};
7770 }
7771 EXPORT_SYMBOL(__vlan_get_protocol_offset);
7772