xref: /linux/include/linux/skbuff.h (revision 9410645520e9b820069761f3450ef6661418e279)
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3  *	Definitions for the 'struct sk_buff' memory handlers.
4  *
5  *	Authors:
6  *		Alan Cox, <gw4pts@gw4pts.ampr.org>
7  *		Florian La Roche, <rzsfl@rz.uni-sb.de>
8  */
9 
10 #ifndef _LINUX_SKBUFF_H
11 #define _LINUX_SKBUFF_H
12 
13 #include <linux/kernel.h>
14 #include <linux/compiler.h>
15 #include <linux/time.h>
16 #include <linux/bug.h>
17 #include <linux/bvec.h>
18 #include <linux/cache.h>
19 #include <linux/rbtree.h>
20 #include <linux/socket.h>
21 #include <linux/refcount.h>
22 
23 #include <linux/atomic.h>
24 #include <asm/types.h>
25 #include <linux/spinlock.h>
26 #include <net/checksum.h>
27 #include <linux/rcupdate.h>
28 #include <linux/dma-mapping.h>
29 #include <linux/netdev_features.h>
30 #include <net/flow_dissector.h>
31 #include <linux/in6.h>
32 #include <linux/if_packet.h>
33 #include <linux/llist.h>
34 #include <net/flow.h>
35 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
36 #include <linux/netfilter/nf_conntrack_common.h>
37 #endif
38 #include <net/net_debug.h>
39 #include <net/dropreason-core.h>
40 #include <net/netmem.h>
41 
42 /**
43  * DOC: skb checksums
44  *
45  * The interface for checksum offload between the stack and networking drivers
46  * is as follows...
47  *
48  * IP checksum related features
49  * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
50  *
51  * Drivers advertise checksum offload capabilities in the features of a device.
52  * From the stack's point of view these are capabilities offered by the driver.
53  * A driver typically only advertises features that it is capable of offloading
54  * to its device.
55  *
56  * .. flat-table:: Checksum related device features
57  *   :widths: 1 10
58  *
59  *   * - %NETIF_F_HW_CSUM
60  *     - The driver (or its device) is able to compute one
61  *	 IP (one's complement) checksum for any combination
62  *	 of protocols or protocol layering. The checksum is
63  *	 computed and set in a packet per the CHECKSUM_PARTIAL
64  *	 interface (see below).
65  *
66  *   * - %NETIF_F_IP_CSUM
67  *     - Driver (device) is only able to checksum plain
68  *	 TCP or UDP packets over IPv4. These are specifically
69  *	 unencapsulated packets of the form IPv4|TCP or
70  *	 IPv4|UDP where the Protocol field in the IPv4 header
71  *	 is TCP or UDP. The IPv4 header may contain IP options.
72  *	 This feature cannot be set in features for a device
73  *	 with NETIF_F_HW_CSUM also set. This feature is being
74  *	 DEPRECATED (see below).
75  *
76  *   * - %NETIF_F_IPV6_CSUM
77  *     - Driver (device) is only able to checksum plain
78  *	 TCP or UDP packets over IPv6. These are specifically
79  *	 unencapsulated packets of the form IPv6|TCP or
80  *	 IPv6|UDP where the Next Header field in the IPv6
81  *	 header is either TCP or UDP. IPv6 extension headers
82  *	 are not supported with this feature. This feature
83  *	 cannot be set in features for a device with
84  *	 NETIF_F_HW_CSUM also set. This feature is being
85  *	 DEPRECATED (see below).
86  *
87  *   * - %NETIF_F_RXCSUM
88  *     - Driver (device) performs receive checksum offload.
89  *	 This flag is only used to disable the RX checksum
90  *	 feature for a device. The stack will accept receive
91  *	 checksum indication in packets received on a device
92  *	 regardless of whether NETIF_F_RXCSUM is set.
93  *
94  * Checksumming of received packets by device
95  * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
96  *
97  * Indication of checksum verification is set in &sk_buff.ip_summed.
98  * Possible values are:
99  *
100  * - %CHECKSUM_NONE
101  *
102  *   Device did not checksum this packet e.g. due to lack of capabilities.
103  *   The packet contains full (though not verified) checksum in packet but
104  *   not in skb->csum. Thus, skb->csum is undefined in this case.
105  *
106  * - %CHECKSUM_UNNECESSARY
107  *
108  *   The hardware you're dealing with doesn't calculate the full checksum
109  *   (as in %CHECKSUM_COMPLETE), but it does parse headers and verify checksums
110  *   for specific protocols. For such packets it will set %CHECKSUM_UNNECESSARY
111  *   if their checksums are okay. &sk_buff.csum is still undefined in this case
112  *   though. A driver or device must never modify the checksum field in the
113  *   packet even if checksum is verified.
114  *
115  *   %CHECKSUM_UNNECESSARY is applicable to following protocols:
116  *
117  *     - TCP: IPv6 and IPv4.
118  *     - UDP: IPv4 and IPv6. A device may apply CHECKSUM_UNNECESSARY to a
119  *       zero UDP checksum for either IPv4 or IPv6, the networking stack
120  *       may perform further validation in this case.
121  *     - GRE: only if the checksum is present in the header.
122  *     - SCTP: indicates the CRC in SCTP header has been validated.
123  *     - FCOE: indicates the CRC in FC frame has been validated.
124  *
125  *   &sk_buff.csum_level indicates the number of consecutive checksums found in
126  *   the packet minus one that have been verified as %CHECKSUM_UNNECESSARY.
127  *   For instance if a device receives an IPv6->UDP->GRE->IPv4->TCP packet
128  *   and a device is able to verify the checksums for UDP (possibly zero),
129  *   GRE (checksum flag is set) and TCP, &sk_buff.csum_level would be set to
130  *   two. If the device were only able to verify the UDP checksum and not
131  *   GRE, either because it doesn't support GRE checksum or because GRE
132  *   checksum is bad, skb->csum_level would be set to zero (TCP checksum is
133  *   not considered in this case).
134  *
135  * - %CHECKSUM_COMPLETE
136  *
137  *   This is the most generic way. The device supplied checksum of the _whole_
138  *   packet as seen by netif_rx() and fills in &sk_buff.csum. This means the
139  *   hardware doesn't need to parse L3/L4 headers to implement this.
140  *
141  *   Notes:
142  *
143  *   - Even if device supports only some protocols, but is able to produce
144  *     skb->csum, it MUST use CHECKSUM_COMPLETE, not CHECKSUM_UNNECESSARY.
145  *   - CHECKSUM_COMPLETE is not applicable to SCTP and FCoE protocols.
146  *
147  * - %CHECKSUM_PARTIAL
148  *
149  *   A checksum is set up to be offloaded to a device as described in the
150  *   output description for CHECKSUM_PARTIAL. This may occur on a packet
151  *   received directly from another Linux OS, e.g., a virtualized Linux kernel
152  *   on the same host, or it may be set in the input path in GRO or remote
153  *   checksum offload. For the purposes of checksum verification, the checksum
154  *   referred to by skb->csum_start + skb->csum_offset and any preceding
155  *   checksums in the packet are considered verified. Any checksums in the
156  *   packet that are after the checksum being offloaded are not considered to
157  *   be verified.
158  *
159  * Checksumming on transmit for non-GSO
160  * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
161  *
162  * The stack requests checksum offload in the &sk_buff.ip_summed for a packet.
163  * Values are:
164  *
165  * - %CHECKSUM_PARTIAL
166  *
167  *   The driver is required to checksum the packet as seen by hard_start_xmit()
168  *   from &sk_buff.csum_start up to the end, and to record/write the checksum at
169  *   offset &sk_buff.csum_start + &sk_buff.csum_offset.
170  *   A driver may verify that the
171  *   csum_start and csum_offset values are valid values given the length and
172  *   offset of the packet, but it should not attempt to validate that the
173  *   checksum refers to a legitimate transport layer checksum -- it is the
174  *   purview of the stack to validate that csum_start and csum_offset are set
175  *   correctly.
176  *
177  *   When the stack requests checksum offload for a packet, the driver MUST
178  *   ensure that the checksum is set correctly. A driver can either offload the
179  *   checksum calculation to the device, or call skb_checksum_help (in the case
180  *   that the device does not support offload for a particular checksum).
181  *
182  *   %NETIF_F_IP_CSUM and %NETIF_F_IPV6_CSUM are being deprecated in favor of
183  *   %NETIF_F_HW_CSUM. New devices should use %NETIF_F_HW_CSUM to indicate
184  *   checksum offload capability.
185  *   skb_csum_hwoffload_help() can be called to resolve %CHECKSUM_PARTIAL based
186  *   on network device checksumming capabilities: if a packet does not match
187  *   them, skb_checksum_help() or skb_crc32c_help() (depending on the value of
188  *   &sk_buff.csum_not_inet, see :ref:`crc`)
189  *   is called to resolve the checksum.
190  *
191  * - %CHECKSUM_NONE
192  *
193  *   The skb was already checksummed by the protocol, or a checksum is not
194  *   required.
195  *
196  * - %CHECKSUM_UNNECESSARY
197  *
198  *   This has the same meaning as CHECKSUM_NONE for checksum offload on
199  *   output.
200  *
201  * - %CHECKSUM_COMPLETE
202  *
203  *   Not used in checksum output. If a driver observes a packet with this value
204  *   set in skbuff, it should treat the packet as if %CHECKSUM_NONE were set.
205  *
206  * .. _crc:
207  *
208  * Non-IP checksum (CRC) offloads
209  * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
210  *
211  * .. flat-table::
212  *   :widths: 1 10
213  *
214  *   * - %NETIF_F_SCTP_CRC
215  *     - This feature indicates that a device is capable of
216  *	 offloading the SCTP CRC in a packet. To perform this offload the stack
217  *	 will set csum_start and csum_offset accordingly, set ip_summed to
218  *	 %CHECKSUM_PARTIAL and set csum_not_inet to 1, to provide an indication
219  *	 in the skbuff that the %CHECKSUM_PARTIAL refers to CRC32c.
220  *	 A driver that supports both IP checksum offload and SCTP CRC32c offload
221  *	 must verify which offload is configured for a packet by testing the
222  *	 value of &sk_buff.csum_not_inet; skb_crc32c_csum_help() is provided to
223  *	 resolve %CHECKSUM_PARTIAL on skbs where csum_not_inet is set to 1.
224  *
225  *   * - %NETIF_F_FCOE_CRC
226  *     - This feature indicates that a device is capable of offloading the FCOE
227  *	 CRC in a packet. To perform this offload the stack will set ip_summed
228  *	 to %CHECKSUM_PARTIAL and set csum_start and csum_offset
229  *	 accordingly. Note that there is no indication in the skbuff that the
230  *	 %CHECKSUM_PARTIAL refers to an FCOE checksum, so a driver that supports
231  *	 both IP checksum offload and FCOE CRC offload must verify which offload
232  *	 is configured for a packet, presumably by inspecting packet headers.
233  *
234  * Checksumming on output with GSO
235  * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
236  *
237  * In the case of a GSO packet (skb_is_gso() is true), checksum offload
238  * is implied by the SKB_GSO_* flags in gso_type. Most obviously, if the
239  * gso_type is %SKB_GSO_TCPV4 or %SKB_GSO_TCPV6, TCP checksum offload as
240  * part of the GSO operation is implied. If a checksum is being offloaded
241  * with GSO then ip_summed is %CHECKSUM_PARTIAL, and both csum_start and
242  * csum_offset are set to refer to the outermost checksum being offloaded
243  * (two offloaded checksums are possible with UDP encapsulation).
244  */
245 
246 /* Don't change this without changing skb_csum_unnecessary! */
247 #define CHECKSUM_NONE		0
248 #define CHECKSUM_UNNECESSARY	1
249 #define CHECKSUM_COMPLETE	2
250 #define CHECKSUM_PARTIAL	3
251 
252 /* Maximum value in skb->csum_level */
253 #define SKB_MAX_CSUM_LEVEL	3
254 
255 #define SKB_DATA_ALIGN(X)	ALIGN(X, SMP_CACHE_BYTES)
256 #define SKB_WITH_OVERHEAD(X)	\
257 	((X) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
258 
259 /* For X bytes available in skb->head, what is the minimal
260  * allocation needed, knowing struct skb_shared_info needs
261  * to be aligned.
262  */
263 #define SKB_HEAD_ALIGN(X) (SKB_DATA_ALIGN(X) + \
264 	SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
265 
266 #define SKB_MAX_ORDER(X, ORDER) \
267 	SKB_WITH_OVERHEAD((PAGE_SIZE << (ORDER)) - (X))
268 #define SKB_MAX_HEAD(X)		(SKB_MAX_ORDER((X), 0))
269 #define SKB_MAX_ALLOC		(SKB_MAX_ORDER(0, 2))
270 
271 /* return minimum truesize of one skb containing X bytes of data */
272 #define SKB_TRUESIZE(X) ((X) +						\
273 			 SKB_DATA_ALIGN(sizeof(struct sk_buff)) +	\
274 			 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
275 
276 struct ahash_request;
277 struct net_device;
278 struct scatterlist;
279 struct pipe_inode_info;
280 struct iov_iter;
281 struct napi_struct;
282 struct bpf_prog;
283 union bpf_attr;
284 struct skb_ext;
285 struct ts_config;
286 
287 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
288 struct nf_bridge_info {
289 	enum {
290 		BRNF_PROTO_UNCHANGED,
291 		BRNF_PROTO_8021Q,
292 		BRNF_PROTO_PPPOE
293 	} orig_proto:8;
294 	u8			pkt_otherhost:1;
295 	u8			in_prerouting:1;
296 	u8			bridged_dnat:1;
297 	u8			sabotage_in_done:1;
298 	__u16			frag_max_size;
299 	int			physinif;
300 
301 	/* always valid & non-NULL from FORWARD on, for physdev match */
302 	struct net_device	*physoutdev;
303 	union {
304 		/* prerouting: detect dnat in orig/reply direction */
305 		__be32          ipv4_daddr;
306 		struct in6_addr ipv6_daddr;
307 
308 		/* after prerouting + nat detected: store original source
309 		 * mac since neigh resolution overwrites it, only used while
310 		 * skb is out in neigh layer.
311 		 */
312 		char neigh_header[8];
313 	};
314 };
315 #endif
316 
317 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
318 /* Chain in tc_skb_ext will be used to share the tc chain with
319  * ovs recirc_id. It will be set to the current chain by tc
320  * and read by ovs to recirc_id.
321  */
322 struct tc_skb_ext {
323 	union {
324 		u64 act_miss_cookie;
325 		__u32 chain;
326 	};
327 	__u16 mru;
328 	__u16 zone;
329 	u8 post_ct:1;
330 	u8 post_ct_snat:1;
331 	u8 post_ct_dnat:1;
332 	u8 act_miss:1; /* Set if act_miss_cookie is used */
333 	u8 l2_miss:1; /* Set by bridge upon FDB or MDB miss */
334 };
335 #endif
336 
337 struct sk_buff_head {
338 	/* These two members must be first to match sk_buff. */
339 	struct_group_tagged(sk_buff_list, list,
340 		struct sk_buff	*next;
341 		struct sk_buff	*prev;
342 	);
343 
344 	__u32		qlen;
345 	spinlock_t	lock;
346 };
347 
348 struct sk_buff;
349 
350 #ifndef CONFIG_MAX_SKB_FRAGS
351 # define CONFIG_MAX_SKB_FRAGS 17
352 #endif
353 
354 #define MAX_SKB_FRAGS CONFIG_MAX_SKB_FRAGS
355 
356 /* Set skb_shinfo(skb)->gso_size to this in case you want skb_segment to
357  * segment using its current segmentation instead.
358  */
359 #define GSO_BY_FRAGS	0xFFFF
360 
361 typedef struct skb_frag {
362 	netmem_ref netmem;
363 	unsigned int len;
364 	unsigned int offset;
365 } skb_frag_t;
366 
367 /**
368  * skb_frag_size() - Returns the size of a skb fragment
369  * @frag: skb fragment
370  */
skb_frag_size(const skb_frag_t * frag)371 static inline unsigned int skb_frag_size(const skb_frag_t *frag)
372 {
373 	return frag->len;
374 }
375 
376 /**
377  * skb_frag_size_set() - Sets the size of a skb fragment
378  * @frag: skb fragment
379  * @size: size of fragment
380  */
skb_frag_size_set(skb_frag_t * frag,unsigned int size)381 static inline void skb_frag_size_set(skb_frag_t *frag, unsigned int size)
382 {
383 	frag->len = size;
384 }
385 
386 /**
387  * skb_frag_size_add() - Increments the size of a skb fragment by @delta
388  * @frag: skb fragment
389  * @delta: value to add
390  */
skb_frag_size_add(skb_frag_t * frag,int delta)391 static inline void skb_frag_size_add(skb_frag_t *frag, int delta)
392 {
393 	frag->len += delta;
394 }
395 
396 /**
397  * skb_frag_size_sub() - Decrements the size of a skb fragment by @delta
398  * @frag: skb fragment
399  * @delta: value to subtract
400  */
skb_frag_size_sub(skb_frag_t * frag,int delta)401 static inline void skb_frag_size_sub(skb_frag_t *frag, int delta)
402 {
403 	frag->len -= delta;
404 }
405 
406 /**
407  * skb_frag_must_loop - Test if %p is a high memory page
408  * @p: fragment's page
409  */
skb_frag_must_loop(struct page * p)410 static inline bool skb_frag_must_loop(struct page *p)
411 {
412 #if defined(CONFIG_HIGHMEM)
413 	if (IS_ENABLED(CONFIG_DEBUG_KMAP_LOCAL_FORCE_MAP) || PageHighMem(p))
414 		return true;
415 #endif
416 	return false;
417 }
418 
419 /**
420  *	skb_frag_foreach_page - loop over pages in a fragment
421  *
422  *	@f:		skb frag to operate on
423  *	@f_off:		offset from start of f->netmem
424  *	@f_len:		length from f_off to loop over
425  *	@p:		(temp var) current page
426  *	@p_off:		(temp var) offset from start of current page,
427  *	                           non-zero only on first page.
428  *	@p_len:		(temp var) length in current page,
429  *				   < PAGE_SIZE only on first and last page.
430  *	@copied:	(temp var) length so far, excluding current p_len.
431  *
432  *	A fragment can hold a compound page, in which case per-page
433  *	operations, notably kmap_atomic, must be called for each
434  *	regular page.
435  */
436 #define skb_frag_foreach_page(f, f_off, f_len, p, p_off, p_len, copied)	\
437 	for (p = skb_frag_page(f) + ((f_off) >> PAGE_SHIFT),		\
438 	     p_off = (f_off) & (PAGE_SIZE - 1),				\
439 	     p_len = skb_frag_must_loop(p) ?				\
440 	     min_t(u32, f_len, PAGE_SIZE - p_off) : f_len,		\
441 	     copied = 0;						\
442 	     copied < f_len;						\
443 	     copied += p_len, p++, p_off = 0,				\
444 	     p_len = min_t(u32, f_len - copied, PAGE_SIZE))		\
445 
446 /**
447  * struct skb_shared_hwtstamps - hardware time stamps
448  * @hwtstamp:		hardware time stamp transformed into duration
449  *			since arbitrary point in time
450  * @netdev_data:	address/cookie of network device driver used as
451  *			reference to actual hardware time stamp
452  *
453  * Software time stamps generated by ktime_get_real() are stored in
454  * skb->tstamp.
455  *
456  * hwtstamps can only be compared against other hwtstamps from
457  * the same device.
458  *
459  * This structure is attached to packets as part of the
460  * &skb_shared_info. Use skb_hwtstamps() to get a pointer.
461  */
462 struct skb_shared_hwtstamps {
463 	union {
464 		ktime_t	hwtstamp;
465 		void *netdev_data;
466 	};
467 };
468 
469 /* Definitions for tx_flags in struct skb_shared_info */
470 enum {
471 	/* generate hardware time stamp */
472 	SKBTX_HW_TSTAMP = 1 << 0,
473 
474 	/* generate software time stamp when queueing packet to NIC */
475 	SKBTX_SW_TSTAMP = 1 << 1,
476 
477 	/* device driver is going to provide hardware time stamp */
478 	SKBTX_IN_PROGRESS = 1 << 2,
479 
480 	/* generate hardware time stamp based on cycles if supported */
481 	SKBTX_HW_TSTAMP_USE_CYCLES = 1 << 3,
482 
483 	/* generate wifi status information (where possible) */
484 	SKBTX_WIFI_STATUS = 1 << 4,
485 
486 	/* determine hardware time stamp based on time or cycles */
487 	SKBTX_HW_TSTAMP_NETDEV = 1 << 5,
488 
489 	/* generate software time stamp when entering packet scheduling */
490 	SKBTX_SCHED_TSTAMP = 1 << 6,
491 };
492 
493 #define SKBTX_ANY_SW_TSTAMP	(SKBTX_SW_TSTAMP    | \
494 				 SKBTX_SCHED_TSTAMP)
495 #define SKBTX_ANY_TSTAMP	(SKBTX_HW_TSTAMP | \
496 				 SKBTX_HW_TSTAMP_USE_CYCLES | \
497 				 SKBTX_ANY_SW_TSTAMP)
498 
499 /* Definitions for flags in struct skb_shared_info */
500 enum {
501 	/* use zcopy routines */
502 	SKBFL_ZEROCOPY_ENABLE = BIT(0),
503 
504 	/* This indicates at least one fragment might be overwritten
505 	 * (as in vmsplice(), sendfile() ...)
506 	 * If we need to compute a TX checksum, we'll need to copy
507 	 * all frags to avoid possible bad checksum
508 	 */
509 	SKBFL_SHARED_FRAG = BIT(1),
510 
511 	/* segment contains only zerocopy data and should not be
512 	 * charged to the kernel memory.
513 	 */
514 	SKBFL_PURE_ZEROCOPY = BIT(2),
515 
516 	SKBFL_DONT_ORPHAN = BIT(3),
517 
518 	/* page references are managed by the ubuf_info, so it's safe to
519 	 * use frags only up until ubuf_info is released
520 	 */
521 	SKBFL_MANAGED_FRAG_REFS = BIT(4),
522 };
523 
524 #define SKBFL_ZEROCOPY_FRAG	(SKBFL_ZEROCOPY_ENABLE | SKBFL_SHARED_FRAG)
525 #define SKBFL_ALL_ZEROCOPY	(SKBFL_ZEROCOPY_FRAG | SKBFL_PURE_ZEROCOPY | \
526 				 SKBFL_DONT_ORPHAN | SKBFL_MANAGED_FRAG_REFS)
527 
528 struct ubuf_info_ops {
529 	void (*complete)(struct sk_buff *, struct ubuf_info *,
530 			 bool zerocopy_success);
531 	/* has to be compatible with skb_zcopy_set() */
532 	int (*link_skb)(struct sk_buff *skb, struct ubuf_info *uarg);
533 };
534 
535 /*
536  * The callback notifies userspace to release buffers when skb DMA is done in
537  * lower device, the skb last reference should be 0 when calling this.
538  * The zerocopy_success argument is true if zero copy transmit occurred,
539  * false on data copy or out of memory error caused by data copy attempt.
540  * The ctx field is used to track device context.
541  * The desc field is used to track userspace buffer index.
542  */
543 struct ubuf_info {
544 	const struct ubuf_info_ops *ops;
545 	refcount_t refcnt;
546 	u8 flags;
547 };
548 
549 struct ubuf_info_msgzc {
550 	struct ubuf_info ubuf;
551 
552 	union {
553 		struct {
554 			unsigned long desc;
555 			void *ctx;
556 		};
557 		struct {
558 			u32 id;
559 			u16 len;
560 			u16 zerocopy:1;
561 			u32 bytelen;
562 		};
563 	};
564 
565 	struct mmpin {
566 		struct user_struct *user;
567 		unsigned int num_pg;
568 	} mmp;
569 };
570 
571 #define skb_uarg(SKB)	((struct ubuf_info *)(skb_shinfo(SKB)->destructor_arg))
572 #define uarg_to_msgzc(ubuf_ptr)	container_of((ubuf_ptr), struct ubuf_info_msgzc, \
573 					     ubuf)
574 
575 int mm_account_pinned_pages(struct mmpin *mmp, size_t size);
576 void mm_unaccount_pinned_pages(struct mmpin *mmp);
577 
578 /* Preserve some data across TX submission and completion.
579  *
580  * Note, this state is stored in the driver. Extending the layout
581  * might need some special care.
582  */
583 struct xsk_tx_metadata_compl {
584 	__u64 *tx_timestamp;
585 };
586 
587 /* This data is invariant across clones and lives at
588  * the end of the header data, ie. at skb->end.
589  */
590 struct skb_shared_info {
591 	__u8		flags;
592 	__u8		meta_len;
593 	__u8		nr_frags;
594 	__u8		tx_flags;
595 	unsigned short	gso_size;
596 	/* Warning: this field is not always filled in (UFO)! */
597 	unsigned short	gso_segs;
598 	struct sk_buff	*frag_list;
599 	union {
600 		struct skb_shared_hwtstamps hwtstamps;
601 		struct xsk_tx_metadata_compl xsk_meta;
602 	};
603 	unsigned int	gso_type;
604 	u32		tskey;
605 
606 	/*
607 	 * Warning : all fields before dataref are cleared in __alloc_skb()
608 	 */
609 	atomic_t	dataref;
610 	unsigned int	xdp_frags_size;
611 
612 	/* Intermediate layers must ensure that destructor_arg
613 	 * remains valid until skb destructor */
614 	void *		destructor_arg;
615 
616 	/* must be last field, see pskb_expand_head() */
617 	skb_frag_t	frags[MAX_SKB_FRAGS];
618 };
619 
620 /**
621  * DOC: dataref and headerless skbs
622  *
623  * Transport layers send out clones of payload skbs they hold for
624  * retransmissions. To allow lower layers of the stack to prepend their headers
625  * we split &skb_shared_info.dataref into two halves.
626  * The lower 16 bits count the overall number of references.
627  * The higher 16 bits indicate how many of the references are payload-only.
628  * skb_header_cloned() checks if skb is allowed to add / write the headers.
629  *
630  * The creator of the skb (e.g. TCP) marks its skb as &sk_buff.nohdr
631  * (via __skb_header_release()). Any clone created from marked skb will get
632  * &sk_buff.hdr_len populated with the available headroom.
633  * If there's the only clone in existence it's able to modify the headroom
634  * at will. The sequence of calls inside the transport layer is::
635  *
636  *  <alloc skb>
637  *  skb_reserve()
638  *  __skb_header_release()
639  *  skb_clone()
640  *  // send the clone down the stack
641  *
642  * This is not a very generic construct and it depends on the transport layers
643  * doing the right thing. In practice there's usually only one payload-only skb.
644  * Having multiple payload-only skbs with different lengths of hdr_len is not
645  * possible. The payload-only skbs should never leave their owner.
646  */
647 #define SKB_DATAREF_SHIFT 16
648 #define SKB_DATAREF_MASK ((1 << SKB_DATAREF_SHIFT) - 1)
649 
650 
651 enum {
652 	SKB_FCLONE_UNAVAILABLE,	/* skb has no fclone (from head_cache) */
653 	SKB_FCLONE_ORIG,	/* orig skb (from fclone_cache) */
654 	SKB_FCLONE_CLONE,	/* companion fclone skb (from fclone_cache) */
655 };
656 
657 enum {
658 	SKB_GSO_TCPV4 = 1 << 0,
659 
660 	/* This indicates the skb is from an untrusted source. */
661 	SKB_GSO_DODGY = 1 << 1,
662 
663 	/* This indicates the tcp segment has CWR set. */
664 	SKB_GSO_TCP_ECN = 1 << 2,
665 
666 	SKB_GSO_TCP_FIXEDID = 1 << 3,
667 
668 	SKB_GSO_TCPV6 = 1 << 4,
669 
670 	SKB_GSO_FCOE = 1 << 5,
671 
672 	SKB_GSO_GRE = 1 << 6,
673 
674 	SKB_GSO_GRE_CSUM = 1 << 7,
675 
676 	SKB_GSO_IPXIP4 = 1 << 8,
677 
678 	SKB_GSO_IPXIP6 = 1 << 9,
679 
680 	SKB_GSO_UDP_TUNNEL = 1 << 10,
681 
682 	SKB_GSO_UDP_TUNNEL_CSUM = 1 << 11,
683 
684 	SKB_GSO_PARTIAL = 1 << 12,
685 
686 	SKB_GSO_TUNNEL_REMCSUM = 1 << 13,
687 
688 	SKB_GSO_SCTP = 1 << 14,
689 
690 	SKB_GSO_ESP = 1 << 15,
691 
692 	SKB_GSO_UDP = 1 << 16,
693 
694 	SKB_GSO_UDP_L4 = 1 << 17,
695 
696 	SKB_GSO_FRAGLIST = 1 << 18,
697 };
698 
699 #if BITS_PER_LONG > 32
700 #define NET_SKBUFF_DATA_USES_OFFSET 1
701 #endif
702 
703 #ifdef NET_SKBUFF_DATA_USES_OFFSET
704 typedef unsigned int sk_buff_data_t;
705 #else
706 typedef unsigned char *sk_buff_data_t;
707 #endif
708 
709 enum skb_tstamp_type {
710 	SKB_CLOCK_REALTIME,
711 	SKB_CLOCK_MONOTONIC,
712 	SKB_CLOCK_TAI,
713 	__SKB_CLOCK_MAX = SKB_CLOCK_TAI,
714 };
715 
716 /**
717  * DOC: Basic sk_buff geometry
718  *
719  * struct sk_buff itself is a metadata structure and does not hold any packet
720  * data. All the data is held in associated buffers.
721  *
722  * &sk_buff.head points to the main "head" buffer. The head buffer is divided
723  * into two parts:
724  *
725  *  - data buffer, containing headers and sometimes payload;
726  *    this is the part of the skb operated on by the common helpers
727  *    such as skb_put() or skb_pull();
728  *  - shared info (struct skb_shared_info) which holds an array of pointers
729  *    to read-only data in the (page, offset, length) format.
730  *
731  * Optionally &skb_shared_info.frag_list may point to another skb.
732  *
733  * Basic diagram may look like this::
734  *
735  *                                  ---------------
736  *                                 | sk_buff       |
737  *                                  ---------------
738  *     ,---------------------------  + head
739  *    /          ,-----------------  + data
740  *   /          /      ,-----------  + tail
741  *  |          |      |            , + end
742  *  |          |      |           |
743  *  v          v      v           v
744  *   -----------------------------------------------
745  *  | headroom | data |  tailroom | skb_shared_info |
746  *   -----------------------------------------------
747  *                                 + [page frag]
748  *                                 + [page frag]
749  *                                 + [page frag]
750  *                                 + [page frag]       ---------
751  *                                 + frag_list    --> | sk_buff |
752  *                                                     ---------
753  *
754  */
755 
756 /**
757  *	struct sk_buff - socket buffer
758  *	@next: Next buffer in list
759  *	@prev: Previous buffer in list
760  *	@tstamp: Time we arrived/left
761  *	@skb_mstamp_ns: (aka @tstamp) earliest departure time; start point
762  *		for retransmit timer
763  *	@rbnode: RB tree node, alternative to next/prev for netem/tcp
764  *	@list: queue head
765  *	@ll_node: anchor in an llist (eg socket defer_list)
766  *	@sk: Socket we are owned by
767  *	@dev: Device we arrived on/are leaving by
768  *	@dev_scratch: (aka @dev) alternate use of @dev when @dev would be %NULL
769  *	@cb: Control buffer. Free for use by every layer. Put private vars here
770  *	@_skb_refdst: destination entry (with norefcount bit)
771  *	@len: Length of actual data
772  *	@data_len: Data length
773  *	@mac_len: Length of link layer header
774  *	@hdr_len: writable header length of cloned skb
775  *	@csum: Checksum (must include start/offset pair)
776  *	@csum_start: Offset from skb->head where checksumming should start
777  *	@csum_offset: Offset from csum_start where checksum should be stored
778  *	@priority: Packet queueing priority
779  *	@ignore_df: allow local fragmentation
780  *	@cloned: Head may be cloned (check refcnt to be sure)
781  *	@ip_summed: Driver fed us an IP checksum
782  *	@nohdr: Payload reference only, must not modify header
783  *	@pkt_type: Packet class
784  *	@fclone: skbuff clone status
785  *	@ipvs_property: skbuff is owned by ipvs
786  *	@inner_protocol_type: whether the inner protocol is
787  *		ENCAP_TYPE_ETHER or ENCAP_TYPE_IPPROTO
788  *	@remcsum_offload: remote checksum offload is enabled
789  *	@offload_fwd_mark: Packet was L2-forwarded in hardware
790  *	@offload_l3_fwd_mark: Packet was L3-forwarded in hardware
791  *	@tc_skip_classify: do not classify packet. set by IFB device
792  *	@tc_at_ingress: used within tc_classify to distinguish in/egress
793  *	@redirected: packet was redirected by packet classifier
794  *	@from_ingress: packet was redirected from the ingress path
795  *	@nf_skip_egress: packet shall skip nf egress - see netfilter_netdev.h
796  *	@peeked: this packet has been seen already, so stats have been
797  *		done for it, don't do them again
798  *	@nf_trace: netfilter packet trace flag
799  *	@protocol: Packet protocol from driver
800  *	@destructor: Destruct function
801  *	@tcp_tsorted_anchor: list structure for TCP (tp->tsorted_sent_queue)
802  *	@_sk_redir: socket redirection information for skmsg
803  *	@_nfct: Associated connection, if any (with nfctinfo bits)
804  *	@skb_iif: ifindex of device we arrived on
805  *	@tc_index: Traffic control index
806  *	@hash: the packet hash
807  *	@queue_mapping: Queue mapping for multiqueue devices
808  *	@head_frag: skb was allocated from page fragments,
809  *		not allocated by kmalloc() or vmalloc().
810  *	@pfmemalloc: skbuff was allocated from PFMEMALLOC reserves
811  *	@pp_recycle: mark the packet for recycling instead of freeing (implies
812  *		page_pool support on driver)
813  *	@active_extensions: active extensions (skb_ext_id types)
814  *	@ndisc_nodetype: router type (from link layer)
815  *	@ooo_okay: allow the mapping of a socket to a queue to be changed
816  *	@l4_hash: indicate hash is a canonical 4-tuple hash over transport
817  *		ports.
818  *	@sw_hash: indicates hash was computed in software stack
819  *	@wifi_acked_valid: wifi_acked was set
820  *	@wifi_acked: whether frame was acked on wifi or not
821  *	@no_fcs:  Request NIC to treat last 4 bytes as Ethernet FCS
822  *	@encapsulation: indicates the inner headers in the skbuff are valid
823  *	@encap_hdr_csum: software checksum is needed
824  *	@csum_valid: checksum is already valid
825  *	@csum_not_inet: use CRC32c to resolve CHECKSUM_PARTIAL
826  *	@csum_complete_sw: checksum was completed by software
827  *	@csum_level: indicates the number of consecutive checksums found in
828  *		the packet minus one that have been verified as
829  *		CHECKSUM_UNNECESSARY (max 3)
830  *	@unreadable: indicates that at least 1 of the fragments in this skb is
831  *		unreadable.
832  *	@dst_pending_confirm: need to confirm neighbour
833  *	@decrypted: Decrypted SKB
834  *	@slow_gro: state present at GRO time, slower prepare step required
835  *	@tstamp_type: When set, skb->tstamp has the
836  *		delivery_time clock base of skb->tstamp.
837  *	@napi_id: id of the NAPI struct this skb came from
838  *	@sender_cpu: (aka @napi_id) source CPU in XPS
839  *	@alloc_cpu: CPU which did the skb allocation.
840  *	@secmark: security marking
841  *	@mark: Generic packet mark
842  *	@reserved_tailroom: (aka @mark) number of bytes of free space available
843  *		at the tail of an sk_buff
844  *	@vlan_all: vlan fields (proto & tci)
845  *	@vlan_proto: vlan encapsulation protocol
846  *	@vlan_tci: vlan tag control information
847  *	@inner_protocol: Protocol (encapsulation)
848  *	@inner_ipproto: (aka @inner_protocol) stores ipproto when
849  *		skb->inner_protocol_type == ENCAP_TYPE_IPPROTO;
850  *	@inner_transport_header: Inner transport layer header (encapsulation)
851  *	@inner_network_header: Network layer header (encapsulation)
852  *	@inner_mac_header: Link layer header (encapsulation)
853  *	@transport_header: Transport layer header
854  *	@network_header: Network layer header
855  *	@mac_header: Link layer header
856  *	@kcov_handle: KCOV remote handle for remote coverage collection
857  *	@tail: Tail pointer
858  *	@end: End pointer
859  *	@head: Head of buffer
860  *	@data: Data head pointer
861  *	@truesize: Buffer size
862  *	@users: User count - see {datagram,tcp}.c
863  *	@extensions: allocated extensions, valid if active_extensions is nonzero
864  */
865 
866 struct sk_buff {
867 	union {
868 		struct {
869 			/* These two members must be first to match sk_buff_head. */
870 			struct sk_buff		*next;
871 			struct sk_buff		*prev;
872 
873 			union {
874 				struct net_device	*dev;
875 				/* Some protocols might use this space to store information,
876 				 * while device pointer would be NULL.
877 				 * UDP receive path is one user.
878 				 */
879 				unsigned long		dev_scratch;
880 			};
881 		};
882 		struct rb_node		rbnode; /* used in netem, ip4 defrag, and tcp stack */
883 		struct list_head	list;
884 		struct llist_node	ll_node;
885 	};
886 
887 	struct sock		*sk;
888 
889 	union {
890 		ktime_t		tstamp;
891 		u64		skb_mstamp_ns; /* earliest departure time */
892 	};
893 	/*
894 	 * This is the control buffer. It is free to use for every
895 	 * layer. Please put your private variables there. If you
896 	 * want to keep them across layers you have to do a skb_clone()
897 	 * first. This is owned by whoever has the skb queued ATM.
898 	 */
899 	char			cb[48] __aligned(8);
900 
901 	union {
902 		struct {
903 			unsigned long	_skb_refdst;
904 			void		(*destructor)(struct sk_buff *skb);
905 		};
906 		struct list_head	tcp_tsorted_anchor;
907 #ifdef CONFIG_NET_SOCK_MSG
908 		unsigned long		_sk_redir;
909 #endif
910 	};
911 
912 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
913 	unsigned long		 _nfct;
914 #endif
915 	unsigned int		len,
916 				data_len;
917 	__u16			mac_len,
918 				hdr_len;
919 
920 	/* Following fields are _not_ copied in __copy_skb_header()
921 	 * Note that queue_mapping is here mostly to fill a hole.
922 	 */
923 	__u16			queue_mapping;
924 
925 /* if you move cloned around you also must adapt those constants */
926 #ifdef __BIG_ENDIAN_BITFIELD
927 #define CLONED_MASK	(1 << 7)
928 #else
929 #define CLONED_MASK	1
930 #endif
931 #define CLONED_OFFSET		offsetof(struct sk_buff, __cloned_offset)
932 
933 	/* private: */
934 	__u8			__cloned_offset[0];
935 	/* public: */
936 	__u8			cloned:1,
937 				nohdr:1,
938 				fclone:2,
939 				peeked:1,
940 				head_frag:1,
941 				pfmemalloc:1,
942 				pp_recycle:1; /* page_pool recycle indicator */
943 #ifdef CONFIG_SKB_EXTENSIONS
944 	__u8			active_extensions;
945 #endif
946 
947 	/* Fields enclosed in headers group are copied
948 	 * using a single memcpy() in __copy_skb_header()
949 	 */
950 	struct_group(headers,
951 
952 	/* private: */
953 	__u8			__pkt_type_offset[0];
954 	/* public: */
955 	__u8			pkt_type:3; /* see PKT_TYPE_MAX */
956 	__u8			ignore_df:1;
957 	__u8			dst_pending_confirm:1;
958 	__u8			ip_summed:2;
959 	__u8			ooo_okay:1;
960 
961 	/* private: */
962 	__u8			__mono_tc_offset[0];
963 	/* public: */
964 	__u8			tstamp_type:2;	/* See skb_tstamp_type */
965 #ifdef CONFIG_NET_XGRESS
966 	__u8			tc_at_ingress:1;	/* See TC_AT_INGRESS_MASK */
967 	__u8			tc_skip_classify:1;
968 #endif
969 	__u8			remcsum_offload:1;
970 	__u8			csum_complete_sw:1;
971 	__u8			csum_level:2;
972 	__u8			inner_protocol_type:1;
973 
974 	__u8			l4_hash:1;
975 	__u8			sw_hash:1;
976 #ifdef CONFIG_WIRELESS
977 	__u8			wifi_acked_valid:1;
978 	__u8			wifi_acked:1;
979 #endif
980 	__u8			no_fcs:1;
981 	/* Indicates the inner headers are valid in the skbuff. */
982 	__u8			encapsulation:1;
983 	__u8			encap_hdr_csum:1;
984 	__u8			csum_valid:1;
985 #ifdef CONFIG_IPV6_NDISC_NODETYPE
986 	__u8			ndisc_nodetype:2;
987 #endif
988 
989 #if IS_ENABLED(CONFIG_IP_VS)
990 	__u8			ipvs_property:1;
991 #endif
992 #if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || IS_ENABLED(CONFIG_NF_TABLES)
993 	__u8			nf_trace:1;
994 #endif
995 #ifdef CONFIG_NET_SWITCHDEV
996 	__u8			offload_fwd_mark:1;
997 	__u8			offload_l3_fwd_mark:1;
998 #endif
999 	__u8			redirected:1;
1000 #ifdef CONFIG_NET_REDIRECT
1001 	__u8			from_ingress:1;
1002 #endif
1003 #ifdef CONFIG_NETFILTER_SKIP_EGRESS
1004 	__u8			nf_skip_egress:1;
1005 #endif
1006 #ifdef CONFIG_SKB_DECRYPTED
1007 	__u8			decrypted:1;
1008 #endif
1009 	__u8			slow_gro:1;
1010 #if IS_ENABLED(CONFIG_IP_SCTP)
1011 	__u8			csum_not_inet:1;
1012 #endif
1013 	__u8			unreadable:1;
1014 #if defined(CONFIG_NET_SCHED) || defined(CONFIG_NET_XGRESS)
1015 	__u16			tc_index;	/* traffic control index */
1016 #endif
1017 
1018 	u16			alloc_cpu;
1019 
1020 	union {
1021 		__wsum		csum;
1022 		struct {
1023 			__u16	csum_start;
1024 			__u16	csum_offset;
1025 		};
1026 	};
1027 	__u32			priority;
1028 	int			skb_iif;
1029 	__u32			hash;
1030 	union {
1031 		u32		vlan_all;
1032 		struct {
1033 			__be16	vlan_proto;
1034 			__u16	vlan_tci;
1035 		};
1036 	};
1037 #if defined(CONFIG_NET_RX_BUSY_POLL) || defined(CONFIG_XPS)
1038 	union {
1039 		unsigned int	napi_id;
1040 		unsigned int	sender_cpu;
1041 	};
1042 #endif
1043 #ifdef CONFIG_NETWORK_SECMARK
1044 	__u32		secmark;
1045 #endif
1046 
1047 	union {
1048 		__u32		mark;
1049 		__u32		reserved_tailroom;
1050 	};
1051 
1052 	union {
1053 		__be16		inner_protocol;
1054 		__u8		inner_ipproto;
1055 	};
1056 
1057 	__u16			inner_transport_header;
1058 	__u16			inner_network_header;
1059 	__u16			inner_mac_header;
1060 
1061 	__be16			protocol;
1062 	__u16			transport_header;
1063 	__u16			network_header;
1064 	__u16			mac_header;
1065 
1066 #ifdef CONFIG_KCOV
1067 	u64			kcov_handle;
1068 #endif
1069 
1070 	); /* end headers group */
1071 
1072 	/* These elements must be at the end, see alloc_skb() for details.  */
1073 	sk_buff_data_t		tail;
1074 	sk_buff_data_t		end;
1075 	unsigned char		*head,
1076 				*data;
1077 	unsigned int		truesize;
1078 	refcount_t		users;
1079 
1080 #ifdef CONFIG_SKB_EXTENSIONS
1081 	/* only usable after checking ->active_extensions != 0 */
1082 	struct skb_ext		*extensions;
1083 #endif
1084 };
1085 
1086 /* if you move pkt_type around you also must adapt those constants */
1087 #ifdef __BIG_ENDIAN_BITFIELD
1088 #define PKT_TYPE_MAX	(7 << 5)
1089 #else
1090 #define PKT_TYPE_MAX	7
1091 #endif
1092 #define PKT_TYPE_OFFSET		offsetof(struct sk_buff, __pkt_type_offset)
1093 
1094 /* if you move tc_at_ingress or tstamp_type
1095  * around, you also must adapt these constants.
1096  */
1097 #ifdef __BIG_ENDIAN_BITFIELD
1098 #define SKB_TSTAMP_TYPE_MASK		(3 << 6)
1099 #define SKB_TSTAMP_TYPE_RSHIFT		(6)
1100 #define TC_AT_INGRESS_MASK		(1 << 5)
1101 #else
1102 #define SKB_TSTAMP_TYPE_MASK		(3)
1103 #define TC_AT_INGRESS_MASK		(1 << 2)
1104 #endif
1105 #define SKB_BF_MONO_TC_OFFSET		offsetof(struct sk_buff, __mono_tc_offset)
1106 
1107 #ifdef __KERNEL__
1108 /*
1109  *	Handling routines are only of interest to the kernel
1110  */
1111 
1112 #define SKB_ALLOC_FCLONE	0x01
1113 #define SKB_ALLOC_RX		0x02
1114 #define SKB_ALLOC_NAPI		0x04
1115 
1116 /**
1117  * skb_pfmemalloc - Test if the skb was allocated from PFMEMALLOC reserves
1118  * @skb: buffer
1119  */
skb_pfmemalloc(const struct sk_buff * skb)1120 static inline bool skb_pfmemalloc(const struct sk_buff *skb)
1121 {
1122 	return unlikely(skb->pfmemalloc);
1123 }
1124 
1125 /*
1126  * skb might have a dst pointer attached, refcounted or not.
1127  * _skb_refdst low order bit is set if refcount was _not_ taken
1128  */
1129 #define SKB_DST_NOREF	1UL
1130 #define SKB_DST_PTRMASK	~(SKB_DST_NOREF)
1131 
1132 /**
1133  * skb_dst - returns skb dst_entry
1134  * @skb: buffer
1135  *
1136  * Returns skb dst_entry, regardless of reference taken or not.
1137  */
skb_dst(const struct sk_buff * skb)1138 static inline struct dst_entry *skb_dst(const struct sk_buff *skb)
1139 {
1140 	/* If refdst was not refcounted, check we still are in a
1141 	 * rcu_read_lock section
1142 	 */
1143 	WARN_ON((skb->_skb_refdst & SKB_DST_NOREF) &&
1144 		!rcu_read_lock_held() &&
1145 		!rcu_read_lock_bh_held());
1146 	return (struct dst_entry *)(skb->_skb_refdst & SKB_DST_PTRMASK);
1147 }
1148 
1149 /**
1150  * skb_dst_set - sets skb dst
1151  * @skb: buffer
1152  * @dst: dst entry
1153  *
1154  * Sets skb dst, assuming a reference was taken on dst and should
1155  * be released by skb_dst_drop()
1156  */
skb_dst_set(struct sk_buff * skb,struct dst_entry * dst)1157 static inline void skb_dst_set(struct sk_buff *skb, struct dst_entry *dst)
1158 {
1159 	skb->slow_gro |= !!dst;
1160 	skb->_skb_refdst = (unsigned long)dst;
1161 }
1162 
1163 /**
1164  * skb_dst_set_noref - sets skb dst, hopefully, without taking reference
1165  * @skb: buffer
1166  * @dst: dst entry
1167  *
1168  * Sets skb dst, assuming a reference was not taken on dst.
1169  * If dst entry is cached, we do not take reference and dst_release
1170  * will be avoided by refdst_drop. If dst entry is not cached, we take
1171  * reference, so that last dst_release can destroy the dst immediately.
1172  */
skb_dst_set_noref(struct sk_buff * skb,struct dst_entry * dst)1173 static inline void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst)
1174 {
1175 	WARN_ON(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
1176 	skb->slow_gro |= !!dst;
1177 	skb->_skb_refdst = (unsigned long)dst | SKB_DST_NOREF;
1178 }
1179 
1180 /**
1181  * skb_dst_is_noref - Test if skb dst isn't refcounted
1182  * @skb: buffer
1183  */
skb_dst_is_noref(const struct sk_buff * skb)1184 static inline bool skb_dst_is_noref(const struct sk_buff *skb)
1185 {
1186 	return (skb->_skb_refdst & SKB_DST_NOREF) && skb_dst(skb);
1187 }
1188 
1189 /* For mangling skb->pkt_type from user space side from applications
1190  * such as nft, tc, etc, we only allow a conservative subset of
1191  * possible pkt_types to be set.
1192 */
skb_pkt_type_ok(u32 ptype)1193 static inline bool skb_pkt_type_ok(u32 ptype)
1194 {
1195 	return ptype <= PACKET_OTHERHOST;
1196 }
1197 
1198 /**
1199  * skb_napi_id - Returns the skb's NAPI id
1200  * @skb: buffer
1201  */
skb_napi_id(const struct sk_buff * skb)1202 static inline unsigned int skb_napi_id(const struct sk_buff *skb)
1203 {
1204 #ifdef CONFIG_NET_RX_BUSY_POLL
1205 	return skb->napi_id;
1206 #else
1207 	return 0;
1208 #endif
1209 }
1210 
skb_wifi_acked_valid(const struct sk_buff * skb)1211 static inline bool skb_wifi_acked_valid(const struct sk_buff *skb)
1212 {
1213 #ifdef CONFIG_WIRELESS
1214 	return skb->wifi_acked_valid;
1215 #else
1216 	return 0;
1217 #endif
1218 }
1219 
1220 /**
1221  * skb_unref - decrement the skb's reference count
1222  * @skb: buffer
1223  *
1224  * Returns true if we can free the skb.
1225  */
skb_unref(struct sk_buff * skb)1226 static inline bool skb_unref(struct sk_buff *skb)
1227 {
1228 	if (unlikely(!skb))
1229 		return false;
1230 	if (!IS_ENABLED(CONFIG_DEBUG_NET) && likely(refcount_read(&skb->users) == 1))
1231 		smp_rmb();
1232 	else if (likely(!refcount_dec_and_test(&skb->users)))
1233 		return false;
1234 
1235 	return true;
1236 }
1237 
skb_data_unref(const struct sk_buff * skb,struct skb_shared_info * shinfo)1238 static inline bool skb_data_unref(const struct sk_buff *skb,
1239 				  struct skb_shared_info *shinfo)
1240 {
1241 	int bias;
1242 
1243 	if (!skb->cloned)
1244 		return true;
1245 
1246 	bias = skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1;
1247 
1248 	if (atomic_read(&shinfo->dataref) == bias)
1249 		smp_rmb();
1250 	else if (atomic_sub_return(bias, &shinfo->dataref))
1251 		return false;
1252 
1253 	return true;
1254 }
1255 
1256 void __fix_address sk_skb_reason_drop(struct sock *sk, struct sk_buff *skb,
1257 				      enum skb_drop_reason reason);
1258 
1259 static inline void
kfree_skb_reason(struct sk_buff * skb,enum skb_drop_reason reason)1260 kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason)
1261 {
1262 	sk_skb_reason_drop(NULL, skb, reason);
1263 }
1264 
1265 /**
1266  *	kfree_skb - free an sk_buff with 'NOT_SPECIFIED' reason
1267  *	@skb: buffer to free
1268  */
kfree_skb(struct sk_buff * skb)1269 static inline void kfree_skb(struct sk_buff *skb)
1270 {
1271 	kfree_skb_reason(skb, SKB_DROP_REASON_NOT_SPECIFIED);
1272 }
1273 
1274 void skb_release_head_state(struct sk_buff *skb);
1275 void kfree_skb_list_reason(struct sk_buff *segs,
1276 			   enum skb_drop_reason reason);
1277 void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt);
1278 void skb_tx_error(struct sk_buff *skb);
1279 
kfree_skb_list(struct sk_buff * segs)1280 static inline void kfree_skb_list(struct sk_buff *segs)
1281 {
1282 	kfree_skb_list_reason(segs, SKB_DROP_REASON_NOT_SPECIFIED);
1283 }
1284 
1285 #ifdef CONFIG_TRACEPOINTS
1286 void consume_skb(struct sk_buff *skb);
1287 #else
consume_skb(struct sk_buff * skb)1288 static inline void consume_skb(struct sk_buff *skb)
1289 {
1290 	return kfree_skb(skb);
1291 }
1292 #endif
1293 
1294 void __consume_stateless_skb(struct sk_buff *skb);
1295 void  __kfree_skb(struct sk_buff *skb);
1296 
1297 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);
1298 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
1299 		      bool *fragstolen, int *delta_truesize);
1300 
1301 struct sk_buff *__alloc_skb(unsigned int size, gfp_t priority, int flags,
1302 			    int node);
1303 struct sk_buff *__build_skb(void *data, unsigned int frag_size);
1304 struct sk_buff *build_skb(void *data, unsigned int frag_size);
1305 struct sk_buff *build_skb_around(struct sk_buff *skb,
1306 				 void *data, unsigned int frag_size);
1307 void skb_attempt_defer_free(struct sk_buff *skb);
1308 
1309 struct sk_buff *napi_build_skb(void *data, unsigned int frag_size);
1310 struct sk_buff *slab_build_skb(void *data);
1311 
1312 /**
1313  * alloc_skb - allocate a network buffer
1314  * @size: size to allocate
1315  * @priority: allocation mask
1316  *
1317  * This function is a convenient wrapper around __alloc_skb().
1318  */
alloc_skb(unsigned int size,gfp_t priority)1319 static inline struct sk_buff *alloc_skb(unsigned int size,
1320 					gfp_t priority)
1321 {
1322 	return __alloc_skb(size, priority, 0, NUMA_NO_NODE);
1323 }
1324 
1325 struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
1326 				     unsigned long data_len,
1327 				     int max_page_order,
1328 				     int *errcode,
1329 				     gfp_t gfp_mask);
1330 struct sk_buff *alloc_skb_for_msg(struct sk_buff *first);
1331 
1332 /* Layout of fast clones : [skb1][skb2][fclone_ref] */
1333 struct sk_buff_fclones {
1334 	struct sk_buff	skb1;
1335 
1336 	struct sk_buff	skb2;
1337 
1338 	refcount_t	fclone_ref;
1339 };
1340 
1341 /**
1342  *	skb_fclone_busy - check if fclone is busy
1343  *	@sk: socket
1344  *	@skb: buffer
1345  *
1346  * Returns true if skb is a fast clone, and its clone is not freed.
1347  * Some drivers call skb_orphan() in their ndo_start_xmit(),
1348  * so we also check that didn't happen.
1349  */
skb_fclone_busy(const struct sock * sk,const struct sk_buff * skb)1350 static inline bool skb_fclone_busy(const struct sock *sk,
1351 				   const struct sk_buff *skb)
1352 {
1353 	const struct sk_buff_fclones *fclones;
1354 
1355 	fclones = container_of(skb, struct sk_buff_fclones, skb1);
1356 
1357 	return skb->fclone == SKB_FCLONE_ORIG &&
1358 	       refcount_read(&fclones->fclone_ref) > 1 &&
1359 	       READ_ONCE(fclones->skb2.sk) == sk;
1360 }
1361 
1362 /**
1363  * alloc_skb_fclone - allocate a network buffer from fclone cache
1364  * @size: size to allocate
1365  * @priority: allocation mask
1366  *
1367  * This function is a convenient wrapper around __alloc_skb().
1368  */
alloc_skb_fclone(unsigned int size,gfp_t priority)1369 static inline struct sk_buff *alloc_skb_fclone(unsigned int size,
1370 					       gfp_t priority)
1371 {
1372 	return __alloc_skb(size, priority, SKB_ALLOC_FCLONE, NUMA_NO_NODE);
1373 }
1374 
1375 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src);
1376 void skb_headers_offset_update(struct sk_buff *skb, int off);
1377 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask);
1378 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t priority);
1379 void skb_copy_header(struct sk_buff *new, const struct sk_buff *old);
1380 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t priority);
1381 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
1382 				   gfp_t gfp_mask, bool fclone);
__pskb_copy(struct sk_buff * skb,int headroom,gfp_t gfp_mask)1383 static inline struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom,
1384 					  gfp_t gfp_mask)
1385 {
1386 	return __pskb_copy_fclone(skb, headroom, gfp_mask, false);
1387 }
1388 
1389 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, gfp_t gfp_mask);
1390 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb,
1391 				     unsigned int headroom);
1392 struct sk_buff *skb_expand_head(struct sk_buff *skb, unsigned int headroom);
1393 struct sk_buff *skb_copy_expand(const struct sk_buff *skb, int newheadroom,
1394 				int newtailroom, gfp_t priority);
1395 int __must_check skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
1396 				     int offset, int len);
1397 int __must_check skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg,
1398 			      int offset, int len);
1399 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer);
1400 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error);
1401 
1402 /**
1403  *	skb_pad			-	zero pad the tail of an skb
1404  *	@skb: buffer to pad
1405  *	@pad: space to pad
1406  *
1407  *	Ensure that a buffer is followed by a padding area that is zero
1408  *	filled. Used by network drivers which may DMA or transfer data
1409  *	beyond the buffer end onto the wire.
1410  *
1411  *	May return error in out of memory cases. The skb is freed on error.
1412  */
skb_pad(struct sk_buff * skb,int pad)1413 static inline int skb_pad(struct sk_buff *skb, int pad)
1414 {
1415 	return __skb_pad(skb, pad, true);
1416 }
1417 #define dev_kfree_skb(a)	consume_skb(a)
1418 
1419 int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
1420 			 int offset, size_t size, size_t max_frags);
1421 
1422 struct skb_seq_state {
1423 	__u32		lower_offset;
1424 	__u32		upper_offset;
1425 	__u32		frag_idx;
1426 	__u32		stepped_offset;
1427 	struct sk_buff	*root_skb;
1428 	struct sk_buff	*cur_skb;
1429 	__u8		*frag_data;
1430 	__u32		frag_off;
1431 };
1432 
1433 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
1434 			  unsigned int to, struct skb_seq_state *st);
1435 unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
1436 			  struct skb_seq_state *st);
1437 void skb_abort_seq_read(struct skb_seq_state *st);
1438 int skb_copy_seq_read(struct skb_seq_state *st, int offset, void *to, int len);
1439 
1440 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
1441 			   unsigned int to, struct ts_config *config);
1442 
1443 /*
1444  * Packet hash types specify the type of hash in skb_set_hash.
1445  *
1446  * Hash types refer to the protocol layer addresses which are used to
1447  * construct a packet's hash. The hashes are used to differentiate or identify
1448  * flows of the protocol layer for the hash type. Hash types are either
1449  * layer-2 (L2), layer-3 (L3), or layer-4 (L4).
1450  *
1451  * Properties of hashes:
1452  *
1453  * 1) Two packets in different flows have different hash values
1454  * 2) Two packets in the same flow should have the same hash value
1455  *
1456  * A hash at a higher layer is considered to be more specific. A driver should
1457  * set the most specific hash possible.
1458  *
1459  * A driver cannot indicate a more specific hash than the layer at which a hash
1460  * was computed. For instance an L3 hash cannot be set as an L4 hash.
1461  *
1462  * A driver may indicate a hash level which is less specific than the
1463  * actual layer the hash was computed on. For instance, a hash computed
1464  * at L4 may be considered an L3 hash. This should only be done if the
1465  * driver can't unambiguously determine that the HW computed the hash at
1466  * the higher layer. Note that the "should" in the second property above
1467  * permits this.
1468  */
1469 enum pkt_hash_types {
1470 	PKT_HASH_TYPE_NONE,	/* Undefined type */
1471 	PKT_HASH_TYPE_L2,	/* Input: src_MAC, dest_MAC */
1472 	PKT_HASH_TYPE_L3,	/* Input: src_IP, dst_IP */
1473 	PKT_HASH_TYPE_L4,	/* Input: src_IP, dst_IP, src_port, dst_port */
1474 };
1475 
skb_clear_hash(struct sk_buff * skb)1476 static inline void skb_clear_hash(struct sk_buff *skb)
1477 {
1478 	skb->hash = 0;
1479 	skb->sw_hash = 0;
1480 	skb->l4_hash = 0;
1481 }
1482 
skb_clear_hash_if_not_l4(struct sk_buff * skb)1483 static inline void skb_clear_hash_if_not_l4(struct sk_buff *skb)
1484 {
1485 	if (!skb->l4_hash)
1486 		skb_clear_hash(skb);
1487 }
1488 
1489 static inline void
__skb_set_hash(struct sk_buff * skb,__u32 hash,bool is_sw,bool is_l4)1490 __skb_set_hash(struct sk_buff *skb, __u32 hash, bool is_sw, bool is_l4)
1491 {
1492 	skb->l4_hash = is_l4;
1493 	skb->sw_hash = is_sw;
1494 	skb->hash = hash;
1495 }
1496 
1497 static inline void
skb_set_hash(struct sk_buff * skb,__u32 hash,enum pkt_hash_types type)1498 skb_set_hash(struct sk_buff *skb, __u32 hash, enum pkt_hash_types type)
1499 {
1500 	/* Used by drivers to set hash from HW */
1501 	__skb_set_hash(skb, hash, false, type == PKT_HASH_TYPE_L4);
1502 }
1503 
1504 static inline void
__skb_set_sw_hash(struct sk_buff * skb,__u32 hash,bool is_l4)1505 __skb_set_sw_hash(struct sk_buff *skb, __u32 hash, bool is_l4)
1506 {
1507 	__skb_set_hash(skb, hash, true, is_l4);
1508 }
1509 
1510 u32 __skb_get_hash_symmetric_net(const struct net *net, const struct sk_buff *skb);
1511 
__skb_get_hash_symmetric(const struct sk_buff * skb)1512 static inline u32 __skb_get_hash_symmetric(const struct sk_buff *skb)
1513 {
1514 	return __skb_get_hash_symmetric_net(NULL, skb);
1515 }
1516 
1517 void __skb_get_hash_net(const struct net *net, struct sk_buff *skb);
1518 u32 skb_get_poff(const struct sk_buff *skb);
1519 u32 __skb_get_poff(const struct sk_buff *skb, const void *data,
1520 		   const struct flow_keys_basic *keys, int hlen);
1521 __be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
1522 			    const void *data, int hlen_proto);
1523 
skb_flow_get_ports(const struct sk_buff * skb,int thoff,u8 ip_proto)1524 static inline __be32 skb_flow_get_ports(const struct sk_buff *skb,
1525 					int thoff, u8 ip_proto)
1526 {
1527 	return __skb_flow_get_ports(skb, thoff, ip_proto, NULL, 0);
1528 }
1529 
1530 void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
1531 			     const struct flow_dissector_key *key,
1532 			     unsigned int key_count);
1533 
1534 struct bpf_flow_dissector;
1535 u32 bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx,
1536 		     __be16 proto, int nhoff, int hlen, unsigned int flags);
1537 
1538 bool __skb_flow_dissect(const struct net *net,
1539 			const struct sk_buff *skb,
1540 			struct flow_dissector *flow_dissector,
1541 			void *target_container, const void *data,
1542 			__be16 proto, int nhoff, int hlen, unsigned int flags);
1543 
skb_flow_dissect(const struct sk_buff * skb,struct flow_dissector * flow_dissector,void * target_container,unsigned int flags)1544 static inline bool skb_flow_dissect(const struct sk_buff *skb,
1545 				    struct flow_dissector *flow_dissector,
1546 				    void *target_container, unsigned int flags)
1547 {
1548 	return __skb_flow_dissect(NULL, skb, flow_dissector,
1549 				  target_container, NULL, 0, 0, 0, flags);
1550 }
1551 
skb_flow_dissect_flow_keys(const struct sk_buff * skb,struct flow_keys * flow,unsigned int flags)1552 static inline bool skb_flow_dissect_flow_keys(const struct sk_buff *skb,
1553 					      struct flow_keys *flow,
1554 					      unsigned int flags)
1555 {
1556 	memset(flow, 0, sizeof(*flow));
1557 	return __skb_flow_dissect(NULL, skb, &flow_keys_dissector,
1558 				  flow, NULL, 0, 0, 0, flags);
1559 }
1560 
1561 static inline bool
skb_flow_dissect_flow_keys_basic(const struct net * net,const struct sk_buff * skb,struct flow_keys_basic * flow,const void * data,__be16 proto,int nhoff,int hlen,unsigned int flags)1562 skb_flow_dissect_flow_keys_basic(const struct net *net,
1563 				 const struct sk_buff *skb,
1564 				 struct flow_keys_basic *flow,
1565 				 const void *data, __be16 proto,
1566 				 int nhoff, int hlen, unsigned int flags)
1567 {
1568 	memset(flow, 0, sizeof(*flow));
1569 	return __skb_flow_dissect(net, skb, &flow_keys_basic_dissector, flow,
1570 				  data, proto, nhoff, hlen, flags);
1571 }
1572 
1573 void skb_flow_dissect_meta(const struct sk_buff *skb,
1574 			   struct flow_dissector *flow_dissector,
1575 			   void *target_container);
1576 
1577 /* Gets a skb connection tracking info, ctinfo map should be a
1578  * map of mapsize to translate enum ip_conntrack_info states
1579  * to user states.
1580  */
1581 void
1582 skb_flow_dissect_ct(const struct sk_buff *skb,
1583 		    struct flow_dissector *flow_dissector,
1584 		    void *target_container,
1585 		    u16 *ctinfo_map, size_t mapsize,
1586 		    bool post_ct, u16 zone);
1587 void
1588 skb_flow_dissect_tunnel_info(const struct sk_buff *skb,
1589 			     struct flow_dissector *flow_dissector,
1590 			     void *target_container);
1591 
1592 void skb_flow_dissect_hash(const struct sk_buff *skb,
1593 			   struct flow_dissector *flow_dissector,
1594 			   void *target_container);
1595 
skb_get_hash_net(const struct net * net,struct sk_buff * skb)1596 static inline __u32 skb_get_hash_net(const struct net *net, struct sk_buff *skb)
1597 {
1598 	if (!skb->l4_hash && !skb->sw_hash)
1599 		__skb_get_hash_net(net, skb);
1600 
1601 	return skb->hash;
1602 }
1603 
skb_get_hash(struct sk_buff * skb)1604 static inline __u32 skb_get_hash(struct sk_buff *skb)
1605 {
1606 	if (!skb->l4_hash && !skb->sw_hash)
1607 		__skb_get_hash_net(NULL, skb);
1608 
1609 	return skb->hash;
1610 }
1611 
skb_get_hash_flowi6(struct sk_buff * skb,const struct flowi6 * fl6)1612 static inline __u32 skb_get_hash_flowi6(struct sk_buff *skb, const struct flowi6 *fl6)
1613 {
1614 	if (!skb->l4_hash && !skb->sw_hash) {
1615 		struct flow_keys keys;
1616 		__u32 hash = __get_hash_from_flowi6(fl6, &keys);
1617 
1618 		__skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys));
1619 	}
1620 
1621 	return skb->hash;
1622 }
1623 
1624 __u32 skb_get_hash_perturb(const struct sk_buff *skb,
1625 			   const siphash_key_t *perturb);
1626 
skb_get_hash_raw(const struct sk_buff * skb)1627 static inline __u32 skb_get_hash_raw(const struct sk_buff *skb)
1628 {
1629 	return skb->hash;
1630 }
1631 
skb_copy_hash(struct sk_buff * to,const struct sk_buff * from)1632 static inline void skb_copy_hash(struct sk_buff *to, const struct sk_buff *from)
1633 {
1634 	to->hash = from->hash;
1635 	to->sw_hash = from->sw_hash;
1636 	to->l4_hash = from->l4_hash;
1637 };
1638 
skb_cmp_decrypted(const struct sk_buff * skb1,const struct sk_buff * skb2)1639 static inline int skb_cmp_decrypted(const struct sk_buff *skb1,
1640 				    const struct sk_buff *skb2)
1641 {
1642 #ifdef CONFIG_SKB_DECRYPTED
1643 	return skb2->decrypted - skb1->decrypted;
1644 #else
1645 	return 0;
1646 #endif
1647 }
1648 
skb_is_decrypted(const struct sk_buff * skb)1649 static inline bool skb_is_decrypted(const struct sk_buff *skb)
1650 {
1651 #ifdef CONFIG_SKB_DECRYPTED
1652 	return skb->decrypted;
1653 #else
1654 	return false;
1655 #endif
1656 }
1657 
skb_copy_decrypted(struct sk_buff * to,const struct sk_buff * from)1658 static inline void skb_copy_decrypted(struct sk_buff *to,
1659 				      const struct sk_buff *from)
1660 {
1661 #ifdef CONFIG_SKB_DECRYPTED
1662 	to->decrypted = from->decrypted;
1663 #endif
1664 }
1665 
1666 #ifdef NET_SKBUFF_DATA_USES_OFFSET
skb_end_pointer(const struct sk_buff * skb)1667 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
1668 {
1669 	return skb->head + skb->end;
1670 }
1671 
skb_end_offset(const struct sk_buff * skb)1672 static inline unsigned int skb_end_offset(const struct sk_buff *skb)
1673 {
1674 	return skb->end;
1675 }
1676 
skb_set_end_offset(struct sk_buff * skb,unsigned int offset)1677 static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset)
1678 {
1679 	skb->end = offset;
1680 }
1681 #else
skb_end_pointer(const struct sk_buff * skb)1682 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
1683 {
1684 	return skb->end;
1685 }
1686 
skb_end_offset(const struct sk_buff * skb)1687 static inline unsigned int skb_end_offset(const struct sk_buff *skb)
1688 {
1689 	return skb->end - skb->head;
1690 }
1691 
skb_set_end_offset(struct sk_buff * skb,unsigned int offset)1692 static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset)
1693 {
1694 	skb->end = skb->head + offset;
1695 }
1696 #endif
1697 
1698 extern const struct ubuf_info_ops msg_zerocopy_ubuf_ops;
1699 
1700 struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
1701 				       struct ubuf_info *uarg);
1702 
1703 void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref);
1704 
1705 int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk,
1706 			    struct sk_buff *skb, struct iov_iter *from,
1707 			    size_t length);
1708 
1709 int zerocopy_fill_skb_from_iter(struct sk_buff *skb,
1710 				struct iov_iter *from, size_t length);
1711 
skb_zerocopy_iter_dgram(struct sk_buff * skb,struct msghdr * msg,int len)1712 static inline int skb_zerocopy_iter_dgram(struct sk_buff *skb,
1713 					  struct msghdr *msg, int len)
1714 {
1715 	return __zerocopy_sg_from_iter(msg, skb->sk, skb, &msg->msg_iter, len);
1716 }
1717 
1718 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
1719 			     struct msghdr *msg, int len,
1720 			     struct ubuf_info *uarg);
1721 
1722 /* Internal */
1723 #define skb_shinfo(SKB)	((struct skb_shared_info *)(skb_end_pointer(SKB)))
1724 
skb_hwtstamps(struct sk_buff * skb)1725 static inline struct skb_shared_hwtstamps *skb_hwtstamps(struct sk_buff *skb)
1726 {
1727 	return &skb_shinfo(skb)->hwtstamps;
1728 }
1729 
skb_zcopy(struct sk_buff * skb)1730 static inline struct ubuf_info *skb_zcopy(struct sk_buff *skb)
1731 {
1732 	bool is_zcopy = skb && skb_shinfo(skb)->flags & SKBFL_ZEROCOPY_ENABLE;
1733 
1734 	return is_zcopy ? skb_uarg(skb) : NULL;
1735 }
1736 
skb_zcopy_pure(const struct sk_buff * skb)1737 static inline bool skb_zcopy_pure(const struct sk_buff *skb)
1738 {
1739 	return skb_shinfo(skb)->flags & SKBFL_PURE_ZEROCOPY;
1740 }
1741 
skb_zcopy_managed(const struct sk_buff * skb)1742 static inline bool skb_zcopy_managed(const struct sk_buff *skb)
1743 {
1744 	return skb_shinfo(skb)->flags & SKBFL_MANAGED_FRAG_REFS;
1745 }
1746 
skb_pure_zcopy_same(const struct sk_buff * skb1,const struct sk_buff * skb2)1747 static inline bool skb_pure_zcopy_same(const struct sk_buff *skb1,
1748 				       const struct sk_buff *skb2)
1749 {
1750 	return skb_zcopy_pure(skb1) == skb_zcopy_pure(skb2);
1751 }
1752 
net_zcopy_get(struct ubuf_info * uarg)1753 static inline void net_zcopy_get(struct ubuf_info *uarg)
1754 {
1755 	refcount_inc(&uarg->refcnt);
1756 }
1757 
skb_zcopy_init(struct sk_buff * skb,struct ubuf_info * uarg)1758 static inline void skb_zcopy_init(struct sk_buff *skb, struct ubuf_info *uarg)
1759 {
1760 	skb_shinfo(skb)->destructor_arg = uarg;
1761 	skb_shinfo(skb)->flags |= uarg->flags;
1762 }
1763 
skb_zcopy_set(struct sk_buff * skb,struct ubuf_info * uarg,bool * have_ref)1764 static inline void skb_zcopy_set(struct sk_buff *skb, struct ubuf_info *uarg,
1765 				 bool *have_ref)
1766 {
1767 	if (skb && uarg && !skb_zcopy(skb)) {
1768 		if (unlikely(have_ref && *have_ref))
1769 			*have_ref = false;
1770 		else
1771 			net_zcopy_get(uarg);
1772 		skb_zcopy_init(skb, uarg);
1773 	}
1774 }
1775 
skb_zcopy_set_nouarg(struct sk_buff * skb,void * val)1776 static inline void skb_zcopy_set_nouarg(struct sk_buff *skb, void *val)
1777 {
1778 	skb_shinfo(skb)->destructor_arg = (void *)((uintptr_t) val | 0x1UL);
1779 	skb_shinfo(skb)->flags |= SKBFL_ZEROCOPY_FRAG;
1780 }
1781 
skb_zcopy_is_nouarg(struct sk_buff * skb)1782 static inline bool skb_zcopy_is_nouarg(struct sk_buff *skb)
1783 {
1784 	return (uintptr_t) skb_shinfo(skb)->destructor_arg & 0x1UL;
1785 }
1786 
skb_zcopy_get_nouarg(struct sk_buff * skb)1787 static inline void *skb_zcopy_get_nouarg(struct sk_buff *skb)
1788 {
1789 	return (void *)((uintptr_t) skb_shinfo(skb)->destructor_arg & ~0x1UL);
1790 }
1791 
net_zcopy_put(struct ubuf_info * uarg)1792 static inline void net_zcopy_put(struct ubuf_info *uarg)
1793 {
1794 	if (uarg)
1795 		uarg->ops->complete(NULL, uarg, true);
1796 }
1797 
net_zcopy_put_abort(struct ubuf_info * uarg,bool have_uref)1798 static inline void net_zcopy_put_abort(struct ubuf_info *uarg, bool have_uref)
1799 {
1800 	if (uarg) {
1801 		if (uarg->ops == &msg_zerocopy_ubuf_ops)
1802 			msg_zerocopy_put_abort(uarg, have_uref);
1803 		else if (have_uref)
1804 			net_zcopy_put(uarg);
1805 	}
1806 }
1807 
1808 /* Release a reference on a zerocopy structure */
skb_zcopy_clear(struct sk_buff * skb,bool zerocopy_success)1809 static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy_success)
1810 {
1811 	struct ubuf_info *uarg = skb_zcopy(skb);
1812 
1813 	if (uarg) {
1814 		if (!skb_zcopy_is_nouarg(skb))
1815 			uarg->ops->complete(skb, uarg, zerocopy_success);
1816 
1817 		skb_shinfo(skb)->flags &= ~SKBFL_ALL_ZEROCOPY;
1818 	}
1819 }
1820 
1821 void __skb_zcopy_downgrade_managed(struct sk_buff *skb);
1822 
skb_zcopy_downgrade_managed(struct sk_buff * skb)1823 static inline void skb_zcopy_downgrade_managed(struct sk_buff *skb)
1824 {
1825 	if (unlikely(skb_zcopy_managed(skb)))
1826 		__skb_zcopy_downgrade_managed(skb);
1827 }
1828 
1829 /* Return true if frags in this skb are readable by the host. */
skb_frags_readable(const struct sk_buff * skb)1830 static inline bool skb_frags_readable(const struct sk_buff *skb)
1831 {
1832 	return !skb->unreadable;
1833 }
1834 
skb_mark_not_on_list(struct sk_buff * skb)1835 static inline void skb_mark_not_on_list(struct sk_buff *skb)
1836 {
1837 	skb->next = NULL;
1838 }
1839 
skb_poison_list(struct sk_buff * skb)1840 static inline void skb_poison_list(struct sk_buff *skb)
1841 {
1842 #ifdef CONFIG_DEBUG_NET
1843 	skb->next = SKB_LIST_POISON_NEXT;
1844 #endif
1845 }
1846 
1847 /* Iterate through singly-linked GSO fragments of an skb. */
1848 #define skb_list_walk_safe(first, skb, next_skb)                               \
1849 	for ((skb) = (first), (next_skb) = (skb) ? (skb)->next : NULL; (skb);  \
1850 	     (skb) = (next_skb), (next_skb) = (skb) ? (skb)->next : NULL)
1851 
skb_list_del_init(struct sk_buff * skb)1852 static inline void skb_list_del_init(struct sk_buff *skb)
1853 {
1854 	__list_del_entry(&skb->list);
1855 	skb_mark_not_on_list(skb);
1856 }
1857 
1858 /**
1859  *	skb_queue_empty - check if a queue is empty
1860  *	@list: queue head
1861  *
1862  *	Returns true if the queue is empty, false otherwise.
1863  */
skb_queue_empty(const struct sk_buff_head * list)1864 static inline int skb_queue_empty(const struct sk_buff_head *list)
1865 {
1866 	return list->next == (const struct sk_buff *) list;
1867 }
1868 
1869 /**
1870  *	skb_queue_empty_lockless - check if a queue is empty
1871  *	@list: queue head
1872  *
1873  *	Returns true if the queue is empty, false otherwise.
1874  *	This variant can be used in lockless contexts.
1875  */
skb_queue_empty_lockless(const struct sk_buff_head * list)1876 static inline bool skb_queue_empty_lockless(const struct sk_buff_head *list)
1877 {
1878 	return READ_ONCE(list->next) == (const struct sk_buff *) list;
1879 }
1880 
1881 
1882 /**
1883  *	skb_queue_is_last - check if skb is the last entry in the queue
1884  *	@list: queue head
1885  *	@skb: buffer
1886  *
1887  *	Returns true if @skb is the last buffer on the list.
1888  */
skb_queue_is_last(const struct sk_buff_head * list,const struct sk_buff * skb)1889 static inline bool skb_queue_is_last(const struct sk_buff_head *list,
1890 				     const struct sk_buff *skb)
1891 {
1892 	return skb->next == (const struct sk_buff *) list;
1893 }
1894 
1895 /**
1896  *	skb_queue_is_first - check if skb is the first entry in the queue
1897  *	@list: queue head
1898  *	@skb: buffer
1899  *
1900  *	Returns true if @skb is the first buffer on the list.
1901  */
skb_queue_is_first(const struct sk_buff_head * list,const struct sk_buff * skb)1902 static inline bool skb_queue_is_first(const struct sk_buff_head *list,
1903 				      const struct sk_buff *skb)
1904 {
1905 	return skb->prev == (const struct sk_buff *) list;
1906 }
1907 
1908 /**
1909  *	skb_queue_next - return the next packet in the queue
1910  *	@list: queue head
1911  *	@skb: current buffer
1912  *
1913  *	Return the next packet in @list after @skb.  It is only valid to
1914  *	call this if skb_queue_is_last() evaluates to false.
1915  */
skb_queue_next(const struct sk_buff_head * list,const struct sk_buff * skb)1916 static inline struct sk_buff *skb_queue_next(const struct sk_buff_head *list,
1917 					     const struct sk_buff *skb)
1918 {
1919 	/* This BUG_ON may seem severe, but if we just return then we
1920 	 * are going to dereference garbage.
1921 	 */
1922 	BUG_ON(skb_queue_is_last(list, skb));
1923 	return skb->next;
1924 }
1925 
1926 /**
1927  *	skb_queue_prev - return the prev packet in the queue
1928  *	@list: queue head
1929  *	@skb: current buffer
1930  *
1931  *	Return the prev packet in @list before @skb.  It is only valid to
1932  *	call this if skb_queue_is_first() evaluates to false.
1933  */
skb_queue_prev(const struct sk_buff_head * list,const struct sk_buff * skb)1934 static inline struct sk_buff *skb_queue_prev(const struct sk_buff_head *list,
1935 					     const struct sk_buff *skb)
1936 {
1937 	/* This BUG_ON may seem severe, but if we just return then we
1938 	 * are going to dereference garbage.
1939 	 */
1940 	BUG_ON(skb_queue_is_first(list, skb));
1941 	return skb->prev;
1942 }
1943 
1944 /**
1945  *	skb_get - reference buffer
1946  *	@skb: buffer to reference
1947  *
1948  *	Makes another reference to a socket buffer and returns a pointer
1949  *	to the buffer.
1950  */
skb_get(struct sk_buff * skb)1951 static inline struct sk_buff *skb_get(struct sk_buff *skb)
1952 {
1953 	refcount_inc(&skb->users);
1954 	return skb;
1955 }
1956 
1957 /*
1958  * If users == 1, we are the only owner and can avoid redundant atomic changes.
1959  */
1960 
1961 /**
1962  *	skb_cloned - is the buffer a clone
1963  *	@skb: buffer to check
1964  *
1965  *	Returns true if the buffer was generated with skb_clone() and is
1966  *	one of multiple shared copies of the buffer. Cloned buffers are
1967  *	shared data so must not be written to under normal circumstances.
1968  */
skb_cloned(const struct sk_buff * skb)1969 static inline int skb_cloned(const struct sk_buff *skb)
1970 {
1971 	return skb->cloned &&
1972 	       (atomic_read(&skb_shinfo(skb)->dataref) & SKB_DATAREF_MASK) != 1;
1973 }
1974 
skb_unclone(struct sk_buff * skb,gfp_t pri)1975 static inline int skb_unclone(struct sk_buff *skb, gfp_t pri)
1976 {
1977 	might_sleep_if(gfpflags_allow_blocking(pri));
1978 
1979 	if (skb_cloned(skb))
1980 		return pskb_expand_head(skb, 0, 0, pri);
1981 
1982 	return 0;
1983 }
1984 
1985 /* This variant of skb_unclone() makes sure skb->truesize
1986  * and skb_end_offset() are not changed, whenever a new skb->head is needed.
1987  *
1988  * Indeed there is no guarantee that ksize(kmalloc(X)) == ksize(kmalloc(X))
1989  * when various debugging features are in place.
1990  */
1991 int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri);
skb_unclone_keeptruesize(struct sk_buff * skb,gfp_t pri)1992 static inline int skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri)
1993 {
1994 	might_sleep_if(gfpflags_allow_blocking(pri));
1995 
1996 	if (skb_cloned(skb))
1997 		return __skb_unclone_keeptruesize(skb, pri);
1998 	return 0;
1999 }
2000 
2001 /**
2002  *	skb_header_cloned - is the header a clone
2003  *	@skb: buffer to check
2004  *
2005  *	Returns true if modifying the header part of the buffer requires
2006  *	the data to be copied.
2007  */
skb_header_cloned(const struct sk_buff * skb)2008 static inline int skb_header_cloned(const struct sk_buff *skb)
2009 {
2010 	int dataref;
2011 
2012 	if (!skb->cloned)
2013 		return 0;
2014 
2015 	dataref = atomic_read(&skb_shinfo(skb)->dataref);
2016 	dataref = (dataref & SKB_DATAREF_MASK) - (dataref >> SKB_DATAREF_SHIFT);
2017 	return dataref != 1;
2018 }
2019 
skb_header_unclone(struct sk_buff * skb,gfp_t pri)2020 static inline int skb_header_unclone(struct sk_buff *skb, gfp_t pri)
2021 {
2022 	might_sleep_if(gfpflags_allow_blocking(pri));
2023 
2024 	if (skb_header_cloned(skb))
2025 		return pskb_expand_head(skb, 0, 0, pri);
2026 
2027 	return 0;
2028 }
2029 
2030 /**
2031  * __skb_header_release() - allow clones to use the headroom
2032  * @skb: buffer to operate on
2033  *
2034  * See "DOC: dataref and headerless skbs".
2035  */
__skb_header_release(struct sk_buff * skb)2036 static inline void __skb_header_release(struct sk_buff *skb)
2037 {
2038 	skb->nohdr = 1;
2039 	atomic_set(&skb_shinfo(skb)->dataref, 1 + (1 << SKB_DATAREF_SHIFT));
2040 }
2041 
2042 
2043 /**
2044  *	skb_shared - is the buffer shared
2045  *	@skb: buffer to check
2046  *
2047  *	Returns true if more than one person has a reference to this
2048  *	buffer.
2049  */
skb_shared(const struct sk_buff * skb)2050 static inline int skb_shared(const struct sk_buff *skb)
2051 {
2052 	return refcount_read(&skb->users) != 1;
2053 }
2054 
2055 /**
2056  *	skb_share_check - check if buffer is shared and if so clone it
2057  *	@skb: buffer to check
2058  *	@pri: priority for memory allocation
2059  *
2060  *	If the buffer is shared the buffer is cloned and the old copy
2061  *	drops a reference. A new clone with a single reference is returned.
2062  *	If the buffer is not shared the original buffer is returned. When
2063  *	being called from interrupt status or with spinlocks held pri must
2064  *	be GFP_ATOMIC.
2065  *
2066  *	NULL is returned on a memory allocation failure.
2067  */
skb_share_check(struct sk_buff * skb,gfp_t pri)2068 static inline struct sk_buff *skb_share_check(struct sk_buff *skb, gfp_t pri)
2069 {
2070 	might_sleep_if(gfpflags_allow_blocking(pri));
2071 	if (skb_shared(skb)) {
2072 		struct sk_buff *nskb = skb_clone(skb, pri);
2073 
2074 		if (likely(nskb))
2075 			consume_skb(skb);
2076 		else
2077 			kfree_skb(skb);
2078 		skb = nskb;
2079 	}
2080 	return skb;
2081 }
2082 
2083 /*
2084  *	Copy shared buffers into a new sk_buff. We effectively do COW on
2085  *	packets to handle cases where we have a local reader and forward
2086  *	and a couple of other messy ones. The normal one is tcpdumping
2087  *	a packet that's being forwarded.
2088  */
2089 
2090 /**
2091  *	skb_unshare - make a copy of a shared buffer
2092  *	@skb: buffer to check
2093  *	@pri: priority for memory allocation
2094  *
2095  *	If the socket buffer is a clone then this function creates a new
2096  *	copy of the data, drops a reference count on the old copy and returns
2097  *	the new copy with the reference count at 1. If the buffer is not a clone
2098  *	the original buffer is returned. When called with a spinlock held or
2099  *	from interrupt state @pri must be %GFP_ATOMIC
2100  *
2101  *	%NULL is returned on a memory allocation failure.
2102  */
skb_unshare(struct sk_buff * skb,gfp_t pri)2103 static inline struct sk_buff *skb_unshare(struct sk_buff *skb,
2104 					  gfp_t pri)
2105 {
2106 	might_sleep_if(gfpflags_allow_blocking(pri));
2107 	if (skb_cloned(skb)) {
2108 		struct sk_buff *nskb = skb_copy(skb, pri);
2109 
2110 		/* Free our shared copy */
2111 		if (likely(nskb))
2112 			consume_skb(skb);
2113 		else
2114 			kfree_skb(skb);
2115 		skb = nskb;
2116 	}
2117 	return skb;
2118 }
2119 
2120 /**
2121  *	skb_peek - peek at the head of an &sk_buff_head
2122  *	@list_: list to peek at
2123  *
2124  *	Peek an &sk_buff. Unlike most other operations you _MUST_
2125  *	be careful with this one. A peek leaves the buffer on the
2126  *	list and someone else may run off with it. You must hold
2127  *	the appropriate locks or have a private queue to do this.
2128  *
2129  *	Returns %NULL for an empty list or a pointer to the head element.
2130  *	The reference count is not incremented and the reference is therefore
2131  *	volatile. Use with caution.
2132  */
skb_peek(const struct sk_buff_head * list_)2133 static inline struct sk_buff *skb_peek(const struct sk_buff_head *list_)
2134 {
2135 	struct sk_buff *skb = list_->next;
2136 
2137 	if (skb == (struct sk_buff *)list_)
2138 		skb = NULL;
2139 	return skb;
2140 }
2141 
2142 /**
2143  *	__skb_peek - peek at the head of a non-empty &sk_buff_head
2144  *	@list_: list to peek at
2145  *
2146  *	Like skb_peek(), but the caller knows that the list is not empty.
2147  */
__skb_peek(const struct sk_buff_head * list_)2148 static inline struct sk_buff *__skb_peek(const struct sk_buff_head *list_)
2149 {
2150 	return list_->next;
2151 }
2152 
2153 /**
2154  *	skb_peek_next - peek skb following the given one from a queue
2155  *	@skb: skb to start from
2156  *	@list_: list to peek at
2157  *
2158  *	Returns %NULL when the end of the list is met or a pointer to the
2159  *	next element. The reference count is not incremented and the
2160  *	reference is therefore volatile. Use with caution.
2161  */
skb_peek_next(struct sk_buff * skb,const struct sk_buff_head * list_)2162 static inline struct sk_buff *skb_peek_next(struct sk_buff *skb,
2163 		const struct sk_buff_head *list_)
2164 {
2165 	struct sk_buff *next = skb->next;
2166 
2167 	if (next == (struct sk_buff *)list_)
2168 		next = NULL;
2169 	return next;
2170 }
2171 
2172 /**
2173  *	skb_peek_tail - peek at the tail of an &sk_buff_head
2174  *	@list_: list to peek at
2175  *
2176  *	Peek an &sk_buff. Unlike most other operations you _MUST_
2177  *	be careful with this one. A peek leaves the buffer on the
2178  *	list and someone else may run off with it. You must hold
2179  *	the appropriate locks or have a private queue to do this.
2180  *
2181  *	Returns %NULL for an empty list or a pointer to the tail element.
2182  *	The reference count is not incremented and the reference is therefore
2183  *	volatile. Use with caution.
2184  */
skb_peek_tail(const struct sk_buff_head * list_)2185 static inline struct sk_buff *skb_peek_tail(const struct sk_buff_head *list_)
2186 {
2187 	struct sk_buff *skb = READ_ONCE(list_->prev);
2188 
2189 	if (skb == (struct sk_buff *)list_)
2190 		skb = NULL;
2191 	return skb;
2192 
2193 }
2194 
2195 /**
2196  *	skb_queue_len	- get queue length
2197  *	@list_: list to measure
2198  *
2199  *	Return the length of an &sk_buff queue.
2200  */
skb_queue_len(const struct sk_buff_head * list_)2201 static inline __u32 skb_queue_len(const struct sk_buff_head *list_)
2202 {
2203 	return list_->qlen;
2204 }
2205 
2206 /**
2207  *	skb_queue_len_lockless	- get queue length
2208  *	@list_: list to measure
2209  *
2210  *	Return the length of an &sk_buff queue.
2211  *	This variant can be used in lockless contexts.
2212  */
skb_queue_len_lockless(const struct sk_buff_head * list_)2213 static inline __u32 skb_queue_len_lockless(const struct sk_buff_head *list_)
2214 {
2215 	return READ_ONCE(list_->qlen);
2216 }
2217 
2218 /**
2219  *	__skb_queue_head_init - initialize non-spinlock portions of sk_buff_head
2220  *	@list: queue to initialize
2221  *
2222  *	This initializes only the list and queue length aspects of
2223  *	an sk_buff_head object.  This allows to initialize the list
2224  *	aspects of an sk_buff_head without reinitializing things like
2225  *	the spinlock.  It can also be used for on-stack sk_buff_head
2226  *	objects where the spinlock is known to not be used.
2227  */
__skb_queue_head_init(struct sk_buff_head * list)2228 static inline void __skb_queue_head_init(struct sk_buff_head *list)
2229 {
2230 	list->prev = list->next = (struct sk_buff *)list;
2231 	list->qlen = 0;
2232 }
2233 
2234 /*
2235  * This function creates a split out lock class for each invocation;
2236  * this is needed for now since a whole lot of users of the skb-queue
2237  * infrastructure in drivers have different locking usage (in hardirq)
2238  * than the networking core (in softirq only). In the long run either the
2239  * network layer or drivers should need annotation to consolidate the
2240  * main types of usage into 3 classes.
2241  */
skb_queue_head_init(struct sk_buff_head * list)2242 static inline void skb_queue_head_init(struct sk_buff_head *list)
2243 {
2244 	spin_lock_init(&list->lock);
2245 	__skb_queue_head_init(list);
2246 }
2247 
skb_queue_head_init_class(struct sk_buff_head * list,struct lock_class_key * class)2248 static inline void skb_queue_head_init_class(struct sk_buff_head *list,
2249 		struct lock_class_key *class)
2250 {
2251 	skb_queue_head_init(list);
2252 	lockdep_set_class(&list->lock, class);
2253 }
2254 
2255 /*
2256  *	Insert an sk_buff on a list.
2257  *
2258  *	The "__skb_xxxx()" functions are the non-atomic ones that
2259  *	can only be called with interrupts disabled.
2260  */
__skb_insert(struct sk_buff * newsk,struct sk_buff * prev,struct sk_buff * next,struct sk_buff_head * list)2261 static inline void __skb_insert(struct sk_buff *newsk,
2262 				struct sk_buff *prev, struct sk_buff *next,
2263 				struct sk_buff_head *list)
2264 {
2265 	/* See skb_queue_empty_lockless() and skb_peek_tail()
2266 	 * for the opposite READ_ONCE()
2267 	 */
2268 	WRITE_ONCE(newsk->next, next);
2269 	WRITE_ONCE(newsk->prev, prev);
2270 	WRITE_ONCE(((struct sk_buff_list *)next)->prev, newsk);
2271 	WRITE_ONCE(((struct sk_buff_list *)prev)->next, newsk);
2272 	WRITE_ONCE(list->qlen, list->qlen + 1);
2273 }
2274 
__skb_queue_splice(const struct sk_buff_head * list,struct sk_buff * prev,struct sk_buff * next)2275 static inline void __skb_queue_splice(const struct sk_buff_head *list,
2276 				      struct sk_buff *prev,
2277 				      struct sk_buff *next)
2278 {
2279 	struct sk_buff *first = list->next;
2280 	struct sk_buff *last = list->prev;
2281 
2282 	WRITE_ONCE(first->prev, prev);
2283 	WRITE_ONCE(prev->next, first);
2284 
2285 	WRITE_ONCE(last->next, next);
2286 	WRITE_ONCE(next->prev, last);
2287 }
2288 
2289 /**
2290  *	skb_queue_splice - join two skb lists, this is designed for stacks
2291  *	@list: the new list to add
2292  *	@head: the place to add it in the first list
2293  */
skb_queue_splice(const struct sk_buff_head * list,struct sk_buff_head * head)2294 static inline void skb_queue_splice(const struct sk_buff_head *list,
2295 				    struct sk_buff_head *head)
2296 {
2297 	if (!skb_queue_empty(list)) {
2298 		__skb_queue_splice(list, (struct sk_buff *) head, head->next);
2299 		head->qlen += list->qlen;
2300 	}
2301 }
2302 
2303 /**
2304  *	skb_queue_splice_init - join two skb lists and reinitialise the emptied list
2305  *	@list: the new list to add
2306  *	@head: the place to add it in the first list
2307  *
2308  *	The list at @list is reinitialised
2309  */
skb_queue_splice_init(struct sk_buff_head * list,struct sk_buff_head * head)2310 static inline void skb_queue_splice_init(struct sk_buff_head *list,
2311 					 struct sk_buff_head *head)
2312 {
2313 	if (!skb_queue_empty(list)) {
2314 		__skb_queue_splice(list, (struct sk_buff *) head, head->next);
2315 		head->qlen += list->qlen;
2316 		__skb_queue_head_init(list);
2317 	}
2318 }
2319 
2320 /**
2321  *	skb_queue_splice_tail - join two skb lists, each list being a queue
2322  *	@list: the new list to add
2323  *	@head: the place to add it in the first list
2324  */
skb_queue_splice_tail(const struct sk_buff_head * list,struct sk_buff_head * head)2325 static inline void skb_queue_splice_tail(const struct sk_buff_head *list,
2326 					 struct sk_buff_head *head)
2327 {
2328 	if (!skb_queue_empty(list)) {
2329 		__skb_queue_splice(list, head->prev, (struct sk_buff *) head);
2330 		head->qlen += list->qlen;
2331 	}
2332 }
2333 
2334 /**
2335  *	skb_queue_splice_tail_init - join two skb lists and reinitialise the emptied list
2336  *	@list: the new list to add
2337  *	@head: the place to add it in the first list
2338  *
2339  *	Each of the lists is a queue.
2340  *	The list at @list is reinitialised
2341  */
skb_queue_splice_tail_init(struct sk_buff_head * list,struct sk_buff_head * head)2342 static inline void skb_queue_splice_tail_init(struct sk_buff_head *list,
2343 					      struct sk_buff_head *head)
2344 {
2345 	if (!skb_queue_empty(list)) {
2346 		__skb_queue_splice(list, head->prev, (struct sk_buff *) head);
2347 		head->qlen += list->qlen;
2348 		__skb_queue_head_init(list);
2349 	}
2350 }
2351 
2352 /**
2353  *	__skb_queue_after - queue a buffer at the list head
2354  *	@list: list to use
2355  *	@prev: place after this buffer
2356  *	@newsk: buffer to queue
2357  *
2358  *	Queue a buffer int the middle of a list. This function takes no locks
2359  *	and you must therefore hold required locks before calling it.
2360  *
2361  *	A buffer cannot be placed on two lists at the same time.
2362  */
__skb_queue_after(struct sk_buff_head * list,struct sk_buff * prev,struct sk_buff * newsk)2363 static inline void __skb_queue_after(struct sk_buff_head *list,
2364 				     struct sk_buff *prev,
2365 				     struct sk_buff *newsk)
2366 {
2367 	__skb_insert(newsk, prev, ((struct sk_buff_list *)prev)->next, list);
2368 }
2369 
2370 void skb_append(struct sk_buff *old, struct sk_buff *newsk,
2371 		struct sk_buff_head *list);
2372 
__skb_queue_before(struct sk_buff_head * list,struct sk_buff * next,struct sk_buff * newsk)2373 static inline void __skb_queue_before(struct sk_buff_head *list,
2374 				      struct sk_buff *next,
2375 				      struct sk_buff *newsk)
2376 {
2377 	__skb_insert(newsk, ((struct sk_buff_list *)next)->prev, next, list);
2378 }
2379 
2380 /**
2381  *	__skb_queue_head - queue a buffer at the list head
2382  *	@list: list to use
2383  *	@newsk: buffer to queue
2384  *
2385  *	Queue a buffer at the start of a list. This function takes no locks
2386  *	and you must therefore hold required locks before calling it.
2387  *
2388  *	A buffer cannot be placed on two lists at the same time.
2389  */
__skb_queue_head(struct sk_buff_head * list,struct sk_buff * newsk)2390 static inline void __skb_queue_head(struct sk_buff_head *list,
2391 				    struct sk_buff *newsk)
2392 {
2393 	__skb_queue_after(list, (struct sk_buff *)list, newsk);
2394 }
2395 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk);
2396 
2397 /**
2398  *	__skb_queue_tail - queue a buffer at the list tail
2399  *	@list: list to use
2400  *	@newsk: buffer to queue
2401  *
2402  *	Queue a buffer at the end of a list. This function takes no locks
2403  *	and you must therefore hold required locks before calling it.
2404  *
2405  *	A buffer cannot be placed on two lists at the same time.
2406  */
__skb_queue_tail(struct sk_buff_head * list,struct sk_buff * newsk)2407 static inline void __skb_queue_tail(struct sk_buff_head *list,
2408 				   struct sk_buff *newsk)
2409 {
2410 	__skb_queue_before(list, (struct sk_buff *)list, newsk);
2411 }
2412 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk);
2413 
2414 /*
2415  * remove sk_buff from list. _Must_ be called atomically, and with
2416  * the list known..
2417  */
2418 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list);
__skb_unlink(struct sk_buff * skb,struct sk_buff_head * list)2419 static inline void __skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
2420 {
2421 	struct sk_buff *next, *prev;
2422 
2423 	WRITE_ONCE(list->qlen, list->qlen - 1);
2424 	next	   = skb->next;
2425 	prev	   = skb->prev;
2426 	skb->next  = skb->prev = NULL;
2427 	WRITE_ONCE(next->prev, prev);
2428 	WRITE_ONCE(prev->next, next);
2429 }
2430 
2431 /**
2432  *	__skb_dequeue - remove from the head of the queue
2433  *	@list: list to dequeue from
2434  *
2435  *	Remove the head of the list. This function does not take any locks
2436  *	so must be used with appropriate locks held only. The head item is
2437  *	returned or %NULL if the list is empty.
2438  */
__skb_dequeue(struct sk_buff_head * list)2439 static inline struct sk_buff *__skb_dequeue(struct sk_buff_head *list)
2440 {
2441 	struct sk_buff *skb = skb_peek(list);
2442 	if (skb)
2443 		__skb_unlink(skb, list);
2444 	return skb;
2445 }
2446 struct sk_buff *skb_dequeue(struct sk_buff_head *list);
2447 
2448 /**
2449  *	__skb_dequeue_tail - remove from the tail of the queue
2450  *	@list: list to dequeue from
2451  *
2452  *	Remove the tail of the list. This function does not take any locks
2453  *	so must be used with appropriate locks held only. The tail item is
2454  *	returned or %NULL if the list is empty.
2455  */
__skb_dequeue_tail(struct sk_buff_head * list)2456 static inline struct sk_buff *__skb_dequeue_tail(struct sk_buff_head *list)
2457 {
2458 	struct sk_buff *skb = skb_peek_tail(list);
2459 	if (skb)
2460 		__skb_unlink(skb, list);
2461 	return skb;
2462 }
2463 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list);
2464 
2465 
skb_is_nonlinear(const struct sk_buff * skb)2466 static inline bool skb_is_nonlinear(const struct sk_buff *skb)
2467 {
2468 	return skb->data_len;
2469 }
2470 
skb_headlen(const struct sk_buff * skb)2471 static inline unsigned int skb_headlen(const struct sk_buff *skb)
2472 {
2473 	return skb->len - skb->data_len;
2474 }
2475 
__skb_pagelen(const struct sk_buff * skb)2476 static inline unsigned int __skb_pagelen(const struct sk_buff *skb)
2477 {
2478 	unsigned int i, len = 0;
2479 
2480 	for (i = skb_shinfo(skb)->nr_frags - 1; (int)i >= 0; i--)
2481 		len += skb_frag_size(&skb_shinfo(skb)->frags[i]);
2482 	return len;
2483 }
2484 
skb_pagelen(const struct sk_buff * skb)2485 static inline unsigned int skb_pagelen(const struct sk_buff *skb)
2486 {
2487 	return skb_headlen(skb) + __skb_pagelen(skb);
2488 }
2489 
skb_frag_fill_netmem_desc(skb_frag_t * frag,netmem_ref netmem,int off,int size)2490 static inline void skb_frag_fill_netmem_desc(skb_frag_t *frag,
2491 					     netmem_ref netmem, int off,
2492 					     int size)
2493 {
2494 	frag->netmem = netmem;
2495 	frag->offset = off;
2496 	skb_frag_size_set(frag, size);
2497 }
2498 
skb_frag_fill_page_desc(skb_frag_t * frag,struct page * page,int off,int size)2499 static inline void skb_frag_fill_page_desc(skb_frag_t *frag,
2500 					   struct page *page,
2501 					   int off, int size)
2502 {
2503 	skb_frag_fill_netmem_desc(frag, page_to_netmem(page), off, size);
2504 }
2505 
__skb_fill_netmem_desc_noacc(struct skb_shared_info * shinfo,int i,netmem_ref netmem,int off,int size)2506 static inline void __skb_fill_netmem_desc_noacc(struct skb_shared_info *shinfo,
2507 						int i, netmem_ref netmem,
2508 						int off, int size)
2509 {
2510 	skb_frag_t *frag = &shinfo->frags[i];
2511 
2512 	skb_frag_fill_netmem_desc(frag, netmem, off, size);
2513 }
2514 
__skb_fill_page_desc_noacc(struct skb_shared_info * shinfo,int i,struct page * page,int off,int size)2515 static inline void __skb_fill_page_desc_noacc(struct skb_shared_info *shinfo,
2516 					      int i, struct page *page,
2517 					      int off, int size)
2518 {
2519 	__skb_fill_netmem_desc_noacc(shinfo, i, page_to_netmem(page), off,
2520 				     size);
2521 }
2522 
2523 /**
2524  * skb_len_add - adds a number to len fields of skb
2525  * @skb: buffer to add len to
2526  * @delta: number of bytes to add
2527  */
skb_len_add(struct sk_buff * skb,int delta)2528 static inline void skb_len_add(struct sk_buff *skb, int delta)
2529 {
2530 	skb->len += delta;
2531 	skb->data_len += delta;
2532 	skb->truesize += delta;
2533 }
2534 
2535 /**
2536  * __skb_fill_netmem_desc - initialise a fragment in an skb
2537  * @skb: buffer containing fragment to be initialised
2538  * @i: fragment index to initialise
2539  * @netmem: the netmem to use for this fragment
2540  * @off: the offset to the data with @page
2541  * @size: the length of the data
2542  *
2543  * Initialises the @i'th fragment of @skb to point to &size bytes at
2544  * offset @off within @page.
2545  *
2546  * Does not take any additional reference on the fragment.
2547  */
__skb_fill_netmem_desc(struct sk_buff * skb,int i,netmem_ref netmem,int off,int size)2548 static inline void __skb_fill_netmem_desc(struct sk_buff *skb, int i,
2549 					  netmem_ref netmem, int off, int size)
2550 {
2551 	struct page *page;
2552 
2553 	__skb_fill_netmem_desc_noacc(skb_shinfo(skb), i, netmem, off, size);
2554 
2555 	if (netmem_is_net_iov(netmem)) {
2556 		skb->unreadable = true;
2557 		return;
2558 	}
2559 
2560 	page = netmem_to_page(netmem);
2561 
2562 	/* Propagate page pfmemalloc to the skb if we can. The problem is
2563 	 * that not all callers have unique ownership of the page but rely
2564 	 * on page_is_pfmemalloc doing the right thing(tm).
2565 	 */
2566 	page = compound_head(page);
2567 	if (page_is_pfmemalloc(page))
2568 		skb->pfmemalloc = true;
2569 }
2570 
__skb_fill_page_desc(struct sk_buff * skb,int i,struct page * page,int off,int size)2571 static inline void __skb_fill_page_desc(struct sk_buff *skb, int i,
2572 					struct page *page, int off, int size)
2573 {
2574 	__skb_fill_netmem_desc(skb, i, page_to_netmem(page), off, size);
2575 }
2576 
skb_fill_netmem_desc(struct sk_buff * skb,int i,netmem_ref netmem,int off,int size)2577 static inline void skb_fill_netmem_desc(struct sk_buff *skb, int i,
2578 					netmem_ref netmem, int off, int size)
2579 {
2580 	__skb_fill_netmem_desc(skb, i, netmem, off, size);
2581 	skb_shinfo(skb)->nr_frags = i + 1;
2582 }
2583 
2584 /**
2585  * skb_fill_page_desc - initialise a paged fragment in an skb
2586  * @skb: buffer containing fragment to be initialised
2587  * @i: paged fragment index to initialise
2588  * @page: the page to use for this fragment
2589  * @off: the offset to the data with @page
2590  * @size: the length of the data
2591  *
2592  * As per __skb_fill_page_desc() -- initialises the @i'th fragment of
2593  * @skb to point to @size bytes at offset @off within @page. In
2594  * addition updates @skb such that @i is the last fragment.
2595  *
2596  * Does not take any additional reference on the fragment.
2597  */
skb_fill_page_desc(struct sk_buff * skb,int i,struct page * page,int off,int size)2598 static inline void skb_fill_page_desc(struct sk_buff *skb, int i,
2599 				      struct page *page, int off, int size)
2600 {
2601 	skb_fill_netmem_desc(skb, i, page_to_netmem(page), off, size);
2602 }
2603 
2604 /**
2605  * skb_fill_page_desc_noacc - initialise a paged fragment in an skb
2606  * @skb: buffer containing fragment to be initialised
2607  * @i: paged fragment index to initialise
2608  * @page: the page to use for this fragment
2609  * @off: the offset to the data with @page
2610  * @size: the length of the data
2611  *
2612  * Variant of skb_fill_page_desc() which does not deal with
2613  * pfmemalloc, if page is not owned by us.
2614  */
skb_fill_page_desc_noacc(struct sk_buff * skb,int i,struct page * page,int off,int size)2615 static inline void skb_fill_page_desc_noacc(struct sk_buff *skb, int i,
2616 					    struct page *page, int off,
2617 					    int size)
2618 {
2619 	struct skb_shared_info *shinfo = skb_shinfo(skb);
2620 
2621 	__skb_fill_page_desc_noacc(shinfo, i, page, off, size);
2622 	shinfo->nr_frags = i + 1;
2623 }
2624 
2625 void skb_add_rx_frag_netmem(struct sk_buff *skb, int i, netmem_ref netmem,
2626 			    int off, int size, unsigned int truesize);
2627 
skb_add_rx_frag(struct sk_buff * skb,int i,struct page * page,int off,int size,unsigned int truesize)2628 static inline void skb_add_rx_frag(struct sk_buff *skb, int i,
2629 				   struct page *page, int off, int size,
2630 				   unsigned int truesize)
2631 {
2632 	skb_add_rx_frag_netmem(skb, i, page_to_netmem(page), off, size,
2633 			       truesize);
2634 }
2635 
2636 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
2637 			  unsigned int truesize);
2638 
2639 #define SKB_LINEAR_ASSERT(skb)  BUG_ON(skb_is_nonlinear(skb))
2640 
2641 #ifdef NET_SKBUFF_DATA_USES_OFFSET
skb_tail_pointer(const struct sk_buff * skb)2642 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
2643 {
2644 	return skb->head + skb->tail;
2645 }
2646 
skb_reset_tail_pointer(struct sk_buff * skb)2647 static inline void skb_reset_tail_pointer(struct sk_buff *skb)
2648 {
2649 	skb->tail = skb->data - skb->head;
2650 }
2651 
skb_set_tail_pointer(struct sk_buff * skb,const int offset)2652 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
2653 {
2654 	skb_reset_tail_pointer(skb);
2655 	skb->tail += offset;
2656 }
2657 
2658 #else /* NET_SKBUFF_DATA_USES_OFFSET */
skb_tail_pointer(const struct sk_buff * skb)2659 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
2660 {
2661 	return skb->tail;
2662 }
2663 
skb_reset_tail_pointer(struct sk_buff * skb)2664 static inline void skb_reset_tail_pointer(struct sk_buff *skb)
2665 {
2666 	skb->tail = skb->data;
2667 }
2668 
skb_set_tail_pointer(struct sk_buff * skb,const int offset)2669 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
2670 {
2671 	skb->tail = skb->data + offset;
2672 }
2673 
2674 #endif /* NET_SKBUFF_DATA_USES_OFFSET */
2675 
skb_assert_len(struct sk_buff * skb)2676 static inline void skb_assert_len(struct sk_buff *skb)
2677 {
2678 #ifdef CONFIG_DEBUG_NET
2679 	if (WARN_ONCE(!skb->len, "%s\n", __func__))
2680 		DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
2681 #endif /* CONFIG_DEBUG_NET */
2682 }
2683 
2684 /*
2685  *	Add data to an sk_buff
2686  */
2687 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len);
2688 void *skb_put(struct sk_buff *skb, unsigned int len);
__skb_put(struct sk_buff * skb,unsigned int len)2689 static inline void *__skb_put(struct sk_buff *skb, unsigned int len)
2690 {
2691 	void *tmp = skb_tail_pointer(skb);
2692 	SKB_LINEAR_ASSERT(skb);
2693 	skb->tail += len;
2694 	skb->len  += len;
2695 	return tmp;
2696 }
2697 
__skb_put_zero(struct sk_buff * skb,unsigned int len)2698 static inline void *__skb_put_zero(struct sk_buff *skb, unsigned int len)
2699 {
2700 	void *tmp = __skb_put(skb, len);
2701 
2702 	memset(tmp, 0, len);
2703 	return tmp;
2704 }
2705 
__skb_put_data(struct sk_buff * skb,const void * data,unsigned int len)2706 static inline void *__skb_put_data(struct sk_buff *skb, const void *data,
2707 				   unsigned int len)
2708 {
2709 	void *tmp = __skb_put(skb, len);
2710 
2711 	memcpy(tmp, data, len);
2712 	return tmp;
2713 }
2714 
__skb_put_u8(struct sk_buff * skb,u8 val)2715 static inline void __skb_put_u8(struct sk_buff *skb, u8 val)
2716 {
2717 	*(u8 *)__skb_put(skb, 1) = val;
2718 }
2719 
skb_put_zero(struct sk_buff * skb,unsigned int len)2720 static inline void *skb_put_zero(struct sk_buff *skb, unsigned int len)
2721 {
2722 	void *tmp = skb_put(skb, len);
2723 
2724 	memset(tmp, 0, len);
2725 
2726 	return tmp;
2727 }
2728 
skb_put_data(struct sk_buff * skb,const void * data,unsigned int len)2729 static inline void *skb_put_data(struct sk_buff *skb, const void *data,
2730 				 unsigned int len)
2731 {
2732 	void *tmp = skb_put(skb, len);
2733 
2734 	memcpy(tmp, data, len);
2735 
2736 	return tmp;
2737 }
2738 
skb_put_u8(struct sk_buff * skb,u8 val)2739 static inline void skb_put_u8(struct sk_buff *skb, u8 val)
2740 {
2741 	*(u8 *)skb_put(skb, 1) = val;
2742 }
2743 
2744 void *skb_push(struct sk_buff *skb, unsigned int len);
__skb_push(struct sk_buff * skb,unsigned int len)2745 static inline void *__skb_push(struct sk_buff *skb, unsigned int len)
2746 {
2747 	DEBUG_NET_WARN_ON_ONCE(len > INT_MAX);
2748 
2749 	skb->data -= len;
2750 	skb->len  += len;
2751 	return skb->data;
2752 }
2753 
2754 void *skb_pull(struct sk_buff *skb, unsigned int len);
__skb_pull(struct sk_buff * skb,unsigned int len)2755 static inline void *__skb_pull(struct sk_buff *skb, unsigned int len)
2756 {
2757 	DEBUG_NET_WARN_ON_ONCE(len > INT_MAX);
2758 
2759 	skb->len -= len;
2760 	if (unlikely(skb->len < skb->data_len)) {
2761 #if defined(CONFIG_DEBUG_NET)
2762 		skb->len += len;
2763 		pr_err("__skb_pull(len=%u)\n", len);
2764 		skb_dump(KERN_ERR, skb, false);
2765 #endif
2766 		BUG();
2767 	}
2768 	return skb->data += len;
2769 }
2770 
skb_pull_inline(struct sk_buff * skb,unsigned int len)2771 static inline void *skb_pull_inline(struct sk_buff *skb, unsigned int len)
2772 {
2773 	return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len);
2774 }
2775 
2776 void *skb_pull_data(struct sk_buff *skb, size_t len);
2777 
2778 void *__pskb_pull_tail(struct sk_buff *skb, int delta);
2779 
2780 static inline enum skb_drop_reason
pskb_may_pull_reason(struct sk_buff * skb,unsigned int len)2781 pskb_may_pull_reason(struct sk_buff *skb, unsigned int len)
2782 {
2783 	DEBUG_NET_WARN_ON_ONCE(len > INT_MAX);
2784 
2785 	if (likely(len <= skb_headlen(skb)))
2786 		return SKB_NOT_DROPPED_YET;
2787 
2788 	if (unlikely(len > skb->len))
2789 		return SKB_DROP_REASON_PKT_TOO_SMALL;
2790 
2791 	if (unlikely(!__pskb_pull_tail(skb, len - skb_headlen(skb))))
2792 		return SKB_DROP_REASON_NOMEM;
2793 
2794 	return SKB_NOT_DROPPED_YET;
2795 }
2796 
pskb_may_pull(struct sk_buff * skb,unsigned int len)2797 static inline bool pskb_may_pull(struct sk_buff *skb, unsigned int len)
2798 {
2799 	return pskb_may_pull_reason(skb, len) == SKB_NOT_DROPPED_YET;
2800 }
2801 
pskb_pull(struct sk_buff * skb,unsigned int len)2802 static inline void *pskb_pull(struct sk_buff *skb, unsigned int len)
2803 {
2804 	if (!pskb_may_pull(skb, len))
2805 		return NULL;
2806 
2807 	skb->len -= len;
2808 	return skb->data += len;
2809 }
2810 
2811 void skb_condense(struct sk_buff *skb);
2812 
2813 /**
2814  *	skb_headroom - bytes at buffer head
2815  *	@skb: buffer to check
2816  *
2817  *	Return the number of bytes of free space at the head of an &sk_buff.
2818  */
skb_headroom(const struct sk_buff * skb)2819 static inline unsigned int skb_headroom(const struct sk_buff *skb)
2820 {
2821 	return skb->data - skb->head;
2822 }
2823 
2824 /**
2825  *	skb_tailroom - bytes at buffer end
2826  *	@skb: buffer to check
2827  *
2828  *	Return the number of bytes of free space at the tail of an sk_buff
2829  */
skb_tailroom(const struct sk_buff * skb)2830 static inline int skb_tailroom(const struct sk_buff *skb)
2831 {
2832 	return skb_is_nonlinear(skb) ? 0 : skb->end - skb->tail;
2833 }
2834 
2835 /**
2836  *	skb_availroom - bytes at buffer end
2837  *	@skb: buffer to check
2838  *
2839  *	Return the number of bytes of free space at the tail of an sk_buff
2840  *	allocated by sk_stream_alloc()
2841  */
skb_availroom(const struct sk_buff * skb)2842 static inline int skb_availroom(const struct sk_buff *skb)
2843 {
2844 	if (skb_is_nonlinear(skb))
2845 		return 0;
2846 
2847 	return skb->end - skb->tail - skb->reserved_tailroom;
2848 }
2849 
2850 /**
2851  *	skb_reserve - adjust headroom
2852  *	@skb: buffer to alter
2853  *	@len: bytes to move
2854  *
2855  *	Increase the headroom of an empty &sk_buff by reducing the tail
2856  *	room. This is only allowed for an empty buffer.
2857  */
skb_reserve(struct sk_buff * skb,int len)2858 static inline void skb_reserve(struct sk_buff *skb, int len)
2859 {
2860 	skb->data += len;
2861 	skb->tail += len;
2862 }
2863 
2864 /**
2865  *	skb_tailroom_reserve - adjust reserved_tailroom
2866  *	@skb: buffer to alter
2867  *	@mtu: maximum amount of headlen permitted
2868  *	@needed_tailroom: minimum amount of reserved_tailroom
2869  *
2870  *	Set reserved_tailroom so that headlen can be as large as possible but
2871  *	not larger than mtu and tailroom cannot be smaller than
2872  *	needed_tailroom.
2873  *	The required headroom should already have been reserved before using
2874  *	this function.
2875  */
skb_tailroom_reserve(struct sk_buff * skb,unsigned int mtu,unsigned int needed_tailroom)2876 static inline void skb_tailroom_reserve(struct sk_buff *skb, unsigned int mtu,
2877 					unsigned int needed_tailroom)
2878 {
2879 	SKB_LINEAR_ASSERT(skb);
2880 	if (mtu < skb_tailroom(skb) - needed_tailroom)
2881 		/* use at most mtu */
2882 		skb->reserved_tailroom = skb_tailroom(skb) - mtu;
2883 	else
2884 		/* use up to all available space */
2885 		skb->reserved_tailroom = needed_tailroom;
2886 }
2887 
2888 #define ENCAP_TYPE_ETHER	0
2889 #define ENCAP_TYPE_IPPROTO	1
2890 
skb_set_inner_protocol(struct sk_buff * skb,__be16 protocol)2891 static inline void skb_set_inner_protocol(struct sk_buff *skb,
2892 					  __be16 protocol)
2893 {
2894 	skb->inner_protocol = protocol;
2895 	skb->inner_protocol_type = ENCAP_TYPE_ETHER;
2896 }
2897 
skb_set_inner_ipproto(struct sk_buff * skb,__u8 ipproto)2898 static inline void skb_set_inner_ipproto(struct sk_buff *skb,
2899 					 __u8 ipproto)
2900 {
2901 	skb->inner_ipproto = ipproto;
2902 	skb->inner_protocol_type = ENCAP_TYPE_IPPROTO;
2903 }
2904 
skb_reset_inner_headers(struct sk_buff * skb)2905 static inline void skb_reset_inner_headers(struct sk_buff *skb)
2906 {
2907 	skb->inner_mac_header = skb->mac_header;
2908 	skb->inner_network_header = skb->network_header;
2909 	skb->inner_transport_header = skb->transport_header;
2910 }
2911 
skb_reset_mac_len(struct sk_buff * skb)2912 static inline void skb_reset_mac_len(struct sk_buff *skb)
2913 {
2914 	skb->mac_len = skb->network_header - skb->mac_header;
2915 }
2916 
skb_inner_transport_header(const struct sk_buff * skb)2917 static inline unsigned char *skb_inner_transport_header(const struct sk_buff
2918 							*skb)
2919 {
2920 	return skb->head + skb->inner_transport_header;
2921 }
2922 
skb_inner_transport_offset(const struct sk_buff * skb)2923 static inline int skb_inner_transport_offset(const struct sk_buff *skb)
2924 {
2925 	return skb_inner_transport_header(skb) - skb->data;
2926 }
2927 
skb_reset_inner_transport_header(struct sk_buff * skb)2928 static inline void skb_reset_inner_transport_header(struct sk_buff *skb)
2929 {
2930 	skb->inner_transport_header = skb->data - skb->head;
2931 }
2932 
skb_set_inner_transport_header(struct sk_buff * skb,const int offset)2933 static inline void skb_set_inner_transport_header(struct sk_buff *skb,
2934 						   const int offset)
2935 {
2936 	skb_reset_inner_transport_header(skb);
2937 	skb->inner_transport_header += offset;
2938 }
2939 
skb_inner_network_header(const struct sk_buff * skb)2940 static inline unsigned char *skb_inner_network_header(const struct sk_buff *skb)
2941 {
2942 	return skb->head + skb->inner_network_header;
2943 }
2944 
skb_reset_inner_network_header(struct sk_buff * skb)2945 static inline void skb_reset_inner_network_header(struct sk_buff *skb)
2946 {
2947 	skb->inner_network_header = skb->data - skb->head;
2948 }
2949 
skb_set_inner_network_header(struct sk_buff * skb,const int offset)2950 static inline void skb_set_inner_network_header(struct sk_buff *skb,
2951 						const int offset)
2952 {
2953 	skb_reset_inner_network_header(skb);
2954 	skb->inner_network_header += offset;
2955 }
2956 
skb_inner_network_header_was_set(const struct sk_buff * skb)2957 static inline bool skb_inner_network_header_was_set(const struct sk_buff *skb)
2958 {
2959 	return skb->inner_network_header > 0;
2960 }
2961 
skb_inner_mac_header(const struct sk_buff * skb)2962 static inline unsigned char *skb_inner_mac_header(const struct sk_buff *skb)
2963 {
2964 	return skb->head + skb->inner_mac_header;
2965 }
2966 
skb_reset_inner_mac_header(struct sk_buff * skb)2967 static inline void skb_reset_inner_mac_header(struct sk_buff *skb)
2968 {
2969 	skb->inner_mac_header = skb->data - skb->head;
2970 }
2971 
skb_set_inner_mac_header(struct sk_buff * skb,const int offset)2972 static inline void skb_set_inner_mac_header(struct sk_buff *skb,
2973 					    const int offset)
2974 {
2975 	skb_reset_inner_mac_header(skb);
2976 	skb->inner_mac_header += offset;
2977 }
skb_transport_header_was_set(const struct sk_buff * skb)2978 static inline bool skb_transport_header_was_set(const struct sk_buff *skb)
2979 {
2980 	return skb->transport_header != (typeof(skb->transport_header))~0U;
2981 }
2982 
skb_transport_header(const struct sk_buff * skb)2983 static inline unsigned char *skb_transport_header(const struct sk_buff *skb)
2984 {
2985 	DEBUG_NET_WARN_ON_ONCE(!skb_transport_header_was_set(skb));
2986 	return skb->head + skb->transport_header;
2987 }
2988 
skb_reset_transport_header(struct sk_buff * skb)2989 static inline void skb_reset_transport_header(struct sk_buff *skb)
2990 {
2991 	skb->transport_header = skb->data - skb->head;
2992 }
2993 
skb_set_transport_header(struct sk_buff * skb,const int offset)2994 static inline void skb_set_transport_header(struct sk_buff *skb,
2995 					    const int offset)
2996 {
2997 	skb_reset_transport_header(skb);
2998 	skb->transport_header += offset;
2999 }
3000 
skb_network_header(const struct sk_buff * skb)3001 static inline unsigned char *skb_network_header(const struct sk_buff *skb)
3002 {
3003 	return skb->head + skb->network_header;
3004 }
3005 
skb_reset_network_header(struct sk_buff * skb)3006 static inline void skb_reset_network_header(struct sk_buff *skb)
3007 {
3008 	skb->network_header = skb->data - skb->head;
3009 }
3010 
skb_set_network_header(struct sk_buff * skb,const int offset)3011 static inline void skb_set_network_header(struct sk_buff *skb, const int offset)
3012 {
3013 	skb_reset_network_header(skb);
3014 	skb->network_header += offset;
3015 }
3016 
skb_mac_header_was_set(const struct sk_buff * skb)3017 static inline int skb_mac_header_was_set(const struct sk_buff *skb)
3018 {
3019 	return skb->mac_header != (typeof(skb->mac_header))~0U;
3020 }
3021 
skb_mac_header(const struct sk_buff * skb)3022 static inline unsigned char *skb_mac_header(const struct sk_buff *skb)
3023 {
3024 	DEBUG_NET_WARN_ON_ONCE(!skb_mac_header_was_set(skb));
3025 	return skb->head + skb->mac_header;
3026 }
3027 
skb_mac_offset(const struct sk_buff * skb)3028 static inline int skb_mac_offset(const struct sk_buff *skb)
3029 {
3030 	return skb_mac_header(skb) - skb->data;
3031 }
3032 
skb_mac_header_len(const struct sk_buff * skb)3033 static inline u32 skb_mac_header_len(const struct sk_buff *skb)
3034 {
3035 	DEBUG_NET_WARN_ON_ONCE(!skb_mac_header_was_set(skb));
3036 	return skb->network_header - skb->mac_header;
3037 }
3038 
skb_unset_mac_header(struct sk_buff * skb)3039 static inline void skb_unset_mac_header(struct sk_buff *skb)
3040 {
3041 	skb->mac_header = (typeof(skb->mac_header))~0U;
3042 }
3043 
skb_reset_mac_header(struct sk_buff * skb)3044 static inline void skb_reset_mac_header(struct sk_buff *skb)
3045 {
3046 	skb->mac_header = skb->data - skb->head;
3047 }
3048 
skb_set_mac_header(struct sk_buff * skb,const int offset)3049 static inline void skb_set_mac_header(struct sk_buff *skb, const int offset)
3050 {
3051 	skb_reset_mac_header(skb);
3052 	skb->mac_header += offset;
3053 }
3054 
skb_pop_mac_header(struct sk_buff * skb)3055 static inline void skb_pop_mac_header(struct sk_buff *skb)
3056 {
3057 	skb->mac_header = skb->network_header;
3058 }
3059 
skb_probe_transport_header(struct sk_buff * skb)3060 static inline void skb_probe_transport_header(struct sk_buff *skb)
3061 {
3062 	struct flow_keys_basic keys;
3063 
3064 	if (skb_transport_header_was_set(skb))
3065 		return;
3066 
3067 	if (skb_flow_dissect_flow_keys_basic(NULL, skb, &keys,
3068 					     NULL, 0, 0, 0, 0))
3069 		skb_set_transport_header(skb, keys.control.thoff);
3070 }
3071 
skb_mac_header_rebuild(struct sk_buff * skb)3072 static inline void skb_mac_header_rebuild(struct sk_buff *skb)
3073 {
3074 	if (skb_mac_header_was_set(skb)) {
3075 		const unsigned char *old_mac = skb_mac_header(skb);
3076 
3077 		skb_set_mac_header(skb, -skb->mac_len);
3078 		memmove(skb_mac_header(skb), old_mac, skb->mac_len);
3079 	}
3080 }
3081 
3082 /* Move the full mac header up to current network_header.
3083  * Leaves skb->data pointing at offset skb->mac_len into the mac_header.
3084  * Must be provided the complete mac header length.
3085  */
skb_mac_header_rebuild_full(struct sk_buff * skb,u32 full_mac_len)3086 static inline void skb_mac_header_rebuild_full(struct sk_buff *skb, u32 full_mac_len)
3087 {
3088 	if (skb_mac_header_was_set(skb)) {
3089 		const unsigned char *old_mac = skb_mac_header(skb);
3090 
3091 		skb_set_mac_header(skb, -full_mac_len);
3092 		memmove(skb_mac_header(skb), old_mac, full_mac_len);
3093 		__skb_push(skb, full_mac_len - skb->mac_len);
3094 	}
3095 }
3096 
skb_checksum_start_offset(const struct sk_buff * skb)3097 static inline int skb_checksum_start_offset(const struct sk_buff *skb)
3098 {
3099 	return skb->csum_start - skb_headroom(skb);
3100 }
3101 
skb_checksum_start(const struct sk_buff * skb)3102 static inline unsigned char *skb_checksum_start(const struct sk_buff *skb)
3103 {
3104 	return skb->head + skb->csum_start;
3105 }
3106 
skb_transport_offset(const struct sk_buff * skb)3107 static inline int skb_transport_offset(const struct sk_buff *skb)
3108 {
3109 	return skb_transport_header(skb) - skb->data;
3110 }
3111 
skb_network_header_len(const struct sk_buff * skb)3112 static inline u32 skb_network_header_len(const struct sk_buff *skb)
3113 {
3114 	DEBUG_NET_WARN_ON_ONCE(!skb_transport_header_was_set(skb));
3115 	return skb->transport_header - skb->network_header;
3116 }
3117 
skb_inner_network_header_len(const struct sk_buff * skb)3118 static inline u32 skb_inner_network_header_len(const struct sk_buff *skb)
3119 {
3120 	return skb->inner_transport_header - skb->inner_network_header;
3121 }
3122 
skb_network_offset(const struct sk_buff * skb)3123 static inline int skb_network_offset(const struct sk_buff *skb)
3124 {
3125 	return skb_network_header(skb) - skb->data;
3126 }
3127 
skb_inner_network_offset(const struct sk_buff * skb)3128 static inline int skb_inner_network_offset(const struct sk_buff *skb)
3129 {
3130 	return skb_inner_network_header(skb) - skb->data;
3131 }
3132 
pskb_network_may_pull(struct sk_buff * skb,unsigned int len)3133 static inline int pskb_network_may_pull(struct sk_buff *skb, unsigned int len)
3134 {
3135 	return pskb_may_pull(skb, skb_network_offset(skb) + len);
3136 }
3137 
3138 /*
3139  * CPUs often take a performance hit when accessing unaligned memory
3140  * locations. The actual performance hit varies, it can be small if the
3141  * hardware handles it or large if we have to take an exception and fix it
3142  * in software.
3143  *
3144  * Since an ethernet header is 14 bytes network drivers often end up with
3145  * the IP header at an unaligned offset. The IP header can be aligned by
3146  * shifting the start of the packet by 2 bytes. Drivers should do this
3147  * with:
3148  *
3149  * skb_reserve(skb, NET_IP_ALIGN);
3150  *
3151  * The downside to this alignment of the IP header is that the DMA is now
3152  * unaligned. On some architectures the cost of an unaligned DMA is high
3153  * and this cost outweighs the gains made by aligning the IP header.
3154  *
3155  * Since this trade off varies between architectures, we allow NET_IP_ALIGN
3156  * to be overridden.
3157  */
3158 #ifndef NET_IP_ALIGN
3159 #define NET_IP_ALIGN	2
3160 #endif
3161 
3162 /*
3163  * The networking layer reserves some headroom in skb data (via
3164  * dev_alloc_skb). This is used to avoid having to reallocate skb data when
3165  * the header has to grow. In the default case, if the header has to grow
3166  * 32 bytes or less we avoid the reallocation.
3167  *
3168  * Unfortunately this headroom changes the DMA alignment of the resulting
3169  * network packet. As for NET_IP_ALIGN, this unaligned DMA is expensive
3170  * on some architectures. An architecture can override this value,
3171  * perhaps setting it to a cacheline in size (since that will maintain
3172  * cacheline alignment of the DMA). It must be a power of 2.
3173  *
3174  * Various parts of the networking layer expect at least 32 bytes of
3175  * headroom, you should not reduce this.
3176  *
3177  * Using max(32, L1_CACHE_BYTES) makes sense (especially with RPS)
3178  * to reduce average number of cache lines per packet.
3179  * get_rps_cpu() for example only access one 64 bytes aligned block :
3180  * NET_IP_ALIGN(2) + ethernet_header(14) + IP_header(20/40) + ports(8)
3181  */
3182 #ifndef NET_SKB_PAD
3183 #define NET_SKB_PAD	max(32, L1_CACHE_BYTES)
3184 #endif
3185 
3186 int ___pskb_trim(struct sk_buff *skb, unsigned int len);
3187 
__skb_set_length(struct sk_buff * skb,unsigned int len)3188 static inline void __skb_set_length(struct sk_buff *skb, unsigned int len)
3189 {
3190 	if (WARN_ON(skb_is_nonlinear(skb)))
3191 		return;
3192 	skb->len = len;
3193 	skb_set_tail_pointer(skb, len);
3194 }
3195 
__skb_trim(struct sk_buff * skb,unsigned int len)3196 static inline void __skb_trim(struct sk_buff *skb, unsigned int len)
3197 {
3198 	__skb_set_length(skb, len);
3199 }
3200 
3201 void skb_trim(struct sk_buff *skb, unsigned int len);
3202 
__pskb_trim(struct sk_buff * skb,unsigned int len)3203 static inline int __pskb_trim(struct sk_buff *skb, unsigned int len)
3204 {
3205 	if (skb->data_len)
3206 		return ___pskb_trim(skb, len);
3207 	__skb_trim(skb, len);
3208 	return 0;
3209 }
3210 
pskb_trim(struct sk_buff * skb,unsigned int len)3211 static inline int pskb_trim(struct sk_buff *skb, unsigned int len)
3212 {
3213 	return (len < skb->len) ? __pskb_trim(skb, len) : 0;
3214 }
3215 
3216 /**
3217  *	pskb_trim_unique - remove end from a paged unique (not cloned) buffer
3218  *	@skb: buffer to alter
3219  *	@len: new length
3220  *
3221  *	This is identical to pskb_trim except that the caller knows that
3222  *	the skb is not cloned so we should never get an error due to out-
3223  *	of-memory.
3224  */
pskb_trim_unique(struct sk_buff * skb,unsigned int len)3225 static inline void pskb_trim_unique(struct sk_buff *skb, unsigned int len)
3226 {
3227 	int err = pskb_trim(skb, len);
3228 	BUG_ON(err);
3229 }
3230 
__skb_grow(struct sk_buff * skb,unsigned int len)3231 static inline int __skb_grow(struct sk_buff *skb, unsigned int len)
3232 {
3233 	unsigned int diff = len - skb->len;
3234 
3235 	if (skb_tailroom(skb) < diff) {
3236 		int ret = pskb_expand_head(skb, 0, diff - skb_tailroom(skb),
3237 					   GFP_ATOMIC);
3238 		if (ret)
3239 			return ret;
3240 	}
3241 	__skb_set_length(skb, len);
3242 	return 0;
3243 }
3244 
3245 /**
3246  *	skb_orphan - orphan a buffer
3247  *	@skb: buffer to orphan
3248  *
3249  *	If a buffer currently has an owner then we call the owner's
3250  *	destructor function and make the @skb unowned. The buffer continues
3251  *	to exist but is no longer charged to its former owner.
3252  */
skb_orphan(struct sk_buff * skb)3253 static inline void skb_orphan(struct sk_buff *skb)
3254 {
3255 	if (skb->destructor) {
3256 		skb->destructor(skb);
3257 		skb->destructor = NULL;
3258 		skb->sk		= NULL;
3259 	} else {
3260 		BUG_ON(skb->sk);
3261 	}
3262 }
3263 
3264 /**
3265  *	skb_orphan_frags - orphan the frags contained in a buffer
3266  *	@skb: buffer to orphan frags from
3267  *	@gfp_mask: allocation mask for replacement pages
3268  *
3269  *	For each frag in the SKB which needs a destructor (i.e. has an
3270  *	owner) create a copy of that frag and release the original
3271  *	page by calling the destructor.
3272  */
skb_orphan_frags(struct sk_buff * skb,gfp_t gfp_mask)3273 static inline int skb_orphan_frags(struct sk_buff *skb, gfp_t gfp_mask)
3274 {
3275 	if (likely(!skb_zcopy(skb)))
3276 		return 0;
3277 	if (skb_shinfo(skb)->flags & SKBFL_DONT_ORPHAN)
3278 		return 0;
3279 	return skb_copy_ubufs(skb, gfp_mask);
3280 }
3281 
3282 /* Frags must be orphaned, even if refcounted, if skb might loop to rx path */
skb_orphan_frags_rx(struct sk_buff * skb,gfp_t gfp_mask)3283 static inline int skb_orphan_frags_rx(struct sk_buff *skb, gfp_t gfp_mask)
3284 {
3285 	if (likely(!skb_zcopy(skb)))
3286 		return 0;
3287 	return skb_copy_ubufs(skb, gfp_mask);
3288 }
3289 
3290 /**
3291  *	__skb_queue_purge_reason - empty a list
3292  *	@list: list to empty
3293  *	@reason: drop reason
3294  *
3295  *	Delete all buffers on an &sk_buff list. Each buffer is removed from
3296  *	the list and one reference dropped. This function does not take the
3297  *	list lock and the caller must hold the relevant locks to use it.
3298  */
__skb_queue_purge_reason(struct sk_buff_head * list,enum skb_drop_reason reason)3299 static inline void __skb_queue_purge_reason(struct sk_buff_head *list,
3300 					    enum skb_drop_reason reason)
3301 {
3302 	struct sk_buff *skb;
3303 
3304 	while ((skb = __skb_dequeue(list)) != NULL)
3305 		kfree_skb_reason(skb, reason);
3306 }
3307 
__skb_queue_purge(struct sk_buff_head * list)3308 static inline void __skb_queue_purge(struct sk_buff_head *list)
3309 {
3310 	__skb_queue_purge_reason(list, SKB_DROP_REASON_QUEUE_PURGE);
3311 }
3312 
3313 void skb_queue_purge_reason(struct sk_buff_head *list,
3314 			    enum skb_drop_reason reason);
3315 
skb_queue_purge(struct sk_buff_head * list)3316 static inline void skb_queue_purge(struct sk_buff_head *list)
3317 {
3318 	skb_queue_purge_reason(list, SKB_DROP_REASON_QUEUE_PURGE);
3319 }
3320 
3321 unsigned int skb_rbtree_purge(struct rb_root *root);
3322 void skb_errqueue_purge(struct sk_buff_head *list);
3323 
3324 void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask);
3325 
3326 /**
3327  * netdev_alloc_frag - allocate a page fragment
3328  * @fragsz: fragment size
3329  *
3330  * Allocates a frag from a page for receive buffer.
3331  * Uses GFP_ATOMIC allocations.
3332  */
netdev_alloc_frag(unsigned int fragsz)3333 static inline void *netdev_alloc_frag(unsigned int fragsz)
3334 {
3335 	return __netdev_alloc_frag_align(fragsz, ~0u);
3336 }
3337 
netdev_alloc_frag_align(unsigned int fragsz,unsigned int align)3338 static inline void *netdev_alloc_frag_align(unsigned int fragsz,
3339 					    unsigned int align)
3340 {
3341 	WARN_ON_ONCE(!is_power_of_2(align));
3342 	return __netdev_alloc_frag_align(fragsz, -align);
3343 }
3344 
3345 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int length,
3346 				   gfp_t gfp_mask);
3347 
3348 /**
3349  *	netdev_alloc_skb - allocate an skbuff for rx on a specific device
3350  *	@dev: network device to receive on
3351  *	@length: length to allocate
3352  *
3353  *	Allocate a new &sk_buff and assign it a usage count of one. The
3354  *	buffer has unspecified headroom built in. Users should allocate
3355  *	the headroom they think they need without accounting for the
3356  *	built in space. The built in space is used for optimisations.
3357  *
3358  *	%NULL is returned if there is no free memory. Although this function
3359  *	allocates memory it can be called from an interrupt.
3360  */
netdev_alloc_skb(struct net_device * dev,unsigned int length)3361 static inline struct sk_buff *netdev_alloc_skb(struct net_device *dev,
3362 					       unsigned int length)
3363 {
3364 	return __netdev_alloc_skb(dev, length, GFP_ATOMIC);
3365 }
3366 
3367 /* legacy helper around __netdev_alloc_skb() */
__dev_alloc_skb(unsigned int length,gfp_t gfp_mask)3368 static inline struct sk_buff *__dev_alloc_skb(unsigned int length,
3369 					      gfp_t gfp_mask)
3370 {
3371 	return __netdev_alloc_skb(NULL, length, gfp_mask);
3372 }
3373 
3374 /* legacy helper around netdev_alloc_skb() */
dev_alloc_skb(unsigned int length)3375 static inline struct sk_buff *dev_alloc_skb(unsigned int length)
3376 {
3377 	return netdev_alloc_skb(NULL, length);
3378 }
3379 
3380 
__netdev_alloc_skb_ip_align(struct net_device * dev,unsigned int length,gfp_t gfp)3381 static inline struct sk_buff *__netdev_alloc_skb_ip_align(struct net_device *dev,
3382 		unsigned int length, gfp_t gfp)
3383 {
3384 	struct sk_buff *skb = __netdev_alloc_skb(dev, length + NET_IP_ALIGN, gfp);
3385 
3386 	if (NET_IP_ALIGN && skb)
3387 		skb_reserve(skb, NET_IP_ALIGN);
3388 	return skb;
3389 }
3390 
netdev_alloc_skb_ip_align(struct net_device * dev,unsigned int length)3391 static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev,
3392 		unsigned int length)
3393 {
3394 	return __netdev_alloc_skb_ip_align(dev, length, GFP_ATOMIC);
3395 }
3396 
skb_free_frag(void * addr)3397 static inline void skb_free_frag(void *addr)
3398 {
3399 	page_frag_free(addr);
3400 }
3401 
3402 void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask);
3403 
napi_alloc_frag(unsigned int fragsz)3404 static inline void *napi_alloc_frag(unsigned int fragsz)
3405 {
3406 	return __napi_alloc_frag_align(fragsz, ~0u);
3407 }
3408 
napi_alloc_frag_align(unsigned int fragsz,unsigned int align)3409 static inline void *napi_alloc_frag_align(unsigned int fragsz,
3410 					  unsigned int align)
3411 {
3412 	WARN_ON_ONCE(!is_power_of_2(align));
3413 	return __napi_alloc_frag_align(fragsz, -align);
3414 }
3415 
3416 struct sk_buff *napi_alloc_skb(struct napi_struct *napi, unsigned int length);
3417 void napi_consume_skb(struct sk_buff *skb, int budget);
3418 
3419 void napi_skb_free_stolen_head(struct sk_buff *skb);
3420 void __napi_kfree_skb(struct sk_buff *skb, enum skb_drop_reason reason);
3421 
3422 /**
3423  * __dev_alloc_pages - allocate page for network Rx
3424  * @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx
3425  * @order: size of the allocation
3426  *
3427  * Allocate a new page.
3428  *
3429  * %NULL is returned if there is no free memory.
3430 */
__dev_alloc_pages_noprof(gfp_t gfp_mask,unsigned int order)3431 static inline struct page *__dev_alloc_pages_noprof(gfp_t gfp_mask,
3432 					     unsigned int order)
3433 {
3434 	/* This piece of code contains several assumptions.
3435 	 * 1.  This is for device Rx, therefore a cold page is preferred.
3436 	 * 2.  The expectation is the user wants a compound page.
3437 	 * 3.  If requesting a order 0 page it will not be compound
3438 	 *     due to the check to see if order has a value in prep_new_page
3439 	 * 4.  __GFP_MEMALLOC is ignored if __GFP_NOMEMALLOC is set due to
3440 	 *     code in gfp_to_alloc_flags that should be enforcing this.
3441 	 */
3442 	gfp_mask |= __GFP_COMP | __GFP_MEMALLOC;
3443 
3444 	return alloc_pages_node_noprof(NUMA_NO_NODE, gfp_mask, order);
3445 }
3446 #define __dev_alloc_pages(...)	alloc_hooks(__dev_alloc_pages_noprof(__VA_ARGS__))
3447 
3448 /*
3449  * This specialized allocator has to be a macro for its allocations to be
3450  * accounted separately (to have a separate alloc_tag).
3451  */
3452 #define dev_alloc_pages(_order) __dev_alloc_pages(GFP_ATOMIC | __GFP_NOWARN, _order)
3453 
3454 /**
3455  * __dev_alloc_page - allocate a page for network Rx
3456  * @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx
3457  *
3458  * Allocate a new page.
3459  *
3460  * %NULL is returned if there is no free memory.
3461  */
__dev_alloc_page_noprof(gfp_t gfp_mask)3462 static inline struct page *__dev_alloc_page_noprof(gfp_t gfp_mask)
3463 {
3464 	return __dev_alloc_pages_noprof(gfp_mask, 0);
3465 }
3466 #define __dev_alloc_page(...)	alloc_hooks(__dev_alloc_page_noprof(__VA_ARGS__))
3467 
3468 /*
3469  * This specialized allocator has to be a macro for its allocations to be
3470  * accounted separately (to have a separate alloc_tag).
3471  */
3472 #define dev_alloc_page()	dev_alloc_pages(0)
3473 
3474 /**
3475  * dev_page_is_reusable - check whether a page can be reused for network Rx
3476  * @page: the page to test
3477  *
3478  * A page shouldn't be considered for reusing/recycling if it was allocated
3479  * under memory pressure or at a distant memory node.
3480  *
3481  * Returns false if this page should be returned to page allocator, true
3482  * otherwise.
3483  */
dev_page_is_reusable(const struct page * page)3484 static inline bool dev_page_is_reusable(const struct page *page)
3485 {
3486 	return likely(page_to_nid(page) == numa_mem_id() &&
3487 		      !page_is_pfmemalloc(page));
3488 }
3489 
3490 /**
3491  *	skb_propagate_pfmemalloc - Propagate pfmemalloc if skb is allocated after RX page
3492  *	@page: The page that was allocated from skb_alloc_page
3493  *	@skb: The skb that may need pfmemalloc set
3494  */
skb_propagate_pfmemalloc(const struct page * page,struct sk_buff * skb)3495 static inline void skb_propagate_pfmemalloc(const struct page *page,
3496 					    struct sk_buff *skb)
3497 {
3498 	if (page_is_pfmemalloc(page))
3499 		skb->pfmemalloc = true;
3500 }
3501 
3502 /**
3503  * skb_frag_off() - Returns the offset of a skb fragment
3504  * @frag: the paged fragment
3505  */
skb_frag_off(const skb_frag_t * frag)3506 static inline unsigned int skb_frag_off(const skb_frag_t *frag)
3507 {
3508 	return frag->offset;
3509 }
3510 
3511 /**
3512  * skb_frag_off_add() - Increments the offset of a skb fragment by @delta
3513  * @frag: skb fragment
3514  * @delta: value to add
3515  */
skb_frag_off_add(skb_frag_t * frag,int delta)3516 static inline void skb_frag_off_add(skb_frag_t *frag, int delta)
3517 {
3518 	frag->offset += delta;
3519 }
3520 
3521 /**
3522  * skb_frag_off_set() - Sets the offset of a skb fragment
3523  * @frag: skb fragment
3524  * @offset: offset of fragment
3525  */
skb_frag_off_set(skb_frag_t * frag,unsigned int offset)3526 static inline void skb_frag_off_set(skb_frag_t *frag, unsigned int offset)
3527 {
3528 	frag->offset = offset;
3529 }
3530 
3531 /**
3532  * skb_frag_off_copy() - Sets the offset of a skb fragment from another fragment
3533  * @fragto: skb fragment where offset is set
3534  * @fragfrom: skb fragment offset is copied from
3535  */
skb_frag_off_copy(skb_frag_t * fragto,const skb_frag_t * fragfrom)3536 static inline void skb_frag_off_copy(skb_frag_t *fragto,
3537 				     const skb_frag_t *fragfrom)
3538 {
3539 	fragto->offset = fragfrom->offset;
3540 }
3541 
3542 /* Return: true if the skb_frag contains a net_iov. */
skb_frag_is_net_iov(const skb_frag_t * frag)3543 static inline bool skb_frag_is_net_iov(const skb_frag_t *frag)
3544 {
3545 	return netmem_is_net_iov(frag->netmem);
3546 }
3547 
3548 /**
3549  * skb_frag_net_iov - retrieve the net_iov referred to by fragment
3550  * @frag: the fragment
3551  *
3552  * Return: the &struct net_iov associated with @frag. Returns NULL if this
3553  * frag has no associated net_iov.
3554  */
skb_frag_net_iov(const skb_frag_t * frag)3555 static inline struct net_iov *skb_frag_net_iov(const skb_frag_t *frag)
3556 {
3557 	if (!skb_frag_is_net_iov(frag))
3558 		return NULL;
3559 
3560 	return netmem_to_net_iov(frag->netmem);
3561 }
3562 
3563 /**
3564  * skb_frag_page - retrieve the page referred to by a paged fragment
3565  * @frag: the paged fragment
3566  *
3567  * Return: the &struct page associated with @frag. Returns NULL if this frag
3568  * has no associated page.
3569  */
skb_frag_page(const skb_frag_t * frag)3570 static inline struct page *skb_frag_page(const skb_frag_t *frag)
3571 {
3572 	if (skb_frag_is_net_iov(frag))
3573 		return NULL;
3574 
3575 	return netmem_to_page(frag->netmem);
3576 }
3577 
3578 /**
3579  * skb_frag_netmem - retrieve the netmem referred to by a fragment
3580  * @frag: the fragment
3581  *
3582  * Return: the &netmem_ref associated with @frag.
3583  */
skb_frag_netmem(const skb_frag_t * frag)3584 static inline netmem_ref skb_frag_netmem(const skb_frag_t *frag)
3585 {
3586 	return frag->netmem;
3587 }
3588 
3589 int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb,
3590 		    unsigned int headroom);
3591 int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb,
3592 			 struct bpf_prog *prog);
3593 
3594 /**
3595  * skb_frag_address - gets the address of the data contained in a paged fragment
3596  * @frag: the paged fragment buffer
3597  *
3598  * Returns the address of the data within @frag. The page must already
3599  * be mapped.
3600  */
skb_frag_address(const skb_frag_t * frag)3601 static inline void *skb_frag_address(const skb_frag_t *frag)
3602 {
3603 	if (!skb_frag_page(frag))
3604 		return NULL;
3605 
3606 	return page_address(skb_frag_page(frag)) + skb_frag_off(frag);
3607 }
3608 
3609 /**
3610  * skb_frag_address_safe - gets the address of the data contained in a paged fragment
3611  * @frag: the paged fragment buffer
3612  *
3613  * Returns the address of the data within @frag. Checks that the page
3614  * is mapped and returns %NULL otherwise.
3615  */
skb_frag_address_safe(const skb_frag_t * frag)3616 static inline void *skb_frag_address_safe(const skb_frag_t *frag)
3617 {
3618 	void *ptr = page_address(skb_frag_page(frag));
3619 	if (unlikely(!ptr))
3620 		return NULL;
3621 
3622 	return ptr + skb_frag_off(frag);
3623 }
3624 
3625 /**
3626  * skb_frag_page_copy() - sets the page in a fragment from another fragment
3627  * @fragto: skb fragment where page is set
3628  * @fragfrom: skb fragment page is copied from
3629  */
skb_frag_page_copy(skb_frag_t * fragto,const skb_frag_t * fragfrom)3630 static inline void skb_frag_page_copy(skb_frag_t *fragto,
3631 				      const skb_frag_t *fragfrom)
3632 {
3633 	fragto->netmem = fragfrom->netmem;
3634 }
3635 
3636 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio);
3637 
3638 /**
3639  * skb_frag_dma_map - maps a paged fragment via the DMA API
3640  * @dev: the device to map the fragment to
3641  * @frag: the paged fragment to map
3642  * @offset: the offset within the fragment (starting at the
3643  *          fragment's own offset)
3644  * @size: the number of bytes to map
3645  * @dir: the direction of the mapping (``PCI_DMA_*``)
3646  *
3647  * Maps the page associated with @frag to @device.
3648  */
skb_frag_dma_map(struct device * dev,const skb_frag_t * frag,size_t offset,size_t size,enum dma_data_direction dir)3649 static inline dma_addr_t skb_frag_dma_map(struct device *dev,
3650 					  const skb_frag_t *frag,
3651 					  size_t offset, size_t size,
3652 					  enum dma_data_direction dir)
3653 {
3654 	return dma_map_page(dev, skb_frag_page(frag),
3655 			    skb_frag_off(frag) + offset, size, dir);
3656 }
3657 
pskb_copy(struct sk_buff * skb,gfp_t gfp_mask)3658 static inline struct sk_buff *pskb_copy(struct sk_buff *skb,
3659 					gfp_t gfp_mask)
3660 {
3661 	return __pskb_copy(skb, skb_headroom(skb), gfp_mask);
3662 }
3663 
3664 
pskb_copy_for_clone(struct sk_buff * skb,gfp_t gfp_mask)3665 static inline struct sk_buff *pskb_copy_for_clone(struct sk_buff *skb,
3666 						  gfp_t gfp_mask)
3667 {
3668 	return __pskb_copy_fclone(skb, skb_headroom(skb), gfp_mask, true);
3669 }
3670 
3671 
3672 /**
3673  *	skb_clone_writable - is the header of a clone writable
3674  *	@skb: buffer to check
3675  *	@len: length up to which to write
3676  *
3677  *	Returns true if modifying the header part of the cloned buffer
3678  *	does not requires the data to be copied.
3679  */
skb_clone_writable(const struct sk_buff * skb,unsigned int len)3680 static inline int skb_clone_writable(const struct sk_buff *skb, unsigned int len)
3681 {
3682 	return !skb_header_cloned(skb) &&
3683 	       skb_headroom(skb) + len <= skb->hdr_len;
3684 }
3685 
skb_try_make_writable(struct sk_buff * skb,unsigned int write_len)3686 static inline int skb_try_make_writable(struct sk_buff *skb,
3687 					unsigned int write_len)
3688 {
3689 	return skb_cloned(skb) && !skb_clone_writable(skb, write_len) &&
3690 	       pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3691 }
3692 
__skb_cow(struct sk_buff * skb,unsigned int headroom,int cloned)3693 static inline int __skb_cow(struct sk_buff *skb, unsigned int headroom,
3694 			    int cloned)
3695 {
3696 	int delta = 0;
3697 
3698 	if (headroom > skb_headroom(skb))
3699 		delta = headroom - skb_headroom(skb);
3700 
3701 	if (delta || cloned)
3702 		return pskb_expand_head(skb, ALIGN(delta, NET_SKB_PAD), 0,
3703 					GFP_ATOMIC);
3704 	return 0;
3705 }
3706 
3707 /**
3708  *	skb_cow - copy header of skb when it is required
3709  *	@skb: buffer to cow
3710  *	@headroom: needed headroom
3711  *
3712  *	If the skb passed lacks sufficient headroom or its data part
3713  *	is shared, data is reallocated. If reallocation fails, an error
3714  *	is returned and original skb is not changed.
3715  *
3716  *	The result is skb with writable area skb->head...skb->tail
3717  *	and at least @headroom of space at head.
3718  */
skb_cow(struct sk_buff * skb,unsigned int headroom)3719 static inline int skb_cow(struct sk_buff *skb, unsigned int headroom)
3720 {
3721 	return __skb_cow(skb, headroom, skb_cloned(skb));
3722 }
3723 
3724 /**
3725  *	skb_cow_head - skb_cow but only making the head writable
3726  *	@skb: buffer to cow
3727  *	@headroom: needed headroom
3728  *
3729  *	This function is identical to skb_cow except that we replace the
3730  *	skb_cloned check by skb_header_cloned.  It should be used when
3731  *	you only need to push on some header and do not need to modify
3732  *	the data.
3733  */
skb_cow_head(struct sk_buff * skb,unsigned int headroom)3734 static inline int skb_cow_head(struct sk_buff *skb, unsigned int headroom)
3735 {
3736 	return __skb_cow(skb, headroom, skb_header_cloned(skb));
3737 }
3738 
3739 /**
3740  *	skb_padto	- pad an skbuff up to a minimal size
3741  *	@skb: buffer to pad
3742  *	@len: minimal length
3743  *
3744  *	Pads up a buffer to ensure the trailing bytes exist and are
3745  *	blanked. If the buffer already contains sufficient data it
3746  *	is untouched. Otherwise it is extended. Returns zero on
3747  *	success. The skb is freed on error.
3748  */
skb_padto(struct sk_buff * skb,unsigned int len)3749 static inline int skb_padto(struct sk_buff *skb, unsigned int len)
3750 {
3751 	unsigned int size = skb->len;
3752 	if (likely(size >= len))
3753 		return 0;
3754 	return skb_pad(skb, len - size);
3755 }
3756 
3757 /**
3758  *	__skb_put_padto - increase size and pad an skbuff up to a minimal size
3759  *	@skb: buffer to pad
3760  *	@len: minimal length
3761  *	@free_on_error: free buffer on error
3762  *
3763  *	Pads up a buffer to ensure the trailing bytes exist and are
3764  *	blanked. If the buffer already contains sufficient data it
3765  *	is untouched. Otherwise it is extended. Returns zero on
3766  *	success. The skb is freed on error if @free_on_error is true.
3767  */
__skb_put_padto(struct sk_buff * skb,unsigned int len,bool free_on_error)3768 static inline int __must_check __skb_put_padto(struct sk_buff *skb,
3769 					       unsigned int len,
3770 					       bool free_on_error)
3771 {
3772 	unsigned int size = skb->len;
3773 
3774 	if (unlikely(size < len)) {
3775 		len -= size;
3776 		if (__skb_pad(skb, len, free_on_error))
3777 			return -ENOMEM;
3778 		__skb_put(skb, len);
3779 	}
3780 	return 0;
3781 }
3782 
3783 /**
3784  *	skb_put_padto - increase size and pad an skbuff up to a minimal size
3785  *	@skb: buffer to pad
3786  *	@len: minimal length
3787  *
3788  *	Pads up a buffer to ensure the trailing bytes exist and are
3789  *	blanked. If the buffer already contains sufficient data it
3790  *	is untouched. Otherwise it is extended. Returns zero on
3791  *	success. The skb is freed on error.
3792  */
skb_put_padto(struct sk_buff * skb,unsigned int len)3793 static inline int __must_check skb_put_padto(struct sk_buff *skb, unsigned int len)
3794 {
3795 	return __skb_put_padto(skb, len, true);
3796 }
3797 
3798 bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i)
3799 	__must_check;
3800 
skb_add_data(struct sk_buff * skb,struct iov_iter * from,int copy)3801 static inline int skb_add_data(struct sk_buff *skb,
3802 			       struct iov_iter *from, int copy)
3803 {
3804 	const int off = skb->len;
3805 
3806 	if (skb->ip_summed == CHECKSUM_NONE) {
3807 		__wsum csum = 0;
3808 		if (csum_and_copy_from_iter_full(skb_put(skb, copy), copy,
3809 					         &csum, from)) {
3810 			skb->csum = csum_block_add(skb->csum, csum, off);
3811 			return 0;
3812 		}
3813 	} else if (copy_from_iter_full(skb_put(skb, copy), copy, from))
3814 		return 0;
3815 
3816 	__skb_trim(skb, off);
3817 	return -EFAULT;
3818 }
3819 
skb_can_coalesce(struct sk_buff * skb,int i,const struct page * page,int off)3820 static inline bool skb_can_coalesce(struct sk_buff *skb, int i,
3821 				    const struct page *page, int off)
3822 {
3823 	if (skb_zcopy(skb))
3824 		return false;
3825 	if (i) {
3826 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
3827 
3828 		return page == skb_frag_page(frag) &&
3829 		       off == skb_frag_off(frag) + skb_frag_size(frag);
3830 	}
3831 	return false;
3832 }
3833 
__skb_linearize(struct sk_buff * skb)3834 static inline int __skb_linearize(struct sk_buff *skb)
3835 {
3836 	return __pskb_pull_tail(skb, skb->data_len) ? 0 : -ENOMEM;
3837 }
3838 
3839 /**
3840  *	skb_linearize - convert paged skb to linear one
3841  *	@skb: buffer to linarize
3842  *
3843  *	If there is no free memory -ENOMEM is returned, otherwise zero
3844  *	is returned and the old skb data released.
3845  */
skb_linearize(struct sk_buff * skb)3846 static inline int skb_linearize(struct sk_buff *skb)
3847 {
3848 	return skb_is_nonlinear(skb) ? __skb_linearize(skb) : 0;
3849 }
3850 
3851 /**
3852  * skb_has_shared_frag - can any frag be overwritten
3853  * @skb: buffer to test
3854  *
3855  * Return true if the skb has at least one frag that might be modified
3856  * by an external entity (as in vmsplice()/sendfile())
3857  */
skb_has_shared_frag(const struct sk_buff * skb)3858 static inline bool skb_has_shared_frag(const struct sk_buff *skb)
3859 {
3860 	return skb_is_nonlinear(skb) &&
3861 	       skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG;
3862 }
3863 
3864 /**
3865  *	skb_linearize_cow - make sure skb is linear and writable
3866  *	@skb: buffer to process
3867  *
3868  *	If there is no free memory -ENOMEM is returned, otherwise zero
3869  *	is returned and the old skb data released.
3870  */
skb_linearize_cow(struct sk_buff * skb)3871 static inline int skb_linearize_cow(struct sk_buff *skb)
3872 {
3873 	return skb_is_nonlinear(skb) || skb_cloned(skb) ?
3874 	       __skb_linearize(skb) : 0;
3875 }
3876 
3877 static __always_inline void
__skb_postpull_rcsum(struct sk_buff * skb,const void * start,unsigned int len,unsigned int off)3878 __skb_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len,
3879 		     unsigned int off)
3880 {
3881 	if (skb->ip_summed == CHECKSUM_COMPLETE)
3882 		skb->csum = csum_block_sub(skb->csum,
3883 					   csum_partial(start, len, 0), off);
3884 	else if (skb->ip_summed == CHECKSUM_PARTIAL &&
3885 		 skb_checksum_start_offset(skb) < 0)
3886 		skb->ip_summed = CHECKSUM_NONE;
3887 }
3888 
3889 /**
3890  *	skb_postpull_rcsum - update checksum for received skb after pull
3891  *	@skb: buffer to update
3892  *	@start: start of data before pull
3893  *	@len: length of data pulled
3894  *
3895  *	After doing a pull on a received packet, you need to call this to
3896  *	update the CHECKSUM_COMPLETE checksum, or set ip_summed to
3897  *	CHECKSUM_NONE so that it can be recomputed from scratch.
3898  */
skb_postpull_rcsum(struct sk_buff * skb,const void * start,unsigned int len)3899 static inline void skb_postpull_rcsum(struct sk_buff *skb,
3900 				      const void *start, unsigned int len)
3901 {
3902 	if (skb->ip_summed == CHECKSUM_COMPLETE)
3903 		skb->csum = wsum_negate(csum_partial(start, len,
3904 						     wsum_negate(skb->csum)));
3905 	else if (skb->ip_summed == CHECKSUM_PARTIAL &&
3906 		 skb_checksum_start_offset(skb) < 0)
3907 		skb->ip_summed = CHECKSUM_NONE;
3908 }
3909 
3910 static __always_inline void
__skb_postpush_rcsum(struct sk_buff * skb,const void * start,unsigned int len,unsigned int off)3911 __skb_postpush_rcsum(struct sk_buff *skb, const void *start, unsigned int len,
3912 		     unsigned int off)
3913 {
3914 	if (skb->ip_summed == CHECKSUM_COMPLETE)
3915 		skb->csum = csum_block_add(skb->csum,
3916 					   csum_partial(start, len, 0), off);
3917 }
3918 
3919 /**
3920  *	skb_postpush_rcsum - update checksum for received skb after push
3921  *	@skb: buffer to update
3922  *	@start: start of data after push
3923  *	@len: length of data pushed
3924  *
3925  *	After doing a push on a received packet, you need to call this to
3926  *	update the CHECKSUM_COMPLETE checksum.
3927  */
skb_postpush_rcsum(struct sk_buff * skb,const void * start,unsigned int len)3928 static inline void skb_postpush_rcsum(struct sk_buff *skb,
3929 				      const void *start, unsigned int len)
3930 {
3931 	__skb_postpush_rcsum(skb, start, len, 0);
3932 }
3933 
3934 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len);
3935 
3936 /**
3937  *	skb_push_rcsum - push skb and update receive checksum
3938  *	@skb: buffer to update
3939  *	@len: length of data pulled
3940  *
3941  *	This function performs an skb_push on the packet and updates
3942  *	the CHECKSUM_COMPLETE checksum.  It should be used on
3943  *	receive path processing instead of skb_push unless you know
3944  *	that the checksum difference is zero (e.g., a valid IP header)
3945  *	or you are setting ip_summed to CHECKSUM_NONE.
3946  */
skb_push_rcsum(struct sk_buff * skb,unsigned int len)3947 static inline void *skb_push_rcsum(struct sk_buff *skb, unsigned int len)
3948 {
3949 	skb_push(skb, len);
3950 	skb_postpush_rcsum(skb, skb->data, len);
3951 	return skb->data;
3952 }
3953 
3954 int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len);
3955 /**
3956  *	pskb_trim_rcsum - trim received skb and update checksum
3957  *	@skb: buffer to trim
3958  *	@len: new length
3959  *
3960  *	This is exactly the same as pskb_trim except that it ensures the
3961  *	checksum of received packets are still valid after the operation.
3962  *	It can change skb pointers.
3963  */
3964 
pskb_trim_rcsum(struct sk_buff * skb,unsigned int len)3965 static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len)
3966 {
3967 	if (likely(len >= skb->len))
3968 		return 0;
3969 	return pskb_trim_rcsum_slow(skb, len);
3970 }
3971 
__skb_trim_rcsum(struct sk_buff * skb,unsigned int len)3972 static inline int __skb_trim_rcsum(struct sk_buff *skb, unsigned int len)
3973 {
3974 	if (skb->ip_summed == CHECKSUM_COMPLETE)
3975 		skb->ip_summed = CHECKSUM_NONE;
3976 	__skb_trim(skb, len);
3977 	return 0;
3978 }
3979 
__skb_grow_rcsum(struct sk_buff * skb,unsigned int len)3980 static inline int __skb_grow_rcsum(struct sk_buff *skb, unsigned int len)
3981 {
3982 	if (skb->ip_summed == CHECKSUM_COMPLETE)
3983 		skb->ip_summed = CHECKSUM_NONE;
3984 	return __skb_grow(skb, len);
3985 }
3986 
3987 #define rb_to_skb(rb) rb_entry_safe(rb, struct sk_buff, rbnode)
3988 #define skb_rb_first(root) rb_to_skb(rb_first(root))
3989 #define skb_rb_last(root)  rb_to_skb(rb_last(root))
3990 #define skb_rb_next(skb)   rb_to_skb(rb_next(&(skb)->rbnode))
3991 #define skb_rb_prev(skb)   rb_to_skb(rb_prev(&(skb)->rbnode))
3992 
3993 #define skb_queue_walk(queue, skb) \
3994 		for (skb = (queue)->next;					\
3995 		     skb != (struct sk_buff *)(queue);				\
3996 		     skb = skb->next)
3997 
3998 #define skb_queue_walk_safe(queue, skb, tmp)					\
3999 		for (skb = (queue)->next, tmp = skb->next;			\
4000 		     skb != (struct sk_buff *)(queue);				\
4001 		     skb = tmp, tmp = skb->next)
4002 
4003 #define skb_queue_walk_from(queue, skb)						\
4004 		for (; skb != (struct sk_buff *)(queue);			\
4005 		     skb = skb->next)
4006 
4007 #define skb_rbtree_walk(skb, root)						\
4008 		for (skb = skb_rb_first(root); skb != NULL;			\
4009 		     skb = skb_rb_next(skb))
4010 
4011 #define skb_rbtree_walk_from(skb)						\
4012 		for (; skb != NULL;						\
4013 		     skb = skb_rb_next(skb))
4014 
4015 #define skb_rbtree_walk_from_safe(skb, tmp)					\
4016 		for (; tmp = skb ? skb_rb_next(skb) : NULL, (skb != NULL);	\
4017 		     skb = tmp)
4018 
4019 #define skb_queue_walk_from_safe(queue, skb, tmp)				\
4020 		for (tmp = skb->next;						\
4021 		     skb != (struct sk_buff *)(queue);				\
4022 		     skb = tmp, tmp = skb->next)
4023 
4024 #define skb_queue_reverse_walk(queue, skb) \
4025 		for (skb = (queue)->prev;					\
4026 		     skb != (struct sk_buff *)(queue);				\
4027 		     skb = skb->prev)
4028 
4029 #define skb_queue_reverse_walk_safe(queue, skb, tmp)				\
4030 		for (skb = (queue)->prev, tmp = skb->prev;			\
4031 		     skb != (struct sk_buff *)(queue);				\
4032 		     skb = tmp, tmp = skb->prev)
4033 
4034 #define skb_queue_reverse_walk_from_safe(queue, skb, tmp)			\
4035 		for (tmp = skb->prev;						\
4036 		     skb != (struct sk_buff *)(queue);				\
4037 		     skb = tmp, tmp = skb->prev)
4038 
skb_has_frag_list(const struct sk_buff * skb)4039 static inline bool skb_has_frag_list(const struct sk_buff *skb)
4040 {
4041 	return skb_shinfo(skb)->frag_list != NULL;
4042 }
4043 
skb_frag_list_init(struct sk_buff * skb)4044 static inline void skb_frag_list_init(struct sk_buff *skb)
4045 {
4046 	skb_shinfo(skb)->frag_list = NULL;
4047 }
4048 
4049 #define skb_walk_frags(skb, iter)	\
4050 	for (iter = skb_shinfo(skb)->frag_list; iter; iter = iter->next)
4051 
4052 
4053 int __skb_wait_for_more_packets(struct sock *sk, struct sk_buff_head *queue,
4054 				int *err, long *timeo_p,
4055 				const struct sk_buff *skb);
4056 struct sk_buff *__skb_try_recv_from_queue(struct sock *sk,
4057 					  struct sk_buff_head *queue,
4058 					  unsigned int flags,
4059 					  int *off, int *err,
4060 					  struct sk_buff **last);
4061 struct sk_buff *__skb_try_recv_datagram(struct sock *sk,
4062 					struct sk_buff_head *queue,
4063 					unsigned int flags, int *off, int *err,
4064 					struct sk_buff **last);
4065 struct sk_buff *__skb_recv_datagram(struct sock *sk,
4066 				    struct sk_buff_head *sk_queue,
4067 				    unsigned int flags, int *off, int *err);
4068 struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned int flags, int *err);
4069 __poll_t datagram_poll(struct file *file, struct socket *sock,
4070 			   struct poll_table_struct *wait);
4071 int skb_copy_datagram_iter(const struct sk_buff *from, int offset,
4072 			   struct iov_iter *to, int size);
skb_copy_datagram_msg(const struct sk_buff * from,int offset,struct msghdr * msg,int size)4073 static inline int skb_copy_datagram_msg(const struct sk_buff *from, int offset,
4074 					struct msghdr *msg, int size)
4075 {
4076 	return skb_copy_datagram_iter(from, offset, &msg->msg_iter, size);
4077 }
4078 int skb_copy_and_csum_datagram_msg(struct sk_buff *skb, int hlen,
4079 				   struct msghdr *msg);
4080 int skb_copy_and_hash_datagram_iter(const struct sk_buff *skb, int offset,
4081 			   struct iov_iter *to, int len,
4082 			   struct ahash_request *hash);
4083 int skb_copy_datagram_from_iter(struct sk_buff *skb, int offset,
4084 				 struct iov_iter *from, int len);
4085 int zerocopy_sg_from_iter(struct sk_buff *skb, struct iov_iter *frm);
4086 void skb_free_datagram(struct sock *sk, struct sk_buff *skb);
4087 int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags);
4088 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len);
4089 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len);
4090 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, u8 *to,
4091 			      int len);
4092 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
4093 		    struct pipe_inode_info *pipe, unsigned int len,
4094 		    unsigned int flags);
4095 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
4096 			 int len);
4097 int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len);
4098 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to);
4099 unsigned int skb_zerocopy_headlen(const struct sk_buff *from);
4100 int skb_zerocopy(struct sk_buff *to, struct sk_buff *from,
4101 		 int len, int hlen);
4102 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len);
4103 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen);
4104 void skb_scrub_packet(struct sk_buff *skb, bool xnet);
4105 struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features);
4106 struct sk_buff *skb_segment_list(struct sk_buff *skb, netdev_features_t features,
4107 				 unsigned int offset);
4108 struct sk_buff *skb_vlan_untag(struct sk_buff *skb);
4109 int skb_ensure_writable(struct sk_buff *skb, unsigned int write_len);
4110 int skb_ensure_writable_head_tail(struct sk_buff *skb, struct net_device *dev);
4111 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci);
4112 int skb_vlan_pop(struct sk_buff *skb);
4113 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci);
4114 int skb_eth_pop(struct sk_buff *skb);
4115 int skb_eth_push(struct sk_buff *skb, const unsigned char *dst,
4116 		 const unsigned char *src);
4117 int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto,
4118 		  int mac_len, bool ethernet);
4119 int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len,
4120 		 bool ethernet);
4121 int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse);
4122 int skb_mpls_dec_ttl(struct sk_buff *skb);
4123 struct sk_buff *pskb_extract(struct sk_buff *skb, int off, int to_copy,
4124 			     gfp_t gfp);
4125 
memcpy_from_msg(void * data,struct msghdr * msg,int len)4126 static inline int memcpy_from_msg(void *data, struct msghdr *msg, int len)
4127 {
4128 	return copy_from_iter_full(data, len, &msg->msg_iter) ? 0 : -EFAULT;
4129 }
4130 
memcpy_to_msg(struct msghdr * msg,void * data,int len)4131 static inline int memcpy_to_msg(struct msghdr *msg, void *data, int len)
4132 {
4133 	return copy_to_iter(data, len, &msg->msg_iter) == len ? 0 : -EFAULT;
4134 }
4135 
4136 struct skb_checksum_ops {
4137 	__wsum (*update)(const void *mem, int len, __wsum wsum);
4138 	__wsum (*combine)(__wsum csum, __wsum csum2, int offset, int len);
4139 };
4140 
4141 extern const struct skb_checksum_ops *crc32c_csum_stub __read_mostly;
4142 
4143 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
4144 		      __wsum csum, const struct skb_checksum_ops *ops);
4145 __wsum skb_checksum(const struct sk_buff *skb, int offset, int len,
4146 		    __wsum csum);
4147 
4148 static inline void * __must_check
__skb_header_pointer(const struct sk_buff * skb,int offset,int len,const void * data,int hlen,void * buffer)4149 __skb_header_pointer(const struct sk_buff *skb, int offset, int len,
4150 		     const void *data, int hlen, void *buffer)
4151 {
4152 	if (likely(hlen - offset >= len))
4153 		return (void *)data + offset;
4154 
4155 	if (!skb || unlikely(skb_copy_bits(skb, offset, buffer, len) < 0))
4156 		return NULL;
4157 
4158 	return buffer;
4159 }
4160 
4161 static inline void * __must_check
skb_header_pointer(const struct sk_buff * skb,int offset,int len,void * buffer)4162 skb_header_pointer(const struct sk_buff *skb, int offset, int len, void *buffer)
4163 {
4164 	return __skb_header_pointer(skb, offset, len, skb->data,
4165 				    skb_headlen(skb), buffer);
4166 }
4167 
4168 static inline void * __must_check
skb_pointer_if_linear(const struct sk_buff * skb,int offset,int len)4169 skb_pointer_if_linear(const struct sk_buff *skb, int offset, int len)
4170 {
4171 	if (likely(skb_headlen(skb) - offset >= len))
4172 		return skb->data + offset;
4173 	return NULL;
4174 }
4175 
4176 /**
4177  *	skb_needs_linearize - check if we need to linearize a given skb
4178  *			      depending on the given device features.
4179  *	@skb: socket buffer to check
4180  *	@features: net device features
4181  *
4182  *	Returns true if either:
4183  *	1. skb has frag_list and the device doesn't support FRAGLIST, or
4184  *	2. skb is fragmented and the device does not support SG.
4185  */
skb_needs_linearize(struct sk_buff * skb,netdev_features_t features)4186 static inline bool skb_needs_linearize(struct sk_buff *skb,
4187 				       netdev_features_t features)
4188 {
4189 	return skb_is_nonlinear(skb) &&
4190 	       ((skb_has_frag_list(skb) && !(features & NETIF_F_FRAGLIST)) ||
4191 		(skb_shinfo(skb)->nr_frags && !(features & NETIF_F_SG)));
4192 }
4193 
skb_copy_from_linear_data(const struct sk_buff * skb,void * to,const unsigned int len)4194 static inline void skb_copy_from_linear_data(const struct sk_buff *skb,
4195 					     void *to,
4196 					     const unsigned int len)
4197 {
4198 	memcpy(to, skb->data, len);
4199 }
4200 
skb_copy_from_linear_data_offset(const struct sk_buff * skb,const int offset,void * to,const unsigned int len)4201 static inline void skb_copy_from_linear_data_offset(const struct sk_buff *skb,
4202 						    const int offset, void *to,
4203 						    const unsigned int len)
4204 {
4205 	memcpy(to, skb->data + offset, len);
4206 }
4207 
skb_copy_to_linear_data(struct sk_buff * skb,const void * from,const unsigned int len)4208 static inline void skb_copy_to_linear_data(struct sk_buff *skb,
4209 					   const void *from,
4210 					   const unsigned int len)
4211 {
4212 	memcpy(skb->data, from, len);
4213 }
4214 
skb_copy_to_linear_data_offset(struct sk_buff * skb,const int offset,const void * from,const unsigned int len)4215 static inline void skb_copy_to_linear_data_offset(struct sk_buff *skb,
4216 						  const int offset,
4217 						  const void *from,
4218 						  const unsigned int len)
4219 {
4220 	memcpy(skb->data + offset, from, len);
4221 }
4222 
4223 void skb_init(void);
4224 
skb_get_ktime(const struct sk_buff * skb)4225 static inline ktime_t skb_get_ktime(const struct sk_buff *skb)
4226 {
4227 	return skb->tstamp;
4228 }
4229 
4230 /**
4231  *	skb_get_timestamp - get timestamp from a skb
4232  *	@skb: skb to get stamp from
4233  *	@stamp: pointer to struct __kernel_old_timeval to store stamp in
4234  *
4235  *	Timestamps are stored in the skb as offsets to a base timestamp.
4236  *	This function converts the offset back to a struct timeval and stores
4237  *	it in stamp.
4238  */
skb_get_timestamp(const struct sk_buff * skb,struct __kernel_old_timeval * stamp)4239 static inline void skb_get_timestamp(const struct sk_buff *skb,
4240 				     struct __kernel_old_timeval *stamp)
4241 {
4242 	*stamp = ns_to_kernel_old_timeval(skb->tstamp);
4243 }
4244 
skb_get_new_timestamp(const struct sk_buff * skb,struct __kernel_sock_timeval * stamp)4245 static inline void skb_get_new_timestamp(const struct sk_buff *skb,
4246 					 struct __kernel_sock_timeval *stamp)
4247 {
4248 	struct timespec64 ts = ktime_to_timespec64(skb->tstamp);
4249 
4250 	stamp->tv_sec = ts.tv_sec;
4251 	stamp->tv_usec = ts.tv_nsec / 1000;
4252 }
4253 
skb_get_timestampns(const struct sk_buff * skb,struct __kernel_old_timespec * stamp)4254 static inline void skb_get_timestampns(const struct sk_buff *skb,
4255 				       struct __kernel_old_timespec *stamp)
4256 {
4257 	struct timespec64 ts = ktime_to_timespec64(skb->tstamp);
4258 
4259 	stamp->tv_sec = ts.tv_sec;
4260 	stamp->tv_nsec = ts.tv_nsec;
4261 }
4262 
skb_get_new_timestampns(const struct sk_buff * skb,struct __kernel_timespec * stamp)4263 static inline void skb_get_new_timestampns(const struct sk_buff *skb,
4264 					   struct __kernel_timespec *stamp)
4265 {
4266 	struct timespec64 ts = ktime_to_timespec64(skb->tstamp);
4267 
4268 	stamp->tv_sec = ts.tv_sec;
4269 	stamp->tv_nsec = ts.tv_nsec;
4270 }
4271 
__net_timestamp(struct sk_buff * skb)4272 static inline void __net_timestamp(struct sk_buff *skb)
4273 {
4274 	skb->tstamp = ktime_get_real();
4275 	skb->tstamp_type = SKB_CLOCK_REALTIME;
4276 }
4277 
net_timedelta(ktime_t t)4278 static inline ktime_t net_timedelta(ktime_t t)
4279 {
4280 	return ktime_sub(ktime_get_real(), t);
4281 }
4282 
skb_set_delivery_time(struct sk_buff * skb,ktime_t kt,u8 tstamp_type)4283 static inline void skb_set_delivery_time(struct sk_buff *skb, ktime_t kt,
4284 					 u8 tstamp_type)
4285 {
4286 	skb->tstamp = kt;
4287 
4288 	if (kt)
4289 		skb->tstamp_type = tstamp_type;
4290 	else
4291 		skb->tstamp_type = SKB_CLOCK_REALTIME;
4292 }
4293 
skb_set_delivery_type_by_clockid(struct sk_buff * skb,ktime_t kt,clockid_t clockid)4294 static inline void skb_set_delivery_type_by_clockid(struct sk_buff *skb,
4295 						    ktime_t kt, clockid_t clockid)
4296 {
4297 	u8 tstamp_type = SKB_CLOCK_REALTIME;
4298 
4299 	switch (clockid) {
4300 	case CLOCK_REALTIME:
4301 		break;
4302 	case CLOCK_MONOTONIC:
4303 		tstamp_type = SKB_CLOCK_MONOTONIC;
4304 		break;
4305 	case CLOCK_TAI:
4306 		tstamp_type = SKB_CLOCK_TAI;
4307 		break;
4308 	default:
4309 		WARN_ON_ONCE(1);
4310 		kt = 0;
4311 	}
4312 
4313 	skb_set_delivery_time(skb, kt, tstamp_type);
4314 }
4315 
4316 DECLARE_STATIC_KEY_FALSE(netstamp_needed_key);
4317 
4318 /* It is used in the ingress path to clear the delivery_time.
4319  * If needed, set the skb->tstamp to the (rcv) timestamp.
4320  */
skb_clear_delivery_time(struct sk_buff * skb)4321 static inline void skb_clear_delivery_time(struct sk_buff *skb)
4322 {
4323 	if (skb->tstamp_type) {
4324 		skb->tstamp_type = SKB_CLOCK_REALTIME;
4325 		if (static_branch_unlikely(&netstamp_needed_key))
4326 			skb->tstamp = ktime_get_real();
4327 		else
4328 			skb->tstamp = 0;
4329 	}
4330 }
4331 
skb_clear_tstamp(struct sk_buff * skb)4332 static inline void skb_clear_tstamp(struct sk_buff *skb)
4333 {
4334 	if (skb->tstamp_type)
4335 		return;
4336 
4337 	skb->tstamp = 0;
4338 }
4339 
skb_tstamp(const struct sk_buff * skb)4340 static inline ktime_t skb_tstamp(const struct sk_buff *skb)
4341 {
4342 	if (skb->tstamp_type)
4343 		return 0;
4344 
4345 	return skb->tstamp;
4346 }
4347 
skb_tstamp_cond(const struct sk_buff * skb,bool cond)4348 static inline ktime_t skb_tstamp_cond(const struct sk_buff *skb, bool cond)
4349 {
4350 	if (skb->tstamp_type != SKB_CLOCK_MONOTONIC && skb->tstamp)
4351 		return skb->tstamp;
4352 
4353 	if (static_branch_unlikely(&netstamp_needed_key) || cond)
4354 		return ktime_get_real();
4355 
4356 	return 0;
4357 }
4358 
skb_metadata_len(const struct sk_buff * skb)4359 static inline u8 skb_metadata_len(const struct sk_buff *skb)
4360 {
4361 	return skb_shinfo(skb)->meta_len;
4362 }
4363 
skb_metadata_end(const struct sk_buff * skb)4364 static inline void *skb_metadata_end(const struct sk_buff *skb)
4365 {
4366 	return skb_mac_header(skb);
4367 }
4368 
__skb_metadata_differs(const struct sk_buff * skb_a,const struct sk_buff * skb_b,u8 meta_len)4369 static inline bool __skb_metadata_differs(const struct sk_buff *skb_a,
4370 					  const struct sk_buff *skb_b,
4371 					  u8 meta_len)
4372 {
4373 	const void *a = skb_metadata_end(skb_a);
4374 	const void *b = skb_metadata_end(skb_b);
4375 	u64 diffs = 0;
4376 
4377 	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ||
4378 	    BITS_PER_LONG != 64)
4379 		goto slow;
4380 
4381 	/* Using more efficient variant than plain call to memcmp(). */
4382 	switch (meta_len) {
4383 #define __it(x, op) (x -= sizeof(u##op))
4384 #define __it_diff(a, b, op) (*(u##op *)__it(a, op)) ^ (*(u##op *)__it(b, op))
4385 	case 32: diffs |= __it_diff(a, b, 64);
4386 		fallthrough;
4387 	case 24: diffs |= __it_diff(a, b, 64);
4388 		fallthrough;
4389 	case 16: diffs |= __it_diff(a, b, 64);
4390 		fallthrough;
4391 	case  8: diffs |= __it_diff(a, b, 64);
4392 		break;
4393 	case 28: diffs |= __it_diff(a, b, 64);
4394 		fallthrough;
4395 	case 20: diffs |= __it_diff(a, b, 64);
4396 		fallthrough;
4397 	case 12: diffs |= __it_diff(a, b, 64);
4398 		fallthrough;
4399 	case  4: diffs |= __it_diff(a, b, 32);
4400 		break;
4401 	default:
4402 slow:
4403 		return memcmp(a - meta_len, b - meta_len, meta_len);
4404 	}
4405 	return diffs;
4406 }
4407 
skb_metadata_differs(const struct sk_buff * skb_a,const struct sk_buff * skb_b)4408 static inline bool skb_metadata_differs(const struct sk_buff *skb_a,
4409 					const struct sk_buff *skb_b)
4410 {
4411 	u8 len_a = skb_metadata_len(skb_a);
4412 	u8 len_b = skb_metadata_len(skb_b);
4413 
4414 	if (!(len_a | len_b))
4415 		return false;
4416 
4417 	return len_a != len_b ?
4418 	       true : __skb_metadata_differs(skb_a, skb_b, len_a);
4419 }
4420 
skb_metadata_set(struct sk_buff * skb,u8 meta_len)4421 static inline void skb_metadata_set(struct sk_buff *skb, u8 meta_len)
4422 {
4423 	skb_shinfo(skb)->meta_len = meta_len;
4424 }
4425 
skb_metadata_clear(struct sk_buff * skb)4426 static inline void skb_metadata_clear(struct sk_buff *skb)
4427 {
4428 	skb_metadata_set(skb, 0);
4429 }
4430 
4431 struct sk_buff *skb_clone_sk(struct sk_buff *skb);
4432 
4433 #ifdef CONFIG_NETWORK_PHY_TIMESTAMPING
4434 
4435 void skb_clone_tx_timestamp(struct sk_buff *skb);
4436 bool skb_defer_rx_timestamp(struct sk_buff *skb);
4437 
4438 #else /* CONFIG_NETWORK_PHY_TIMESTAMPING */
4439 
skb_clone_tx_timestamp(struct sk_buff * skb)4440 static inline void skb_clone_tx_timestamp(struct sk_buff *skb)
4441 {
4442 }
4443 
skb_defer_rx_timestamp(struct sk_buff * skb)4444 static inline bool skb_defer_rx_timestamp(struct sk_buff *skb)
4445 {
4446 	return false;
4447 }
4448 
4449 #endif /* !CONFIG_NETWORK_PHY_TIMESTAMPING */
4450 
4451 /**
4452  * skb_complete_tx_timestamp() - deliver cloned skb with tx timestamps
4453  *
4454  * PHY drivers may accept clones of transmitted packets for
4455  * timestamping via their phy_driver.txtstamp method. These drivers
4456  * must call this function to return the skb back to the stack with a
4457  * timestamp.
4458  *
4459  * @skb: clone of the original outgoing packet
4460  * @hwtstamps: hardware time stamps
4461  *
4462  */
4463 void skb_complete_tx_timestamp(struct sk_buff *skb,
4464 			       struct skb_shared_hwtstamps *hwtstamps);
4465 
4466 void __skb_tstamp_tx(struct sk_buff *orig_skb, const struct sk_buff *ack_skb,
4467 		     struct skb_shared_hwtstamps *hwtstamps,
4468 		     struct sock *sk, int tstype);
4469 
4470 /**
4471  * skb_tstamp_tx - queue clone of skb with send time stamps
4472  * @orig_skb:	the original outgoing packet
4473  * @hwtstamps:	hardware time stamps, may be NULL if not available
4474  *
4475  * If the skb has a socket associated, then this function clones the
4476  * skb (thus sharing the actual data and optional structures), stores
4477  * the optional hardware time stamping information (if non NULL) or
4478  * generates a software time stamp (otherwise), then queues the clone
4479  * to the error queue of the socket.  Errors are silently ignored.
4480  */
4481 void skb_tstamp_tx(struct sk_buff *orig_skb,
4482 		   struct skb_shared_hwtstamps *hwtstamps);
4483 
4484 /**
4485  * skb_tx_timestamp() - Driver hook for transmit timestamping
4486  *
4487  * Ethernet MAC Drivers should call this function in their hard_xmit()
4488  * function immediately before giving the sk_buff to the MAC hardware.
4489  *
4490  * Specifically, one should make absolutely sure that this function is
4491  * called before TX completion of this packet can trigger.  Otherwise
4492  * the packet could potentially already be freed.
4493  *
4494  * @skb: A socket buffer.
4495  */
skb_tx_timestamp(struct sk_buff * skb)4496 static inline void skb_tx_timestamp(struct sk_buff *skb)
4497 {
4498 	skb_clone_tx_timestamp(skb);
4499 	if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP)
4500 		skb_tstamp_tx(skb, NULL);
4501 }
4502 
4503 /**
4504  * skb_complete_wifi_ack - deliver skb with wifi status
4505  *
4506  * @skb: the original outgoing packet
4507  * @acked: ack status
4508  *
4509  */
4510 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked);
4511 
4512 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len);
4513 __sum16 __skb_checksum_complete(struct sk_buff *skb);
4514 
skb_csum_unnecessary(const struct sk_buff * skb)4515 static inline int skb_csum_unnecessary(const struct sk_buff *skb)
4516 {
4517 	return ((skb->ip_summed == CHECKSUM_UNNECESSARY) ||
4518 		skb->csum_valid ||
4519 		(skb->ip_summed == CHECKSUM_PARTIAL &&
4520 		 skb_checksum_start_offset(skb) >= 0));
4521 }
4522 
4523 /**
4524  *	skb_checksum_complete - Calculate checksum of an entire packet
4525  *	@skb: packet to process
4526  *
4527  *	This function calculates the checksum over the entire packet plus
4528  *	the value of skb->csum.  The latter can be used to supply the
4529  *	checksum of a pseudo header as used by TCP/UDP.  It returns the
4530  *	checksum.
4531  *
4532  *	For protocols that contain complete checksums such as ICMP/TCP/UDP,
4533  *	this function can be used to verify that checksum on received
4534  *	packets.  In that case the function should return zero if the
4535  *	checksum is correct.  In particular, this function will return zero
4536  *	if skb->ip_summed is CHECKSUM_UNNECESSARY which indicates that the
4537  *	hardware has already verified the correctness of the checksum.
4538  */
skb_checksum_complete(struct sk_buff * skb)4539 static inline __sum16 skb_checksum_complete(struct sk_buff *skb)
4540 {
4541 	return skb_csum_unnecessary(skb) ?
4542 	       0 : __skb_checksum_complete(skb);
4543 }
4544 
__skb_decr_checksum_unnecessary(struct sk_buff * skb)4545 static inline void __skb_decr_checksum_unnecessary(struct sk_buff *skb)
4546 {
4547 	if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
4548 		if (skb->csum_level == 0)
4549 			skb->ip_summed = CHECKSUM_NONE;
4550 		else
4551 			skb->csum_level--;
4552 	}
4553 }
4554 
__skb_incr_checksum_unnecessary(struct sk_buff * skb)4555 static inline void __skb_incr_checksum_unnecessary(struct sk_buff *skb)
4556 {
4557 	if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
4558 		if (skb->csum_level < SKB_MAX_CSUM_LEVEL)
4559 			skb->csum_level++;
4560 	} else if (skb->ip_summed == CHECKSUM_NONE) {
4561 		skb->ip_summed = CHECKSUM_UNNECESSARY;
4562 		skb->csum_level = 0;
4563 	}
4564 }
4565 
__skb_reset_checksum_unnecessary(struct sk_buff * skb)4566 static inline void __skb_reset_checksum_unnecessary(struct sk_buff *skb)
4567 {
4568 	if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
4569 		skb->ip_summed = CHECKSUM_NONE;
4570 		skb->csum_level = 0;
4571 	}
4572 }
4573 
4574 /* Check if we need to perform checksum complete validation.
4575  *
4576  * Returns true if checksum complete is needed, false otherwise
4577  * (either checksum is unnecessary or zero checksum is allowed).
4578  */
__skb_checksum_validate_needed(struct sk_buff * skb,bool zero_okay,__sum16 check)4579 static inline bool __skb_checksum_validate_needed(struct sk_buff *skb,
4580 						  bool zero_okay,
4581 						  __sum16 check)
4582 {
4583 	if (skb_csum_unnecessary(skb) || (zero_okay && !check)) {
4584 		skb->csum_valid = 1;
4585 		__skb_decr_checksum_unnecessary(skb);
4586 		return false;
4587 	}
4588 
4589 	return true;
4590 }
4591 
4592 /* For small packets <= CHECKSUM_BREAK perform checksum complete directly
4593  * in checksum_init.
4594  */
4595 #define CHECKSUM_BREAK 76
4596 
4597 /* Unset checksum-complete
4598  *
4599  * Unset checksum complete can be done when packet is being modified
4600  * (uncompressed for instance) and checksum-complete value is
4601  * invalidated.
4602  */
skb_checksum_complete_unset(struct sk_buff * skb)4603 static inline void skb_checksum_complete_unset(struct sk_buff *skb)
4604 {
4605 	if (skb->ip_summed == CHECKSUM_COMPLETE)
4606 		skb->ip_summed = CHECKSUM_NONE;
4607 }
4608 
4609 /* Validate (init) checksum based on checksum complete.
4610  *
4611  * Return values:
4612  *   0: checksum is validated or try to in skb_checksum_complete. In the latter
4613  *	case the ip_summed will not be CHECKSUM_UNNECESSARY and the pseudo
4614  *	checksum is stored in skb->csum for use in __skb_checksum_complete
4615  *   non-zero: value of invalid checksum
4616  *
4617  */
__skb_checksum_validate_complete(struct sk_buff * skb,bool complete,__wsum psum)4618 static inline __sum16 __skb_checksum_validate_complete(struct sk_buff *skb,
4619 						       bool complete,
4620 						       __wsum psum)
4621 {
4622 	if (skb->ip_summed == CHECKSUM_COMPLETE) {
4623 		if (!csum_fold(csum_add(psum, skb->csum))) {
4624 			skb->csum_valid = 1;
4625 			return 0;
4626 		}
4627 	}
4628 
4629 	skb->csum = psum;
4630 
4631 	if (complete || skb->len <= CHECKSUM_BREAK) {
4632 		__sum16 csum;
4633 
4634 		csum = __skb_checksum_complete(skb);
4635 		skb->csum_valid = !csum;
4636 		return csum;
4637 	}
4638 
4639 	return 0;
4640 }
4641 
null_compute_pseudo(struct sk_buff * skb,int proto)4642 static inline __wsum null_compute_pseudo(struct sk_buff *skb, int proto)
4643 {
4644 	return 0;
4645 }
4646 
4647 /* Perform checksum validate (init). Note that this is a macro since we only
4648  * want to calculate the pseudo header which is an input function if necessary.
4649  * First we try to validate without any computation (checksum unnecessary) and
4650  * then calculate based on checksum complete calling the function to compute
4651  * pseudo header.
4652  *
4653  * Return values:
4654  *   0: checksum is validated or try to in skb_checksum_complete
4655  *   non-zero: value of invalid checksum
4656  */
4657 #define __skb_checksum_validate(skb, proto, complete,			\
4658 				zero_okay, check, compute_pseudo)	\
4659 ({									\
4660 	__sum16 __ret = 0;						\
4661 	skb->csum_valid = 0;						\
4662 	if (__skb_checksum_validate_needed(skb, zero_okay, check))	\
4663 		__ret = __skb_checksum_validate_complete(skb,		\
4664 				complete, compute_pseudo(skb, proto));	\
4665 	__ret;								\
4666 })
4667 
4668 #define skb_checksum_init(skb, proto, compute_pseudo)			\
4669 	__skb_checksum_validate(skb, proto, false, false, 0, compute_pseudo)
4670 
4671 #define skb_checksum_init_zero_check(skb, proto, check, compute_pseudo)	\
4672 	__skb_checksum_validate(skb, proto, false, true, check, compute_pseudo)
4673 
4674 #define skb_checksum_validate(skb, proto, compute_pseudo)		\
4675 	__skb_checksum_validate(skb, proto, true, false, 0, compute_pseudo)
4676 
4677 #define skb_checksum_validate_zero_check(skb, proto, check,		\
4678 					 compute_pseudo)		\
4679 	__skb_checksum_validate(skb, proto, true, true, check, compute_pseudo)
4680 
4681 #define skb_checksum_simple_validate(skb)				\
4682 	__skb_checksum_validate(skb, 0, true, false, 0, null_compute_pseudo)
4683 
__skb_checksum_convert_check(struct sk_buff * skb)4684 static inline bool __skb_checksum_convert_check(struct sk_buff *skb)
4685 {
4686 	return (skb->ip_summed == CHECKSUM_NONE && skb->csum_valid);
4687 }
4688 
__skb_checksum_convert(struct sk_buff * skb,__wsum pseudo)4689 static inline void __skb_checksum_convert(struct sk_buff *skb, __wsum pseudo)
4690 {
4691 	skb->csum = ~pseudo;
4692 	skb->ip_summed = CHECKSUM_COMPLETE;
4693 }
4694 
4695 #define skb_checksum_try_convert(skb, proto, compute_pseudo)	\
4696 do {									\
4697 	if (__skb_checksum_convert_check(skb))				\
4698 		__skb_checksum_convert(skb, compute_pseudo(skb, proto)); \
4699 } while (0)
4700 
skb_remcsum_adjust_partial(struct sk_buff * skb,void * ptr,u16 start,u16 offset)4701 static inline void skb_remcsum_adjust_partial(struct sk_buff *skb, void *ptr,
4702 					      u16 start, u16 offset)
4703 {
4704 	skb->ip_summed = CHECKSUM_PARTIAL;
4705 	skb->csum_start = ((unsigned char *)ptr + start) - skb->head;
4706 	skb->csum_offset = offset - start;
4707 }
4708 
4709 /* Update skbuf and packet to reflect the remote checksum offload operation.
4710  * When called, ptr indicates the starting point for skb->csum when
4711  * ip_summed is CHECKSUM_COMPLETE. If we need create checksum complete
4712  * here, skb_postpull_rcsum is done so skb->csum start is ptr.
4713  */
skb_remcsum_process(struct sk_buff * skb,void * ptr,int start,int offset,bool nopartial)4714 static inline void skb_remcsum_process(struct sk_buff *skb, void *ptr,
4715 				       int start, int offset, bool nopartial)
4716 {
4717 	__wsum delta;
4718 
4719 	if (!nopartial) {
4720 		skb_remcsum_adjust_partial(skb, ptr, start, offset);
4721 		return;
4722 	}
4723 
4724 	if (unlikely(skb->ip_summed != CHECKSUM_COMPLETE)) {
4725 		__skb_checksum_complete(skb);
4726 		skb_postpull_rcsum(skb, skb->data, ptr - (void *)skb->data);
4727 	}
4728 
4729 	delta = remcsum_adjust(ptr, skb->csum, start, offset);
4730 
4731 	/* Adjust skb->csum since we changed the packet */
4732 	skb->csum = csum_add(skb->csum, delta);
4733 }
4734 
skb_nfct(const struct sk_buff * skb)4735 static inline struct nf_conntrack *skb_nfct(const struct sk_buff *skb)
4736 {
4737 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
4738 	return (void *)(skb->_nfct & NFCT_PTRMASK);
4739 #else
4740 	return NULL;
4741 #endif
4742 }
4743 
skb_get_nfct(const struct sk_buff * skb)4744 static inline unsigned long skb_get_nfct(const struct sk_buff *skb)
4745 {
4746 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
4747 	return skb->_nfct;
4748 #else
4749 	return 0UL;
4750 #endif
4751 }
4752 
skb_set_nfct(struct sk_buff * skb,unsigned long nfct)4753 static inline void skb_set_nfct(struct sk_buff *skb, unsigned long nfct)
4754 {
4755 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
4756 	skb->slow_gro |= !!nfct;
4757 	skb->_nfct = nfct;
4758 #endif
4759 }
4760 
4761 #ifdef CONFIG_SKB_EXTENSIONS
4762 enum skb_ext_id {
4763 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
4764 	SKB_EXT_BRIDGE_NF,
4765 #endif
4766 #ifdef CONFIG_XFRM
4767 	SKB_EXT_SEC_PATH,
4768 #endif
4769 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
4770 	TC_SKB_EXT,
4771 #endif
4772 #if IS_ENABLED(CONFIG_MPTCP)
4773 	SKB_EXT_MPTCP,
4774 #endif
4775 #if IS_ENABLED(CONFIG_MCTP_FLOWS)
4776 	SKB_EXT_MCTP,
4777 #endif
4778 	SKB_EXT_NUM, /* must be last */
4779 };
4780 
4781 /**
4782  *	struct skb_ext - sk_buff extensions
4783  *	@refcnt: 1 on allocation, deallocated on 0
4784  *	@offset: offset to add to @data to obtain extension address
4785  *	@chunks: size currently allocated, stored in SKB_EXT_ALIGN_SHIFT units
4786  *	@data: start of extension data, variable sized
4787  *
4788  *	Note: offsets/lengths are stored in chunks of 8 bytes, this allows
4789  *	to use 'u8' types while allowing up to 2kb worth of extension data.
4790  */
4791 struct skb_ext {
4792 	refcount_t refcnt;
4793 	u8 offset[SKB_EXT_NUM]; /* in chunks of 8 bytes */
4794 	u8 chunks;		/* same */
4795 	char data[] __aligned(8);
4796 };
4797 
4798 struct skb_ext *__skb_ext_alloc(gfp_t flags);
4799 void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id,
4800 		    struct skb_ext *ext);
4801 void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id);
4802 void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id);
4803 void __skb_ext_put(struct skb_ext *ext);
4804 
skb_ext_put(struct sk_buff * skb)4805 static inline void skb_ext_put(struct sk_buff *skb)
4806 {
4807 	if (skb->active_extensions)
4808 		__skb_ext_put(skb->extensions);
4809 }
4810 
__skb_ext_copy(struct sk_buff * dst,const struct sk_buff * src)4811 static inline void __skb_ext_copy(struct sk_buff *dst,
4812 				  const struct sk_buff *src)
4813 {
4814 	dst->active_extensions = src->active_extensions;
4815 
4816 	if (src->active_extensions) {
4817 		struct skb_ext *ext = src->extensions;
4818 
4819 		refcount_inc(&ext->refcnt);
4820 		dst->extensions = ext;
4821 	}
4822 }
4823 
skb_ext_copy(struct sk_buff * dst,const struct sk_buff * src)4824 static inline void skb_ext_copy(struct sk_buff *dst, const struct sk_buff *src)
4825 {
4826 	skb_ext_put(dst);
4827 	__skb_ext_copy(dst, src);
4828 }
4829 
__skb_ext_exist(const struct skb_ext * ext,enum skb_ext_id i)4830 static inline bool __skb_ext_exist(const struct skb_ext *ext, enum skb_ext_id i)
4831 {
4832 	return !!ext->offset[i];
4833 }
4834 
skb_ext_exist(const struct sk_buff * skb,enum skb_ext_id id)4835 static inline bool skb_ext_exist(const struct sk_buff *skb, enum skb_ext_id id)
4836 {
4837 	return skb->active_extensions & (1 << id);
4838 }
4839 
skb_ext_del(struct sk_buff * skb,enum skb_ext_id id)4840 static inline void skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
4841 {
4842 	if (skb_ext_exist(skb, id))
4843 		__skb_ext_del(skb, id);
4844 }
4845 
skb_ext_find(const struct sk_buff * skb,enum skb_ext_id id)4846 static inline void *skb_ext_find(const struct sk_buff *skb, enum skb_ext_id id)
4847 {
4848 	if (skb_ext_exist(skb, id)) {
4849 		struct skb_ext *ext = skb->extensions;
4850 
4851 		return (void *)ext + (ext->offset[id] << 3);
4852 	}
4853 
4854 	return NULL;
4855 }
4856 
skb_ext_reset(struct sk_buff * skb)4857 static inline void skb_ext_reset(struct sk_buff *skb)
4858 {
4859 	if (unlikely(skb->active_extensions)) {
4860 		__skb_ext_put(skb->extensions);
4861 		skb->active_extensions = 0;
4862 	}
4863 }
4864 
skb_has_extensions(struct sk_buff * skb)4865 static inline bool skb_has_extensions(struct sk_buff *skb)
4866 {
4867 	return unlikely(skb->active_extensions);
4868 }
4869 #else
skb_ext_put(struct sk_buff * skb)4870 static inline void skb_ext_put(struct sk_buff *skb) {}
skb_ext_reset(struct sk_buff * skb)4871 static inline void skb_ext_reset(struct sk_buff *skb) {}
skb_ext_del(struct sk_buff * skb,int unused)4872 static inline void skb_ext_del(struct sk_buff *skb, int unused) {}
__skb_ext_copy(struct sk_buff * d,const struct sk_buff * s)4873 static inline void __skb_ext_copy(struct sk_buff *d, const struct sk_buff *s) {}
skb_ext_copy(struct sk_buff * dst,const struct sk_buff * s)4874 static inline void skb_ext_copy(struct sk_buff *dst, const struct sk_buff *s) {}
skb_has_extensions(struct sk_buff * skb)4875 static inline bool skb_has_extensions(struct sk_buff *skb) { return false; }
4876 #endif /* CONFIG_SKB_EXTENSIONS */
4877 
nf_reset_ct(struct sk_buff * skb)4878 static inline void nf_reset_ct(struct sk_buff *skb)
4879 {
4880 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
4881 	nf_conntrack_put(skb_nfct(skb));
4882 	skb->_nfct = 0;
4883 #endif
4884 }
4885 
nf_reset_trace(struct sk_buff * skb)4886 static inline void nf_reset_trace(struct sk_buff *skb)
4887 {
4888 #if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || IS_ENABLED(CONFIG_NF_TABLES)
4889 	skb->nf_trace = 0;
4890 #endif
4891 }
4892 
ipvs_reset(struct sk_buff * skb)4893 static inline void ipvs_reset(struct sk_buff *skb)
4894 {
4895 #if IS_ENABLED(CONFIG_IP_VS)
4896 	skb->ipvs_property = 0;
4897 #endif
4898 }
4899 
4900 /* Note: This doesn't put any conntrack info in dst. */
__nf_copy(struct sk_buff * dst,const struct sk_buff * src,bool copy)4901 static inline void __nf_copy(struct sk_buff *dst, const struct sk_buff *src,
4902 			     bool copy)
4903 {
4904 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
4905 	dst->_nfct = src->_nfct;
4906 	nf_conntrack_get(skb_nfct(src));
4907 #endif
4908 #if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || IS_ENABLED(CONFIG_NF_TABLES)
4909 	if (copy)
4910 		dst->nf_trace = src->nf_trace;
4911 #endif
4912 }
4913 
nf_copy(struct sk_buff * dst,const struct sk_buff * src)4914 static inline void nf_copy(struct sk_buff *dst, const struct sk_buff *src)
4915 {
4916 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
4917 	nf_conntrack_put(skb_nfct(dst));
4918 #endif
4919 	dst->slow_gro = src->slow_gro;
4920 	__nf_copy(dst, src, true);
4921 }
4922 
4923 #ifdef CONFIG_NETWORK_SECMARK
skb_copy_secmark(struct sk_buff * to,const struct sk_buff * from)4924 static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
4925 {
4926 	to->secmark = from->secmark;
4927 }
4928 
skb_init_secmark(struct sk_buff * skb)4929 static inline void skb_init_secmark(struct sk_buff *skb)
4930 {
4931 	skb->secmark = 0;
4932 }
4933 #else
skb_copy_secmark(struct sk_buff * to,const struct sk_buff * from)4934 static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
4935 { }
4936 
skb_init_secmark(struct sk_buff * skb)4937 static inline void skb_init_secmark(struct sk_buff *skb)
4938 { }
4939 #endif
4940 
secpath_exists(const struct sk_buff * skb)4941 static inline int secpath_exists(const struct sk_buff *skb)
4942 {
4943 #ifdef CONFIG_XFRM
4944 	return skb_ext_exist(skb, SKB_EXT_SEC_PATH);
4945 #else
4946 	return 0;
4947 #endif
4948 }
4949 
skb_irq_freeable(const struct sk_buff * skb)4950 static inline bool skb_irq_freeable(const struct sk_buff *skb)
4951 {
4952 	return !skb->destructor &&
4953 		!secpath_exists(skb) &&
4954 		!skb_nfct(skb) &&
4955 		!skb->_skb_refdst &&
4956 		!skb_has_frag_list(skb);
4957 }
4958 
skb_set_queue_mapping(struct sk_buff * skb,u16 queue_mapping)4959 static inline void skb_set_queue_mapping(struct sk_buff *skb, u16 queue_mapping)
4960 {
4961 	skb->queue_mapping = queue_mapping;
4962 }
4963 
skb_get_queue_mapping(const struct sk_buff * skb)4964 static inline u16 skb_get_queue_mapping(const struct sk_buff *skb)
4965 {
4966 	return skb->queue_mapping;
4967 }
4968 
skb_copy_queue_mapping(struct sk_buff * to,const struct sk_buff * from)4969 static inline void skb_copy_queue_mapping(struct sk_buff *to, const struct sk_buff *from)
4970 {
4971 	to->queue_mapping = from->queue_mapping;
4972 }
4973 
skb_record_rx_queue(struct sk_buff * skb,u16 rx_queue)4974 static inline void skb_record_rx_queue(struct sk_buff *skb, u16 rx_queue)
4975 {
4976 	skb->queue_mapping = rx_queue + 1;
4977 }
4978 
skb_get_rx_queue(const struct sk_buff * skb)4979 static inline u16 skb_get_rx_queue(const struct sk_buff *skb)
4980 {
4981 	return skb->queue_mapping - 1;
4982 }
4983 
skb_rx_queue_recorded(const struct sk_buff * skb)4984 static inline bool skb_rx_queue_recorded(const struct sk_buff *skb)
4985 {
4986 	return skb->queue_mapping != 0;
4987 }
4988 
skb_set_dst_pending_confirm(struct sk_buff * skb,u32 val)4989 static inline void skb_set_dst_pending_confirm(struct sk_buff *skb, u32 val)
4990 {
4991 	skb->dst_pending_confirm = val;
4992 }
4993 
skb_get_dst_pending_confirm(const struct sk_buff * skb)4994 static inline bool skb_get_dst_pending_confirm(const struct sk_buff *skb)
4995 {
4996 	return skb->dst_pending_confirm != 0;
4997 }
4998 
skb_sec_path(const struct sk_buff * skb)4999 static inline struct sec_path *skb_sec_path(const struct sk_buff *skb)
5000 {
5001 #ifdef CONFIG_XFRM
5002 	return skb_ext_find(skb, SKB_EXT_SEC_PATH);
5003 #else
5004 	return NULL;
5005 #endif
5006 }
5007 
skb_is_gso(const struct sk_buff * skb)5008 static inline bool skb_is_gso(const struct sk_buff *skb)
5009 {
5010 	return skb_shinfo(skb)->gso_size;
5011 }
5012 
5013 /* Note: Should be called only if skb_is_gso(skb) is true */
skb_is_gso_v6(const struct sk_buff * skb)5014 static inline bool skb_is_gso_v6(const struct sk_buff *skb)
5015 {
5016 	return skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6;
5017 }
5018 
5019 /* Note: Should be called only if skb_is_gso(skb) is true */
skb_is_gso_sctp(const struct sk_buff * skb)5020 static inline bool skb_is_gso_sctp(const struct sk_buff *skb)
5021 {
5022 	return skb_shinfo(skb)->gso_type & SKB_GSO_SCTP;
5023 }
5024 
5025 /* Note: Should be called only if skb_is_gso(skb) is true */
skb_is_gso_tcp(const struct sk_buff * skb)5026 static inline bool skb_is_gso_tcp(const struct sk_buff *skb)
5027 {
5028 	return skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6);
5029 }
5030 
skb_gso_reset(struct sk_buff * skb)5031 static inline void skb_gso_reset(struct sk_buff *skb)
5032 {
5033 	skb_shinfo(skb)->gso_size = 0;
5034 	skb_shinfo(skb)->gso_segs = 0;
5035 	skb_shinfo(skb)->gso_type = 0;
5036 }
5037 
skb_increase_gso_size(struct skb_shared_info * shinfo,u16 increment)5038 static inline void skb_increase_gso_size(struct skb_shared_info *shinfo,
5039 					 u16 increment)
5040 {
5041 	if (WARN_ON_ONCE(shinfo->gso_size == GSO_BY_FRAGS))
5042 		return;
5043 	shinfo->gso_size += increment;
5044 }
5045 
skb_decrease_gso_size(struct skb_shared_info * shinfo,u16 decrement)5046 static inline void skb_decrease_gso_size(struct skb_shared_info *shinfo,
5047 					 u16 decrement)
5048 {
5049 	if (WARN_ON_ONCE(shinfo->gso_size == GSO_BY_FRAGS))
5050 		return;
5051 	shinfo->gso_size -= decrement;
5052 }
5053 
5054 void __skb_warn_lro_forwarding(const struct sk_buff *skb);
5055 
skb_warn_if_lro(const struct sk_buff * skb)5056 static inline bool skb_warn_if_lro(const struct sk_buff *skb)
5057 {
5058 	/* LRO sets gso_size but not gso_type, whereas if GSO is really
5059 	 * wanted then gso_type will be set. */
5060 	const struct skb_shared_info *shinfo = skb_shinfo(skb);
5061 
5062 	if (skb_is_nonlinear(skb) && shinfo->gso_size != 0 &&
5063 	    unlikely(shinfo->gso_type == 0)) {
5064 		__skb_warn_lro_forwarding(skb);
5065 		return true;
5066 	}
5067 	return false;
5068 }
5069 
skb_forward_csum(struct sk_buff * skb)5070 static inline void skb_forward_csum(struct sk_buff *skb)
5071 {
5072 	/* Unfortunately we don't support this one.  Any brave souls? */
5073 	if (skb->ip_summed == CHECKSUM_COMPLETE)
5074 		skb->ip_summed = CHECKSUM_NONE;
5075 }
5076 
5077 /**
5078  * skb_checksum_none_assert - make sure skb ip_summed is CHECKSUM_NONE
5079  * @skb: skb to check
5080  *
5081  * fresh skbs have their ip_summed set to CHECKSUM_NONE.
5082  * Instead of forcing ip_summed to CHECKSUM_NONE, we can
5083  * use this helper, to document places where we make this assertion.
5084  */
skb_checksum_none_assert(const struct sk_buff * skb)5085 static inline void skb_checksum_none_assert(const struct sk_buff *skb)
5086 {
5087 	DEBUG_NET_WARN_ON_ONCE(skb->ip_summed != CHECKSUM_NONE);
5088 }
5089 
5090 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off);
5091 
5092 int skb_checksum_setup(struct sk_buff *skb, bool recalculate);
5093 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
5094 				     unsigned int transport_len,
5095 				     __sum16(*skb_chkf)(struct sk_buff *skb));
5096 
5097 /**
5098  * skb_head_is_locked - Determine if the skb->head is locked down
5099  * @skb: skb to check
5100  *
5101  * The head on skbs build around a head frag can be removed if they are
5102  * not cloned.  This function returns true if the skb head is locked down
5103  * due to either being allocated via kmalloc, or by being a clone with
5104  * multiple references to the head.
5105  */
skb_head_is_locked(const struct sk_buff * skb)5106 static inline bool skb_head_is_locked(const struct sk_buff *skb)
5107 {
5108 	return !skb->head_frag || skb_cloned(skb);
5109 }
5110 
5111 /* Local Checksum Offload.
5112  * Compute outer checksum based on the assumption that the
5113  * inner checksum will be offloaded later.
5114  * See Documentation/networking/checksum-offloads.rst for
5115  * explanation of how this works.
5116  * Fill in outer checksum adjustment (e.g. with sum of outer
5117  * pseudo-header) before calling.
5118  * Also ensure that inner checksum is in linear data area.
5119  */
lco_csum(struct sk_buff * skb)5120 static inline __wsum lco_csum(struct sk_buff *skb)
5121 {
5122 	unsigned char *csum_start = skb_checksum_start(skb);
5123 	unsigned char *l4_hdr = skb_transport_header(skb);
5124 	__wsum partial;
5125 
5126 	/* Start with complement of inner checksum adjustment */
5127 	partial = ~csum_unfold(*(__force __sum16 *)(csum_start +
5128 						    skb->csum_offset));
5129 
5130 	/* Add in checksum of our headers (incl. outer checksum
5131 	 * adjustment filled in by caller) and return result.
5132 	 */
5133 	return csum_partial(l4_hdr, csum_start - l4_hdr, partial);
5134 }
5135 
skb_is_redirected(const struct sk_buff * skb)5136 static inline bool skb_is_redirected(const struct sk_buff *skb)
5137 {
5138 	return skb->redirected;
5139 }
5140 
skb_set_redirected(struct sk_buff * skb,bool from_ingress)5141 static inline void skb_set_redirected(struct sk_buff *skb, bool from_ingress)
5142 {
5143 	skb->redirected = 1;
5144 #ifdef CONFIG_NET_REDIRECT
5145 	skb->from_ingress = from_ingress;
5146 	if (skb->from_ingress)
5147 		skb_clear_tstamp(skb);
5148 #endif
5149 }
5150 
skb_reset_redirect(struct sk_buff * skb)5151 static inline void skb_reset_redirect(struct sk_buff *skb)
5152 {
5153 	skb->redirected = 0;
5154 }
5155 
skb_set_redirected_noclear(struct sk_buff * skb,bool from_ingress)5156 static inline void skb_set_redirected_noclear(struct sk_buff *skb,
5157 					      bool from_ingress)
5158 {
5159 	skb->redirected = 1;
5160 #ifdef CONFIG_NET_REDIRECT
5161 	skb->from_ingress = from_ingress;
5162 #endif
5163 }
5164 
skb_csum_is_sctp(struct sk_buff * skb)5165 static inline bool skb_csum_is_sctp(struct sk_buff *skb)
5166 {
5167 #if IS_ENABLED(CONFIG_IP_SCTP)
5168 	return skb->csum_not_inet;
5169 #else
5170 	return 0;
5171 #endif
5172 }
5173 
skb_reset_csum_not_inet(struct sk_buff * skb)5174 static inline void skb_reset_csum_not_inet(struct sk_buff *skb)
5175 {
5176 	skb->ip_summed = CHECKSUM_NONE;
5177 #if IS_ENABLED(CONFIG_IP_SCTP)
5178 	skb->csum_not_inet = 0;
5179 #endif
5180 }
5181 
skb_set_kcov_handle(struct sk_buff * skb,const u64 kcov_handle)5182 static inline void skb_set_kcov_handle(struct sk_buff *skb,
5183 				       const u64 kcov_handle)
5184 {
5185 #ifdef CONFIG_KCOV
5186 	skb->kcov_handle = kcov_handle;
5187 #endif
5188 }
5189 
skb_get_kcov_handle(struct sk_buff * skb)5190 static inline u64 skb_get_kcov_handle(struct sk_buff *skb)
5191 {
5192 #ifdef CONFIG_KCOV
5193 	return skb->kcov_handle;
5194 #else
5195 	return 0;
5196 #endif
5197 }
5198 
skb_mark_for_recycle(struct sk_buff * skb)5199 static inline void skb_mark_for_recycle(struct sk_buff *skb)
5200 {
5201 #ifdef CONFIG_PAGE_POOL
5202 	skb->pp_recycle = 1;
5203 #endif
5204 }
5205 
5206 ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter,
5207 			     ssize_t maxsize, gfp_t gfp);
5208 
5209 #endif	/* __KERNEL__ */
5210 #endif	/* _LINUX_SKBUFF_H */
5211