xref: /linux/include/net/sock.h (revision 1f20a5769446a1acae67ac9e63d07a594829a789)
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3  * INET		An implementation of the TCP/IP protocol suite for the LINUX
4  *		operating system.  INET is implemented using the  BSD Socket
5  *		interface as the means of communication with the user level.
6  *
7  *		Definitions for the AF_INET socket handler.
8  *
9  * Version:	@(#)sock.h	1.0.4	05/13/93
10  *
11  * Authors:	Ross Biro
12  *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
13  *		Corey Minyard <wf-rch!minyard@relay.EU.net>
14  *		Florian La Roche <flla@stud.uni-sb.de>
15  *
16  * Fixes:
17  *		Alan Cox	:	Volatiles in skbuff pointers. See
18  *					skbuff comments. May be overdone,
19  *					better to prove they can be removed
20  *					than the reverse.
21  *		Alan Cox	:	Added a zapped field for tcp to note
22  *					a socket is reset and must stay shut up
23  *		Alan Cox	:	New fields for options
24  *	Pauline Middelink	:	identd support
25  *		Alan Cox	:	Eliminate low level recv/recvfrom
26  *		David S. Miller	:	New socket lookup architecture.
27  *              Steve Whitehouse:       Default routines for sock_ops
28  *              Arnaldo C. Melo :	removed net_pinfo, tp_pinfo and made
29  *              			protinfo be just a void pointer, as the
30  *              			protocol specific parts were moved to
31  *              			respective headers and ipv4/v6, etc now
32  *              			use private slabcaches for its socks
33  *              Pedro Hortas	:	New flags field for socket options
34  */
35 #ifndef _SOCK_H
36 #define _SOCK_H
37 
38 #include <linux/hardirq.h>
39 #include <linux/kernel.h>
40 #include <linux/list.h>
41 #include <linux/list_nulls.h>
42 #include <linux/timer.h>
43 #include <linux/cache.h>
44 #include <linux/bitops.h>
45 #include <linux/lockdep.h>
46 #include <linux/netdevice.h>
47 #include <linux/skbuff.h>	/* struct sk_buff */
48 #include <linux/mm.h>
49 #include <linux/security.h>
50 #include <linux/slab.h>
51 #include <linux/uaccess.h>
52 #include <linux/page_counter.h>
53 #include <linux/memcontrol.h>
54 #include <linux/static_key.h>
55 #include <linux/sched.h>
56 #include <linux/wait.h>
57 #include <linux/cgroup-defs.h>
58 #include <linux/rbtree.h>
59 #include <linux/rculist_nulls.h>
60 #include <linux/poll.h>
61 #include <linux/sockptr.h>
62 #include <linux/indirect_call_wrapper.h>
63 #include <linux/atomic.h>
64 #include <linux/refcount.h>
65 #include <linux/llist.h>
66 #include <net/dst.h>
67 #include <net/checksum.h>
68 #include <net/tcp_states.h>
69 #include <linux/net_tstamp.h>
70 #include <net/l3mdev.h>
71 #include <uapi/linux/socket.h>
72 
73 /*
74  * This structure really needs to be cleaned up.
75  * Most of it is for TCP, and not used by any of
76  * the other protocols.
77  */
78 
79 /* This is the per-socket lock.  The spinlock provides a synchronization
80  * between user contexts and software interrupt processing, whereas the
81  * mini-semaphore synchronizes multiple users amongst themselves.
82  */
83 typedef struct {
84 	spinlock_t		slock;
85 	int			owned;
86 	wait_queue_head_t	wq;
87 	/*
88 	 * We express the mutex-alike socket_lock semantics
89 	 * to the lock validator by explicitly managing
90 	 * the slock as a lock variant (in addition to
91 	 * the slock itself):
92 	 */
93 #ifdef CONFIG_DEBUG_LOCK_ALLOC
94 	struct lockdep_map dep_map;
95 #endif
96 } socket_lock_t;
97 
98 struct sock;
99 struct proto;
100 struct net;
101 
102 typedef __u32 __bitwise __portpair;
103 typedef __u64 __bitwise __addrpair;
104 
105 /**
106  *	struct sock_common - minimal network layer representation of sockets
107  *	@skc_daddr: Foreign IPv4 addr
108  *	@skc_rcv_saddr: Bound local IPv4 addr
109  *	@skc_addrpair: 8-byte-aligned __u64 union of @skc_daddr & @skc_rcv_saddr
110  *	@skc_hash: hash value used with various protocol lookup tables
111  *	@skc_u16hashes: two u16 hash values used by UDP lookup tables
112  *	@skc_dport: placeholder for inet_dport/tw_dport
113  *	@skc_num: placeholder for inet_num/tw_num
114  *	@skc_portpair: __u32 union of @skc_dport & @skc_num
115  *	@skc_family: network address family
116  *	@skc_state: Connection state
117  *	@skc_reuse: %SO_REUSEADDR setting
118  *	@skc_reuseport: %SO_REUSEPORT setting
119  *	@skc_ipv6only: socket is IPV6 only
120  *	@skc_net_refcnt: socket is using net ref counting
121  *	@skc_bound_dev_if: bound device index if != 0
122  *	@skc_bind_node: bind hash linkage for various protocol lookup tables
123  *	@skc_portaddr_node: second hash linkage for UDP/UDP-Lite protocol
124  *	@skc_prot: protocol handlers inside a network family
125  *	@skc_net: reference to the network namespace of this socket
126  *	@skc_v6_daddr: IPV6 destination address
127  *	@skc_v6_rcv_saddr: IPV6 source address
128  *	@skc_cookie: socket's cookie value
129  *	@skc_node: main hash linkage for various protocol lookup tables
130  *	@skc_nulls_node: main hash linkage for TCP/UDP/UDP-Lite protocol
131  *	@skc_tx_queue_mapping: tx queue number for this connection
132  *	@skc_rx_queue_mapping: rx queue number for this connection
133  *	@skc_flags: place holder for sk_flags
134  *		%SO_LINGER (l_onoff), %SO_BROADCAST, %SO_KEEPALIVE,
135  *		%SO_OOBINLINE settings, %SO_TIMESTAMPING settings
136  *	@skc_listener: connection request listener socket (aka rsk_listener)
137  *		[union with @skc_flags]
138  *	@skc_tw_dr: (aka tw_dr) ptr to &struct inet_timewait_death_row
139  *		[union with @skc_flags]
140  *	@skc_incoming_cpu: record/match cpu processing incoming packets
141  *	@skc_rcv_wnd: (aka rsk_rcv_wnd) TCP receive window size (possibly scaled)
142  *		[union with @skc_incoming_cpu]
143  *	@skc_tw_rcv_nxt: (aka tw_rcv_nxt) TCP window next expected seq number
144  *		[union with @skc_incoming_cpu]
145  *	@skc_refcnt: reference count
146  *
147  *	This is the minimal network layer representation of sockets, the header
148  *	for struct sock and struct inet_timewait_sock.
149  */
150 struct sock_common {
151 	union {
152 		__addrpair	skc_addrpair;
153 		struct {
154 			__be32	skc_daddr;
155 			__be32	skc_rcv_saddr;
156 		};
157 	};
158 	union  {
159 		unsigned int	skc_hash;
160 		__u16		skc_u16hashes[2];
161 	};
162 	/* skc_dport && skc_num must be grouped as well */
163 	union {
164 		__portpair	skc_portpair;
165 		struct {
166 			__be16	skc_dport;
167 			__u16	skc_num;
168 		};
169 	};
170 
171 	unsigned short		skc_family;
172 	volatile unsigned char	skc_state;
173 	unsigned char		skc_reuse:4;
174 	unsigned char		skc_reuseport:1;
175 	unsigned char		skc_ipv6only:1;
176 	unsigned char		skc_net_refcnt:1;
177 	int			skc_bound_dev_if;
178 	union {
179 		struct hlist_node	skc_bind_node;
180 		struct hlist_node	skc_portaddr_node;
181 	};
182 	struct proto		*skc_prot;
183 	possible_net_t		skc_net;
184 
185 #if IS_ENABLED(CONFIG_IPV6)
186 	struct in6_addr		skc_v6_daddr;
187 	struct in6_addr		skc_v6_rcv_saddr;
188 #endif
189 
190 	atomic64_t		skc_cookie;
191 
192 	/* following fields are padding to force
193 	 * offset(struct sock, sk_refcnt) == 128 on 64bit arches
194 	 * assuming IPV6 is enabled. We use this padding differently
195 	 * for different kind of 'sockets'
196 	 */
197 	union {
198 		unsigned long	skc_flags;
199 		struct sock	*skc_listener; /* request_sock */
200 		struct inet_timewait_death_row *skc_tw_dr; /* inet_timewait_sock */
201 	};
202 	/*
203 	 * fields between dontcopy_begin/dontcopy_end
204 	 * are not copied in sock_copy()
205 	 */
206 	/* private: */
207 	int			skc_dontcopy_begin[0];
208 	/* public: */
209 	union {
210 		struct hlist_node	skc_node;
211 		struct hlist_nulls_node skc_nulls_node;
212 	};
213 	unsigned short		skc_tx_queue_mapping;
214 #ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
215 	unsigned short		skc_rx_queue_mapping;
216 #endif
217 	union {
218 		int		skc_incoming_cpu;
219 		u32		skc_rcv_wnd;
220 		u32		skc_tw_rcv_nxt; /* struct tcp_timewait_sock  */
221 	};
222 
223 	refcount_t		skc_refcnt;
224 	/* private: */
225 	int                     skc_dontcopy_end[0];
226 	union {
227 		u32		skc_rxhash;
228 		u32		skc_window_clamp;
229 		u32		skc_tw_snd_nxt; /* struct tcp_timewait_sock */
230 	};
231 	/* public: */
232 };
233 
234 struct bpf_local_storage;
235 struct sk_filter;
236 
237 /**
238   *	struct sock - network layer representation of sockets
239   *	@__sk_common: shared layout with inet_timewait_sock
240   *	@sk_shutdown: mask of %SEND_SHUTDOWN and/or %RCV_SHUTDOWN
241   *	@sk_userlocks: %SO_SNDBUF and %SO_RCVBUF settings
242   *	@sk_lock:	synchronizer
243   *	@sk_kern_sock: True if sock is using kernel lock classes
244   *	@sk_rcvbuf: size of receive buffer in bytes
245   *	@sk_wq: sock wait queue and async head
246   *	@sk_rx_dst: receive input route used by early demux
247   *	@sk_rx_dst_ifindex: ifindex for @sk_rx_dst
248   *	@sk_rx_dst_cookie: cookie for @sk_rx_dst
249   *	@sk_dst_cache: destination cache
250   *	@sk_dst_pending_confirm: need to confirm neighbour
251   *	@sk_policy: flow policy
252   *	@sk_receive_queue: incoming packets
253   *	@sk_wmem_alloc: transmit queue bytes committed
254   *	@sk_tsq_flags: TCP Small Queues flags
255   *	@sk_write_queue: Packet sending queue
256   *	@sk_omem_alloc: "o" is "option" or "other"
257   *	@sk_wmem_queued: persistent queue size
258   *	@sk_forward_alloc: space allocated forward
259   *	@sk_reserved_mem: space reserved and non-reclaimable for the socket
260   *	@sk_napi_id: id of the last napi context to receive data for sk
261   *	@sk_ll_usec: usecs to busypoll when there is no data
262   *	@sk_allocation: allocation mode
263   *	@sk_pacing_rate: Pacing rate (if supported by transport/packet scheduler)
264   *	@sk_pacing_status: Pacing status (requested, handled by sch_fq)
265   *	@sk_max_pacing_rate: Maximum pacing rate (%SO_MAX_PACING_RATE)
266   *	@sk_sndbuf: size of send buffer in bytes
267   *	@sk_no_check_tx: %SO_NO_CHECK setting, set checksum in TX packets
268   *	@sk_no_check_rx: allow zero checksum in RX packets
269   *	@sk_route_caps: route capabilities (e.g. %NETIF_F_TSO)
270   *	@sk_gso_disabled: if set, NETIF_F_GSO_MASK is forbidden.
271   *	@sk_gso_type: GSO type (e.g. %SKB_GSO_TCPV4)
272   *	@sk_gso_max_size: Maximum GSO segment size to build
273   *	@sk_gso_max_segs: Maximum number of GSO segments
274   *	@sk_pacing_shift: scaling factor for TCP Small Queues
275   *	@sk_lingertime: %SO_LINGER l_linger setting
276   *	@sk_backlog: always used with the per-socket spinlock held
277   *	@sk_callback_lock: used with the callbacks in the end of this struct
278   *	@sk_error_queue: rarely used
279   *	@sk_prot_creator: sk_prot of original sock creator (see ipv6_setsockopt,
280   *			  IPV6_ADDRFORM for instance)
281   *	@sk_err: last error
282   *	@sk_err_soft: errors that don't cause failure but are the cause of a
283   *		      persistent failure not just 'timed out'
284   *	@sk_drops: raw/udp drops counter
285   *	@sk_ack_backlog: current listen backlog
286   *	@sk_max_ack_backlog: listen backlog set in listen()
287   *	@sk_uid: user id of owner
288   *	@sk_prefer_busy_poll: prefer busypolling over softirq processing
289   *	@sk_busy_poll_budget: napi processing budget when busypolling
290   *	@sk_priority: %SO_PRIORITY setting
291   *	@sk_type: socket type (%SOCK_STREAM, etc)
292   *	@sk_protocol: which protocol this socket belongs in this network family
293   *	@sk_peer_lock: lock protecting @sk_peer_pid and @sk_peer_cred
294   *	@sk_peer_pid: &struct pid for this socket's peer
295   *	@sk_peer_cred: %SO_PEERCRED setting
296   *	@sk_rcvlowat: %SO_RCVLOWAT setting
297   *	@sk_rcvtimeo: %SO_RCVTIMEO setting
298   *	@sk_sndtimeo: %SO_SNDTIMEO setting
299   *	@sk_txhash: computed flow hash for use on transmit
300   *	@sk_txrehash: enable TX hash rethink
301   *	@sk_filter: socket filtering instructions
302   *	@sk_timer: sock cleanup timer
303   *	@sk_stamp: time stamp of last packet received
304   *	@sk_stamp_seq: lock for accessing sk_stamp on 32 bit architectures only
305   *	@sk_tsflags: SO_TIMESTAMPING flags
306   *	@sk_use_task_frag: allow sk_page_frag() to use current->task_frag.
307   *			   Sockets that can be used under memory reclaim should
308   *			   set this to false.
309   *	@sk_bind_phc: SO_TIMESTAMPING bind PHC index of PTP virtual clock
310   *	              for timestamping
311   *	@sk_tskey: counter to disambiguate concurrent tstamp requests
312   *	@sk_zckey: counter to order MSG_ZEROCOPY notifications
313   *	@sk_socket: Identd and reporting IO signals
314   *	@sk_user_data: RPC layer private data. Write-protected by @sk_callback_lock.
315   *	@sk_frag: cached page frag
316   *	@sk_peek_off: current peek_offset value
317   *	@sk_send_head: front of stuff to transmit
318   *	@tcp_rtx_queue: TCP re-transmit queue [union with @sk_send_head]
319   *	@sk_security: used by security modules
320   *	@sk_mark: generic packet mark
321   *	@sk_cgrp_data: cgroup data for this cgroup
322   *	@sk_memcg: this socket's memory cgroup association
323   *	@sk_write_pending: a write to stream socket waits to start
324   *	@sk_disconnects: number of disconnect operations performed on this sock
325   *	@sk_state_change: callback to indicate change in the state of the sock
326   *	@sk_data_ready: callback to indicate there is data to be processed
327   *	@sk_write_space: callback to indicate there is bf sending space available
328   *	@sk_error_report: callback to indicate errors (e.g. %MSG_ERRQUEUE)
329   *	@sk_backlog_rcv: callback to process the backlog
330   *	@sk_validate_xmit_skb: ptr to an optional validate function
331   *	@sk_destruct: called at sock freeing time, i.e. when all refcnt == 0
332   *	@sk_reuseport_cb: reuseport group container
333   *	@sk_bpf_storage: ptr to cache and control for bpf_sk_storage
334   *	@sk_rcu: used during RCU grace period
335   *	@sk_clockid: clockid used by time-based scheduling (SO_TXTIME)
336   *	@sk_txtime_deadline_mode: set deadline mode for SO_TXTIME
337   *	@sk_txtime_report_errors: set report errors mode for SO_TXTIME
338   *	@sk_txtime_unused: unused txtime flags
339   *	@ns_tracker: tracker for netns reference
340   */
341 struct sock {
342 	/*
343 	 * Now struct inet_timewait_sock also uses sock_common, so please just
344 	 * don't add nothing before this first member (__sk_common) --acme
345 	 */
346 	struct sock_common	__sk_common;
347 #define sk_node			__sk_common.skc_node
348 #define sk_nulls_node		__sk_common.skc_nulls_node
349 #define sk_refcnt		__sk_common.skc_refcnt
350 #define sk_tx_queue_mapping	__sk_common.skc_tx_queue_mapping
351 #ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
352 #define sk_rx_queue_mapping	__sk_common.skc_rx_queue_mapping
353 #endif
354 
355 #define sk_dontcopy_begin	__sk_common.skc_dontcopy_begin
356 #define sk_dontcopy_end		__sk_common.skc_dontcopy_end
357 #define sk_hash			__sk_common.skc_hash
358 #define sk_portpair		__sk_common.skc_portpair
359 #define sk_num			__sk_common.skc_num
360 #define sk_dport		__sk_common.skc_dport
361 #define sk_addrpair		__sk_common.skc_addrpair
362 #define sk_daddr		__sk_common.skc_daddr
363 #define sk_rcv_saddr		__sk_common.skc_rcv_saddr
364 #define sk_family		__sk_common.skc_family
365 #define sk_state		__sk_common.skc_state
366 #define sk_reuse		__sk_common.skc_reuse
367 #define sk_reuseport		__sk_common.skc_reuseport
368 #define sk_ipv6only		__sk_common.skc_ipv6only
369 #define sk_net_refcnt		__sk_common.skc_net_refcnt
370 #define sk_bound_dev_if		__sk_common.skc_bound_dev_if
371 #define sk_bind_node		__sk_common.skc_bind_node
372 #define sk_prot			__sk_common.skc_prot
373 #define sk_net			__sk_common.skc_net
374 #define sk_v6_daddr		__sk_common.skc_v6_daddr
375 #define sk_v6_rcv_saddr	__sk_common.skc_v6_rcv_saddr
376 #define sk_cookie		__sk_common.skc_cookie
377 #define sk_incoming_cpu		__sk_common.skc_incoming_cpu
378 #define sk_flags		__sk_common.skc_flags
379 #define sk_rxhash		__sk_common.skc_rxhash
380 
381 	__cacheline_group_begin(sock_write_rx);
382 
383 	atomic_t		sk_drops;
384 	__s32			sk_peek_off;
385 	struct sk_buff_head	sk_error_queue;
386 	struct sk_buff_head	sk_receive_queue;
387 	/*
388 	 * The backlog queue is special, it is always used with
389 	 * the per-socket spinlock held and requires low latency
390 	 * access. Therefore we special case it's implementation.
391 	 * Note : rmem_alloc is in this structure to fill a hole
392 	 * on 64bit arches, not because its logically part of
393 	 * backlog.
394 	 */
395 	struct {
396 		atomic_t	rmem_alloc;
397 		int		len;
398 		struct sk_buff	*head;
399 		struct sk_buff	*tail;
400 	} sk_backlog;
401 #define sk_rmem_alloc sk_backlog.rmem_alloc
402 
403 	__cacheline_group_end(sock_write_rx);
404 
405 	__cacheline_group_begin(sock_read_rx);
406 	/* early demux fields */
407 	struct dst_entry __rcu	*sk_rx_dst;
408 	int			sk_rx_dst_ifindex;
409 	u32			sk_rx_dst_cookie;
410 
411 #ifdef CONFIG_NET_RX_BUSY_POLL
412 	unsigned int		sk_ll_usec;
413 	unsigned int		sk_napi_id;
414 	u16			sk_busy_poll_budget;
415 	u8			sk_prefer_busy_poll;
416 #endif
417 	u8			sk_userlocks;
418 	int			sk_rcvbuf;
419 
420 	struct sk_filter __rcu	*sk_filter;
421 	union {
422 		struct socket_wq __rcu	*sk_wq;
423 		/* private: */
424 		struct socket_wq	*sk_wq_raw;
425 		/* public: */
426 	};
427 
428 	void			(*sk_data_ready)(struct sock *sk);
429 	long			sk_rcvtimeo;
430 	int			sk_rcvlowat;
431 	__cacheline_group_end(sock_read_rx);
432 
433 	__cacheline_group_begin(sock_read_rxtx);
434 	int			sk_err;
435 	struct socket		*sk_socket;
436 	struct mem_cgroup	*sk_memcg;
437 #ifdef CONFIG_XFRM
438 	struct xfrm_policy __rcu *sk_policy[2];
439 #endif
440 	__cacheline_group_end(sock_read_rxtx);
441 
442 	__cacheline_group_begin(sock_write_rxtx);
443 	socket_lock_t		sk_lock;
444 	u32			sk_reserved_mem;
445 	int			sk_forward_alloc;
446 	u32			sk_tsflags;
447 	__cacheline_group_end(sock_write_rxtx);
448 
449 	__cacheline_group_begin(sock_write_tx);
450 	int			sk_write_pending;
451 	atomic_t		sk_omem_alloc;
452 	int			sk_sndbuf;
453 
454 	int			sk_wmem_queued;
455 	refcount_t		sk_wmem_alloc;
456 	unsigned long		sk_tsq_flags;
457 	union {
458 		struct sk_buff	*sk_send_head;
459 		struct rb_root	tcp_rtx_queue;
460 	};
461 	struct sk_buff_head	sk_write_queue;
462 	u32			sk_dst_pending_confirm;
463 	u32			sk_pacing_status; /* see enum sk_pacing */
464 	struct page_frag	sk_frag;
465 	struct timer_list	sk_timer;
466 
467 	unsigned long		sk_pacing_rate; /* bytes per second */
468 	atomic_t		sk_zckey;
469 	atomic_t		sk_tskey;
470 	__cacheline_group_end(sock_write_tx);
471 
472 	__cacheline_group_begin(sock_read_tx);
473 	unsigned long		sk_max_pacing_rate;
474 	long			sk_sndtimeo;
475 	u32			sk_priority;
476 	u32			sk_mark;
477 	struct dst_entry __rcu	*sk_dst_cache;
478 	netdev_features_t	sk_route_caps;
479 #ifdef CONFIG_SOCK_VALIDATE_XMIT
480 	struct sk_buff*		(*sk_validate_xmit_skb)(struct sock *sk,
481 							struct net_device *dev,
482 							struct sk_buff *skb);
483 #endif
484 	u16			sk_gso_type;
485 	u16			sk_gso_max_segs;
486 	unsigned int		sk_gso_max_size;
487 	gfp_t			sk_allocation;
488 	u32			sk_txhash;
489 	u8			sk_pacing_shift;
490 	bool			sk_use_task_frag;
491 	__cacheline_group_end(sock_read_tx);
492 
493 	/*
494 	 * Because of non atomicity rules, all
495 	 * changes are protected by socket lock.
496 	 */
497 	u8			sk_gso_disabled : 1,
498 				sk_kern_sock : 1,
499 				sk_no_check_tx : 1,
500 				sk_no_check_rx : 1;
501 	u8			sk_shutdown;
502 	u16			sk_type;
503 	u16			sk_protocol;
504 	unsigned long	        sk_lingertime;
505 	struct proto		*sk_prot_creator;
506 	rwlock_t		sk_callback_lock;
507 	int			sk_err_soft;
508 	u32			sk_ack_backlog;
509 	u32			sk_max_ack_backlog;
510 	kuid_t			sk_uid;
511 	spinlock_t		sk_peer_lock;
512 	int			sk_bind_phc;
513 	struct pid		*sk_peer_pid;
514 	const struct cred	*sk_peer_cred;
515 
516 	ktime_t			sk_stamp;
517 #if BITS_PER_LONG==32
518 	seqlock_t		sk_stamp_seq;
519 #endif
520 	int			sk_disconnects;
521 
522 	u8			sk_txrehash;
523 	u8			sk_clockid;
524 	u8			sk_txtime_deadline_mode : 1,
525 				sk_txtime_report_errors : 1,
526 				sk_txtime_unused : 6;
527 
528 	void			*sk_user_data;
529 #ifdef CONFIG_SECURITY
530 	void			*sk_security;
531 #endif
532 	struct sock_cgroup_data	sk_cgrp_data;
533 	void			(*sk_state_change)(struct sock *sk);
534 	void			(*sk_write_space)(struct sock *sk);
535 	void			(*sk_error_report)(struct sock *sk);
536 	int			(*sk_backlog_rcv)(struct sock *sk,
537 						  struct sk_buff *skb);
538 	void                    (*sk_destruct)(struct sock *sk);
539 	struct sock_reuseport __rcu	*sk_reuseport_cb;
540 #ifdef CONFIG_BPF_SYSCALL
541 	struct bpf_local_storage __rcu	*sk_bpf_storage;
542 #endif
543 	struct rcu_head		sk_rcu;
544 	netns_tracker		ns_tracker;
545 };
546 
547 enum sk_pacing {
548 	SK_PACING_NONE		= 0,
549 	SK_PACING_NEEDED	= 1,
550 	SK_PACING_FQ		= 2,
551 };
552 
553 /* flag bits in sk_user_data
554  *
555  * - SK_USER_DATA_NOCOPY:      Pointer stored in sk_user_data might
556  *   not be suitable for copying when cloning the socket. For instance,
557  *   it can point to a reference counted object. sk_user_data bottom
558  *   bit is set if pointer must not be copied.
559  *
560  * - SK_USER_DATA_BPF:         Mark whether sk_user_data field is
561  *   managed/owned by a BPF reuseport array. This bit should be set
562  *   when sk_user_data's sk is added to the bpf's reuseport_array.
563  *
564  * - SK_USER_DATA_PSOCK:       Mark whether pointer stored in
565  *   sk_user_data points to psock type. This bit should be set
566  *   when sk_user_data is assigned to a psock object.
567  */
568 #define SK_USER_DATA_NOCOPY	1UL
569 #define SK_USER_DATA_BPF	2UL
570 #define SK_USER_DATA_PSOCK	4UL
571 #define SK_USER_DATA_PTRMASK	~(SK_USER_DATA_NOCOPY | SK_USER_DATA_BPF |\
572 				  SK_USER_DATA_PSOCK)
573 
574 /**
575  * sk_user_data_is_nocopy - Test if sk_user_data pointer must not be copied
576  * @sk: socket
577  */
578 static inline bool sk_user_data_is_nocopy(const struct sock *sk)
579 {
580 	return ((uintptr_t)sk->sk_user_data & SK_USER_DATA_NOCOPY);
581 }
582 
583 #define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data)))
584 
585 /**
586  * __locked_read_sk_user_data_with_flags - return the pointer
587  * only if argument flags all has been set in sk_user_data. Otherwise
588  * return NULL
589  *
590  * @sk: socket
591  * @flags: flag bits
592  *
593  * The caller must be holding sk->sk_callback_lock.
594  */
595 static inline void *
596 __locked_read_sk_user_data_with_flags(const struct sock *sk,
597 				      uintptr_t flags)
598 {
599 	uintptr_t sk_user_data =
600 		(uintptr_t)rcu_dereference_check(__sk_user_data(sk),
601 						 lockdep_is_held(&sk->sk_callback_lock));
602 
603 	WARN_ON_ONCE(flags & SK_USER_DATA_PTRMASK);
604 
605 	if ((sk_user_data & flags) == flags)
606 		return (void *)(sk_user_data & SK_USER_DATA_PTRMASK);
607 	return NULL;
608 }
609 
610 /**
611  * __rcu_dereference_sk_user_data_with_flags - return the pointer
612  * only if argument flags all has been set in sk_user_data. Otherwise
613  * return NULL
614  *
615  * @sk: socket
616  * @flags: flag bits
617  */
618 static inline void *
619 __rcu_dereference_sk_user_data_with_flags(const struct sock *sk,
620 					  uintptr_t flags)
621 {
622 	uintptr_t sk_user_data = (uintptr_t)rcu_dereference(__sk_user_data(sk));
623 
624 	WARN_ON_ONCE(flags & SK_USER_DATA_PTRMASK);
625 
626 	if ((sk_user_data & flags) == flags)
627 		return (void *)(sk_user_data & SK_USER_DATA_PTRMASK);
628 	return NULL;
629 }
630 
631 #define rcu_dereference_sk_user_data(sk)				\
632 	__rcu_dereference_sk_user_data_with_flags(sk, 0)
633 #define __rcu_assign_sk_user_data_with_flags(sk, ptr, flags)		\
634 ({									\
635 	uintptr_t __tmp1 = (uintptr_t)(ptr),				\
636 		  __tmp2 = (uintptr_t)(flags);				\
637 	WARN_ON_ONCE(__tmp1 & ~SK_USER_DATA_PTRMASK);			\
638 	WARN_ON_ONCE(__tmp2 & SK_USER_DATA_PTRMASK);			\
639 	rcu_assign_pointer(__sk_user_data((sk)),			\
640 			   __tmp1 | __tmp2);				\
641 })
642 #define rcu_assign_sk_user_data(sk, ptr)				\
643 	__rcu_assign_sk_user_data_with_flags(sk, ptr, 0)
644 
645 static inline
646 struct net *sock_net(const struct sock *sk)
647 {
648 	return read_pnet(&sk->sk_net);
649 }
650 
651 static inline
652 void sock_net_set(struct sock *sk, struct net *net)
653 {
654 	write_pnet(&sk->sk_net, net);
655 }
656 
657 /*
658  * SK_CAN_REUSE and SK_NO_REUSE on a socket mean that the socket is OK
659  * or not whether his port will be reused by someone else. SK_FORCE_REUSE
660  * on a socket means that the socket will reuse everybody else's port
661  * without looking at the other's sk_reuse value.
662  */
663 
664 #define SK_NO_REUSE	0
665 #define SK_CAN_REUSE	1
666 #define SK_FORCE_REUSE	2
667 
668 int sk_set_peek_off(struct sock *sk, int val);
669 
670 static inline int sk_peek_offset(const struct sock *sk, int flags)
671 {
672 	if (unlikely(flags & MSG_PEEK)) {
673 		return READ_ONCE(sk->sk_peek_off);
674 	}
675 
676 	return 0;
677 }
678 
679 static inline void sk_peek_offset_bwd(struct sock *sk, int val)
680 {
681 	s32 off = READ_ONCE(sk->sk_peek_off);
682 
683 	if (unlikely(off >= 0)) {
684 		off = max_t(s32, off - val, 0);
685 		WRITE_ONCE(sk->sk_peek_off, off);
686 	}
687 }
688 
689 static inline void sk_peek_offset_fwd(struct sock *sk, int val)
690 {
691 	sk_peek_offset_bwd(sk, -val);
692 }
693 
694 /*
695  * Hashed lists helper routines
696  */
697 static inline struct sock *sk_entry(const struct hlist_node *node)
698 {
699 	return hlist_entry(node, struct sock, sk_node);
700 }
701 
702 static inline struct sock *__sk_head(const struct hlist_head *head)
703 {
704 	return hlist_entry(head->first, struct sock, sk_node);
705 }
706 
707 static inline struct sock *sk_head(const struct hlist_head *head)
708 {
709 	return hlist_empty(head) ? NULL : __sk_head(head);
710 }
711 
712 static inline struct sock *__sk_nulls_head(const struct hlist_nulls_head *head)
713 {
714 	return hlist_nulls_entry(head->first, struct sock, sk_nulls_node);
715 }
716 
717 static inline struct sock *sk_nulls_head(const struct hlist_nulls_head *head)
718 {
719 	return hlist_nulls_empty(head) ? NULL : __sk_nulls_head(head);
720 }
721 
722 static inline struct sock *sk_next(const struct sock *sk)
723 {
724 	return hlist_entry_safe(sk->sk_node.next, struct sock, sk_node);
725 }
726 
727 static inline struct sock *sk_nulls_next(const struct sock *sk)
728 {
729 	return (!is_a_nulls(sk->sk_nulls_node.next)) ?
730 		hlist_nulls_entry(sk->sk_nulls_node.next,
731 				  struct sock, sk_nulls_node) :
732 		NULL;
733 }
734 
735 static inline bool sk_unhashed(const struct sock *sk)
736 {
737 	return hlist_unhashed(&sk->sk_node);
738 }
739 
740 static inline bool sk_hashed(const struct sock *sk)
741 {
742 	return !sk_unhashed(sk);
743 }
744 
745 static inline void sk_node_init(struct hlist_node *node)
746 {
747 	node->pprev = NULL;
748 }
749 
750 static inline void __sk_del_node(struct sock *sk)
751 {
752 	__hlist_del(&sk->sk_node);
753 }
754 
755 /* NB: equivalent to hlist_del_init_rcu */
756 static inline bool __sk_del_node_init(struct sock *sk)
757 {
758 	if (sk_hashed(sk)) {
759 		__sk_del_node(sk);
760 		sk_node_init(&sk->sk_node);
761 		return true;
762 	}
763 	return false;
764 }
765 
766 /* Grab socket reference count. This operation is valid only
767    when sk is ALREADY grabbed f.e. it is found in hash table
768    or a list and the lookup is made under lock preventing hash table
769    modifications.
770  */
771 
772 static __always_inline void sock_hold(struct sock *sk)
773 {
774 	refcount_inc(&sk->sk_refcnt);
775 }
776 
777 /* Ungrab socket in the context, which assumes that socket refcnt
778    cannot hit zero, f.e. it is true in context of any socketcall.
779  */
780 static __always_inline void __sock_put(struct sock *sk)
781 {
782 	refcount_dec(&sk->sk_refcnt);
783 }
784 
785 static inline bool sk_del_node_init(struct sock *sk)
786 {
787 	bool rc = __sk_del_node_init(sk);
788 
789 	if (rc) {
790 		/* paranoid for a while -acme */
791 		WARN_ON(refcount_read(&sk->sk_refcnt) == 1);
792 		__sock_put(sk);
793 	}
794 	return rc;
795 }
796 #define sk_del_node_init_rcu(sk)	sk_del_node_init(sk)
797 
798 static inline bool __sk_nulls_del_node_init_rcu(struct sock *sk)
799 {
800 	if (sk_hashed(sk)) {
801 		hlist_nulls_del_init_rcu(&sk->sk_nulls_node);
802 		return true;
803 	}
804 	return false;
805 }
806 
807 static inline bool sk_nulls_del_node_init_rcu(struct sock *sk)
808 {
809 	bool rc = __sk_nulls_del_node_init_rcu(sk);
810 
811 	if (rc) {
812 		/* paranoid for a while -acme */
813 		WARN_ON(refcount_read(&sk->sk_refcnt) == 1);
814 		__sock_put(sk);
815 	}
816 	return rc;
817 }
818 
819 static inline void __sk_add_node(struct sock *sk, struct hlist_head *list)
820 {
821 	hlist_add_head(&sk->sk_node, list);
822 }
823 
824 static inline void sk_add_node(struct sock *sk, struct hlist_head *list)
825 {
826 	sock_hold(sk);
827 	__sk_add_node(sk, list);
828 }
829 
830 static inline void sk_add_node_rcu(struct sock *sk, struct hlist_head *list)
831 {
832 	sock_hold(sk);
833 	if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
834 	    sk->sk_family == AF_INET6)
835 		hlist_add_tail_rcu(&sk->sk_node, list);
836 	else
837 		hlist_add_head_rcu(&sk->sk_node, list);
838 }
839 
840 static inline void sk_add_node_tail_rcu(struct sock *sk, struct hlist_head *list)
841 {
842 	sock_hold(sk);
843 	hlist_add_tail_rcu(&sk->sk_node, list);
844 }
845 
846 static inline void __sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
847 {
848 	hlist_nulls_add_head_rcu(&sk->sk_nulls_node, list);
849 }
850 
851 static inline void __sk_nulls_add_node_tail_rcu(struct sock *sk, struct hlist_nulls_head *list)
852 {
853 	hlist_nulls_add_tail_rcu(&sk->sk_nulls_node, list);
854 }
855 
856 static inline void sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
857 {
858 	sock_hold(sk);
859 	__sk_nulls_add_node_rcu(sk, list);
860 }
861 
862 static inline void __sk_del_bind_node(struct sock *sk)
863 {
864 	__hlist_del(&sk->sk_bind_node);
865 }
866 
867 static inline void sk_add_bind_node(struct sock *sk,
868 					struct hlist_head *list)
869 {
870 	hlist_add_head(&sk->sk_bind_node, list);
871 }
872 
873 #define sk_for_each(__sk, list) \
874 	hlist_for_each_entry(__sk, list, sk_node)
875 #define sk_for_each_rcu(__sk, list) \
876 	hlist_for_each_entry_rcu(__sk, list, sk_node)
877 #define sk_nulls_for_each(__sk, node, list) \
878 	hlist_nulls_for_each_entry(__sk, node, list, sk_nulls_node)
879 #define sk_nulls_for_each_rcu(__sk, node, list) \
880 	hlist_nulls_for_each_entry_rcu(__sk, node, list, sk_nulls_node)
881 #define sk_for_each_from(__sk) \
882 	hlist_for_each_entry_from(__sk, sk_node)
883 #define sk_nulls_for_each_from(__sk, node) \
884 	if (__sk && ({ node = &(__sk)->sk_nulls_node; 1; })) \
885 		hlist_nulls_for_each_entry_from(__sk, node, sk_nulls_node)
886 #define sk_for_each_safe(__sk, tmp, list) \
887 	hlist_for_each_entry_safe(__sk, tmp, list, sk_node)
888 #define sk_for_each_bound(__sk, list) \
889 	hlist_for_each_entry(__sk, list, sk_bind_node)
890 
891 /**
892  * sk_for_each_entry_offset_rcu - iterate over a list at a given struct offset
893  * @tpos:	the type * to use as a loop cursor.
894  * @pos:	the &struct hlist_node to use as a loop cursor.
895  * @head:	the head for your list.
896  * @offset:	offset of hlist_node within the struct.
897  *
898  */
899 #define sk_for_each_entry_offset_rcu(tpos, pos, head, offset)		       \
900 	for (pos = rcu_dereference(hlist_first_rcu(head));		       \
901 	     pos != NULL &&						       \
902 		({ tpos = (typeof(*tpos) *)((void *)pos - offset); 1;});       \
903 	     pos = rcu_dereference(hlist_next_rcu(pos)))
904 
905 static inline struct user_namespace *sk_user_ns(const struct sock *sk)
906 {
907 	/* Careful only use this in a context where these parameters
908 	 * can not change and must all be valid, such as recvmsg from
909 	 * userspace.
910 	 */
911 	return sk->sk_socket->file->f_cred->user_ns;
912 }
913 
914 /* Sock flags */
915 enum sock_flags {
916 	SOCK_DEAD,
917 	SOCK_DONE,
918 	SOCK_URGINLINE,
919 	SOCK_KEEPOPEN,
920 	SOCK_LINGER,
921 	SOCK_DESTROY,
922 	SOCK_BROADCAST,
923 	SOCK_TIMESTAMP,
924 	SOCK_ZAPPED,
925 	SOCK_USE_WRITE_QUEUE, /* whether to call sk->sk_write_space in sock_wfree */
926 	SOCK_DBG, /* %SO_DEBUG setting */
927 	SOCK_RCVTSTAMP, /* %SO_TIMESTAMP setting */
928 	SOCK_RCVTSTAMPNS, /* %SO_TIMESTAMPNS setting */
929 	SOCK_LOCALROUTE, /* route locally only, %SO_DONTROUTE setting */
930 	SOCK_MEMALLOC, /* VM depends on this socket for swapping */
931 	SOCK_TIMESTAMPING_RX_SOFTWARE,  /* %SOF_TIMESTAMPING_RX_SOFTWARE */
932 	SOCK_FASYNC, /* fasync() active */
933 	SOCK_RXQ_OVFL,
934 	SOCK_ZEROCOPY, /* buffers from userspace */
935 	SOCK_WIFI_STATUS, /* push wifi status to userspace */
936 	SOCK_NOFCS, /* Tell NIC not to do the Ethernet FCS.
937 		     * Will use last 4 bytes of packet sent from
938 		     * user-space instead.
939 		     */
940 	SOCK_FILTER_LOCKED, /* Filter cannot be changed anymore */
941 	SOCK_SELECT_ERR_QUEUE, /* Wake select on error queue */
942 	SOCK_RCU_FREE, /* wait rcu grace period in sk_destruct() */
943 	SOCK_TXTIME,
944 	SOCK_XDP, /* XDP is attached */
945 	SOCK_TSTAMP_NEW, /* Indicates 64 bit timestamps always */
946 	SOCK_RCVMARK, /* Receive SO_MARK  ancillary data with packet */
947 };
948 
949 #define SK_FLAGS_TIMESTAMP ((1UL << SOCK_TIMESTAMP) | (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE))
950 
951 static inline void sock_copy_flags(struct sock *nsk, const struct sock *osk)
952 {
953 	nsk->sk_flags = osk->sk_flags;
954 }
955 
956 static inline void sock_set_flag(struct sock *sk, enum sock_flags flag)
957 {
958 	__set_bit(flag, &sk->sk_flags);
959 }
960 
961 static inline void sock_reset_flag(struct sock *sk, enum sock_flags flag)
962 {
963 	__clear_bit(flag, &sk->sk_flags);
964 }
965 
966 static inline void sock_valbool_flag(struct sock *sk, enum sock_flags bit,
967 				     int valbool)
968 {
969 	if (valbool)
970 		sock_set_flag(sk, bit);
971 	else
972 		sock_reset_flag(sk, bit);
973 }
974 
975 static inline bool sock_flag(const struct sock *sk, enum sock_flags flag)
976 {
977 	return test_bit(flag, &sk->sk_flags);
978 }
979 
980 #ifdef CONFIG_NET
981 DECLARE_STATIC_KEY_FALSE(memalloc_socks_key);
982 static inline int sk_memalloc_socks(void)
983 {
984 	return static_branch_unlikely(&memalloc_socks_key);
985 }
986 
987 void __receive_sock(struct file *file);
988 #else
989 
990 static inline int sk_memalloc_socks(void)
991 {
992 	return 0;
993 }
994 
995 static inline void __receive_sock(struct file *file)
996 { }
997 #endif
998 
999 static inline gfp_t sk_gfp_mask(const struct sock *sk, gfp_t gfp_mask)
1000 {
1001 	return gfp_mask | (sk->sk_allocation & __GFP_MEMALLOC);
1002 }
1003 
1004 static inline void sk_acceptq_removed(struct sock *sk)
1005 {
1006 	WRITE_ONCE(sk->sk_ack_backlog, sk->sk_ack_backlog - 1);
1007 }
1008 
1009 static inline void sk_acceptq_added(struct sock *sk)
1010 {
1011 	WRITE_ONCE(sk->sk_ack_backlog, sk->sk_ack_backlog + 1);
1012 }
1013 
1014 /* Note: If you think the test should be:
1015  *	return READ_ONCE(sk->sk_ack_backlog) >= READ_ONCE(sk->sk_max_ack_backlog);
1016  * Then please take a look at commit 64a146513f8f ("[NET]: Revert incorrect accept queue backlog changes.")
1017  */
1018 static inline bool sk_acceptq_is_full(const struct sock *sk)
1019 {
1020 	return READ_ONCE(sk->sk_ack_backlog) > READ_ONCE(sk->sk_max_ack_backlog);
1021 }
1022 
1023 /*
1024  * Compute minimal free write space needed to queue new packets.
1025  */
1026 static inline int sk_stream_min_wspace(const struct sock *sk)
1027 {
1028 	return READ_ONCE(sk->sk_wmem_queued) >> 1;
1029 }
1030 
1031 static inline int sk_stream_wspace(const struct sock *sk)
1032 {
1033 	return READ_ONCE(sk->sk_sndbuf) - READ_ONCE(sk->sk_wmem_queued);
1034 }
1035 
1036 static inline void sk_wmem_queued_add(struct sock *sk, int val)
1037 {
1038 	WRITE_ONCE(sk->sk_wmem_queued, sk->sk_wmem_queued + val);
1039 }
1040 
1041 static inline void sk_forward_alloc_add(struct sock *sk, int val)
1042 {
1043 	/* Paired with lockless reads of sk->sk_forward_alloc */
1044 	WRITE_ONCE(sk->sk_forward_alloc, sk->sk_forward_alloc + val);
1045 }
1046 
1047 void sk_stream_write_space(struct sock *sk);
1048 
1049 /* OOB backlog add */
1050 static inline void __sk_add_backlog(struct sock *sk, struct sk_buff *skb)
1051 {
1052 	/* dont let skb dst not refcounted, we are going to leave rcu lock */
1053 	skb_dst_force(skb);
1054 
1055 	if (!sk->sk_backlog.tail)
1056 		WRITE_ONCE(sk->sk_backlog.head, skb);
1057 	else
1058 		sk->sk_backlog.tail->next = skb;
1059 
1060 	WRITE_ONCE(sk->sk_backlog.tail, skb);
1061 	skb->next = NULL;
1062 }
1063 
1064 /*
1065  * Take into account size of receive queue and backlog queue
1066  * Do not take into account this skb truesize,
1067  * to allow even a single big packet to come.
1068  */
1069 static inline bool sk_rcvqueues_full(const struct sock *sk, unsigned int limit)
1070 {
1071 	unsigned int qsize = sk->sk_backlog.len + atomic_read(&sk->sk_rmem_alloc);
1072 
1073 	return qsize > limit;
1074 }
1075 
1076 /* The per-socket spinlock must be held here. */
1077 static inline __must_check int sk_add_backlog(struct sock *sk, struct sk_buff *skb,
1078 					      unsigned int limit)
1079 {
1080 	if (sk_rcvqueues_full(sk, limit))
1081 		return -ENOBUFS;
1082 
1083 	/*
1084 	 * If the skb was allocated from pfmemalloc reserves, only
1085 	 * allow SOCK_MEMALLOC sockets to use it as this socket is
1086 	 * helping free memory
1087 	 */
1088 	if (skb_pfmemalloc(skb) && !sock_flag(sk, SOCK_MEMALLOC))
1089 		return -ENOMEM;
1090 
1091 	__sk_add_backlog(sk, skb);
1092 	sk->sk_backlog.len += skb->truesize;
1093 	return 0;
1094 }
1095 
1096 int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb);
1097 
1098 INDIRECT_CALLABLE_DECLARE(int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb));
1099 INDIRECT_CALLABLE_DECLARE(int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb));
1100 
1101 static inline int sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
1102 {
1103 	if (sk_memalloc_socks() && skb_pfmemalloc(skb))
1104 		return __sk_backlog_rcv(sk, skb);
1105 
1106 	return INDIRECT_CALL_INET(sk->sk_backlog_rcv,
1107 				  tcp_v6_do_rcv,
1108 				  tcp_v4_do_rcv,
1109 				  sk, skb);
1110 }
1111 
1112 static inline void sk_incoming_cpu_update(struct sock *sk)
1113 {
1114 	int cpu = raw_smp_processor_id();
1115 
1116 	if (unlikely(READ_ONCE(sk->sk_incoming_cpu) != cpu))
1117 		WRITE_ONCE(sk->sk_incoming_cpu, cpu);
1118 }
1119 
1120 
1121 static inline void sock_rps_save_rxhash(struct sock *sk,
1122 					const struct sk_buff *skb)
1123 {
1124 #ifdef CONFIG_RPS
1125 	/* The following WRITE_ONCE() is paired with the READ_ONCE()
1126 	 * here, and another one in sock_rps_record_flow().
1127 	 */
1128 	if (unlikely(READ_ONCE(sk->sk_rxhash) != skb->hash))
1129 		WRITE_ONCE(sk->sk_rxhash, skb->hash);
1130 #endif
1131 }
1132 
1133 static inline void sock_rps_reset_rxhash(struct sock *sk)
1134 {
1135 #ifdef CONFIG_RPS
1136 	/* Paired with READ_ONCE() in sock_rps_record_flow() */
1137 	WRITE_ONCE(sk->sk_rxhash, 0);
1138 #endif
1139 }
1140 
1141 #define sk_wait_event(__sk, __timeo, __condition, __wait)		\
1142 	({	int __rc, __dis = __sk->sk_disconnects;			\
1143 		release_sock(__sk);					\
1144 		__rc = __condition;					\
1145 		if (!__rc) {						\
1146 			*(__timeo) = wait_woken(__wait,			\
1147 						TASK_INTERRUPTIBLE,	\
1148 						*(__timeo));		\
1149 		}							\
1150 		sched_annotate_sleep();					\
1151 		lock_sock(__sk);					\
1152 		__rc = __dis == __sk->sk_disconnects ? __condition : -EPIPE; \
1153 		__rc;							\
1154 	})
1155 
1156 int sk_stream_wait_connect(struct sock *sk, long *timeo_p);
1157 int sk_stream_wait_memory(struct sock *sk, long *timeo_p);
1158 void sk_stream_wait_close(struct sock *sk, long timeo_p);
1159 int sk_stream_error(struct sock *sk, int flags, int err);
1160 void sk_stream_kill_queues(struct sock *sk);
1161 void sk_set_memalloc(struct sock *sk);
1162 void sk_clear_memalloc(struct sock *sk);
1163 
1164 void __sk_flush_backlog(struct sock *sk);
1165 
1166 static inline bool sk_flush_backlog(struct sock *sk)
1167 {
1168 	if (unlikely(READ_ONCE(sk->sk_backlog.tail))) {
1169 		__sk_flush_backlog(sk);
1170 		return true;
1171 	}
1172 	return false;
1173 }
1174 
1175 int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb);
1176 
1177 struct request_sock_ops;
1178 struct timewait_sock_ops;
1179 struct inet_hashinfo;
1180 struct raw_hashinfo;
1181 struct smc_hashinfo;
1182 struct module;
1183 struct sk_psock;
1184 
1185 /*
1186  * caches using SLAB_TYPESAFE_BY_RCU should let .next pointer from nulls nodes
1187  * un-modified. Special care is taken when initializing object to zero.
1188  */
1189 static inline void sk_prot_clear_nulls(struct sock *sk, int size)
1190 {
1191 	if (offsetof(struct sock, sk_node.next) != 0)
1192 		memset(sk, 0, offsetof(struct sock, sk_node.next));
1193 	memset(&sk->sk_node.pprev, 0,
1194 	       size - offsetof(struct sock, sk_node.pprev));
1195 }
1196 
1197 /* Networking protocol blocks we attach to sockets.
1198  * socket layer -> transport layer interface
1199  */
1200 struct proto {
1201 	void			(*close)(struct sock *sk,
1202 					long timeout);
1203 	int			(*pre_connect)(struct sock *sk,
1204 					struct sockaddr *uaddr,
1205 					int addr_len);
1206 	int			(*connect)(struct sock *sk,
1207 					struct sockaddr *uaddr,
1208 					int addr_len);
1209 	int			(*disconnect)(struct sock *sk, int flags);
1210 
1211 	struct sock *		(*accept)(struct sock *sk, int flags, int *err,
1212 					  bool kern);
1213 
1214 	int			(*ioctl)(struct sock *sk, int cmd,
1215 					 int *karg);
1216 	int			(*init)(struct sock *sk);
1217 	void			(*destroy)(struct sock *sk);
1218 	void			(*shutdown)(struct sock *sk, int how);
1219 	int			(*setsockopt)(struct sock *sk, int level,
1220 					int optname, sockptr_t optval,
1221 					unsigned int optlen);
1222 	int			(*getsockopt)(struct sock *sk, int level,
1223 					int optname, char __user *optval,
1224 					int __user *option);
1225 	void			(*keepalive)(struct sock *sk, int valbool);
1226 #ifdef CONFIG_COMPAT
1227 	int			(*compat_ioctl)(struct sock *sk,
1228 					unsigned int cmd, unsigned long arg);
1229 #endif
1230 	int			(*sendmsg)(struct sock *sk, struct msghdr *msg,
1231 					   size_t len);
1232 	int			(*recvmsg)(struct sock *sk, struct msghdr *msg,
1233 					   size_t len, int flags, int *addr_len);
1234 	void			(*splice_eof)(struct socket *sock);
1235 	int			(*bind)(struct sock *sk,
1236 					struct sockaddr *addr, int addr_len);
1237 	int			(*bind_add)(struct sock *sk,
1238 					struct sockaddr *addr, int addr_len);
1239 
1240 	int			(*backlog_rcv) (struct sock *sk,
1241 						struct sk_buff *skb);
1242 	bool			(*bpf_bypass_getsockopt)(int level,
1243 							 int optname);
1244 
1245 	void		(*release_cb)(struct sock *sk);
1246 
1247 	/* Keeping track of sk's, looking them up, and port selection methods. */
1248 	int			(*hash)(struct sock *sk);
1249 	void			(*unhash)(struct sock *sk);
1250 	void			(*rehash)(struct sock *sk);
1251 	int			(*get_port)(struct sock *sk, unsigned short snum);
1252 	void			(*put_port)(struct sock *sk);
1253 #ifdef CONFIG_BPF_SYSCALL
1254 	int			(*psock_update_sk_prot)(struct sock *sk,
1255 							struct sk_psock *psock,
1256 							bool restore);
1257 #endif
1258 
1259 	/* Keeping track of sockets in use */
1260 #ifdef CONFIG_PROC_FS
1261 	unsigned int		inuse_idx;
1262 #endif
1263 
1264 #if IS_ENABLED(CONFIG_MPTCP)
1265 	int			(*forward_alloc_get)(const struct sock *sk);
1266 #endif
1267 
1268 	bool			(*stream_memory_free)(const struct sock *sk, int wake);
1269 	bool			(*sock_is_readable)(struct sock *sk);
1270 	/* Memory pressure */
1271 	void			(*enter_memory_pressure)(struct sock *sk);
1272 	void			(*leave_memory_pressure)(struct sock *sk);
1273 	atomic_long_t		*memory_allocated;	/* Current allocated memory. */
1274 	int  __percpu		*per_cpu_fw_alloc;
1275 	struct percpu_counter	*sockets_allocated;	/* Current number of sockets. */
1276 
1277 	/*
1278 	 * Pressure flag: try to collapse.
1279 	 * Technical note: it is used by multiple contexts non atomically.
1280 	 * Make sure to use READ_ONCE()/WRITE_ONCE() for all reads/writes.
1281 	 * All the __sk_mem_schedule() is of this nature: accounting
1282 	 * is strict, actions are advisory and have some latency.
1283 	 */
1284 	unsigned long		*memory_pressure;
1285 	long			*sysctl_mem;
1286 
1287 	int			*sysctl_wmem;
1288 	int			*sysctl_rmem;
1289 	u32			sysctl_wmem_offset;
1290 	u32			sysctl_rmem_offset;
1291 
1292 	int			max_header;
1293 	bool			no_autobind;
1294 
1295 	struct kmem_cache	*slab;
1296 	unsigned int		obj_size;
1297 	unsigned int		ipv6_pinfo_offset;
1298 	slab_flags_t		slab_flags;
1299 	unsigned int		useroffset;	/* Usercopy region offset */
1300 	unsigned int		usersize;	/* Usercopy region size */
1301 
1302 	unsigned int __percpu	*orphan_count;
1303 
1304 	struct request_sock_ops	*rsk_prot;
1305 	struct timewait_sock_ops *twsk_prot;
1306 
1307 	union {
1308 		struct inet_hashinfo	*hashinfo;
1309 		struct udp_table	*udp_table;
1310 		struct raw_hashinfo	*raw_hash;
1311 		struct smc_hashinfo	*smc_hash;
1312 	} h;
1313 
1314 	struct module		*owner;
1315 
1316 	char			name[32];
1317 
1318 	struct list_head	node;
1319 	int			(*diag_destroy)(struct sock *sk, int err);
1320 } __randomize_layout;
1321 
1322 int proto_register(struct proto *prot, int alloc_slab);
1323 void proto_unregister(struct proto *prot);
1324 int sock_load_diag_module(int family, int protocol);
1325 
1326 INDIRECT_CALLABLE_DECLARE(bool tcp_stream_memory_free(const struct sock *sk, int wake));
1327 
1328 static inline int sk_forward_alloc_get(const struct sock *sk)
1329 {
1330 #if IS_ENABLED(CONFIG_MPTCP)
1331 	if (sk->sk_prot->forward_alloc_get)
1332 		return sk->sk_prot->forward_alloc_get(sk);
1333 #endif
1334 	return READ_ONCE(sk->sk_forward_alloc);
1335 }
1336 
1337 static inline bool __sk_stream_memory_free(const struct sock *sk, int wake)
1338 {
1339 	if (READ_ONCE(sk->sk_wmem_queued) >= READ_ONCE(sk->sk_sndbuf))
1340 		return false;
1341 
1342 	return sk->sk_prot->stream_memory_free ?
1343 		INDIRECT_CALL_INET_1(sk->sk_prot->stream_memory_free,
1344 				     tcp_stream_memory_free, sk, wake) : true;
1345 }
1346 
1347 static inline bool sk_stream_memory_free(const struct sock *sk)
1348 {
1349 	return __sk_stream_memory_free(sk, 0);
1350 }
1351 
1352 static inline bool __sk_stream_is_writeable(const struct sock *sk, int wake)
1353 {
1354 	return sk_stream_wspace(sk) >= sk_stream_min_wspace(sk) &&
1355 	       __sk_stream_memory_free(sk, wake);
1356 }
1357 
1358 static inline bool sk_stream_is_writeable(const struct sock *sk)
1359 {
1360 	return __sk_stream_is_writeable(sk, 0);
1361 }
1362 
1363 static inline int sk_under_cgroup_hierarchy(struct sock *sk,
1364 					    struct cgroup *ancestor)
1365 {
1366 #ifdef CONFIG_SOCK_CGROUP_DATA
1367 	return cgroup_is_descendant(sock_cgroup_ptr(&sk->sk_cgrp_data),
1368 				    ancestor);
1369 #else
1370 	return -ENOTSUPP;
1371 #endif
1372 }
1373 
1374 static inline bool sk_has_memory_pressure(const struct sock *sk)
1375 {
1376 	return sk->sk_prot->memory_pressure != NULL;
1377 }
1378 
1379 static inline bool sk_under_global_memory_pressure(const struct sock *sk)
1380 {
1381 	return sk->sk_prot->memory_pressure &&
1382 		!!READ_ONCE(*sk->sk_prot->memory_pressure);
1383 }
1384 
1385 static inline bool sk_under_memory_pressure(const struct sock *sk)
1386 {
1387 	if (!sk->sk_prot->memory_pressure)
1388 		return false;
1389 
1390 	if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
1391 	    mem_cgroup_under_socket_pressure(sk->sk_memcg))
1392 		return true;
1393 
1394 	return !!READ_ONCE(*sk->sk_prot->memory_pressure);
1395 }
1396 
1397 static inline long
1398 proto_memory_allocated(const struct proto *prot)
1399 {
1400 	return max(0L, atomic_long_read(prot->memory_allocated));
1401 }
1402 
1403 static inline long
1404 sk_memory_allocated(const struct sock *sk)
1405 {
1406 	return proto_memory_allocated(sk->sk_prot);
1407 }
1408 
1409 /* 1 MB per cpu, in page units */
1410 #define SK_MEMORY_PCPU_RESERVE (1 << (20 - PAGE_SHIFT))
1411 extern int sysctl_mem_pcpu_rsv;
1412 
1413 static inline void proto_memory_pcpu_drain(struct proto *proto)
1414 {
1415 	int val = this_cpu_xchg(*proto->per_cpu_fw_alloc, 0);
1416 
1417 	if (val)
1418 		atomic_long_add(val, proto->memory_allocated);
1419 }
1420 
1421 static inline void
1422 sk_memory_allocated_add(const struct sock *sk, int val)
1423 {
1424 	struct proto *proto = sk->sk_prot;
1425 
1426 	val = this_cpu_add_return(*proto->per_cpu_fw_alloc, val);
1427 
1428 	if (unlikely(val >= READ_ONCE(sysctl_mem_pcpu_rsv)))
1429 		proto_memory_pcpu_drain(proto);
1430 }
1431 
1432 static inline void
1433 sk_memory_allocated_sub(const struct sock *sk, int val)
1434 {
1435 	struct proto *proto = sk->sk_prot;
1436 
1437 	val = this_cpu_sub_return(*proto->per_cpu_fw_alloc, val);
1438 
1439 	if (unlikely(val <= -READ_ONCE(sysctl_mem_pcpu_rsv)))
1440 		proto_memory_pcpu_drain(proto);
1441 }
1442 
1443 #define SK_ALLOC_PERCPU_COUNTER_BATCH 16
1444 
1445 static inline void sk_sockets_allocated_dec(struct sock *sk)
1446 {
1447 	percpu_counter_add_batch(sk->sk_prot->sockets_allocated, -1,
1448 				 SK_ALLOC_PERCPU_COUNTER_BATCH);
1449 }
1450 
1451 static inline void sk_sockets_allocated_inc(struct sock *sk)
1452 {
1453 	percpu_counter_add_batch(sk->sk_prot->sockets_allocated, 1,
1454 				 SK_ALLOC_PERCPU_COUNTER_BATCH);
1455 }
1456 
1457 static inline u64
1458 sk_sockets_allocated_read_positive(struct sock *sk)
1459 {
1460 	return percpu_counter_read_positive(sk->sk_prot->sockets_allocated);
1461 }
1462 
1463 static inline int
1464 proto_sockets_allocated_sum_positive(struct proto *prot)
1465 {
1466 	return percpu_counter_sum_positive(prot->sockets_allocated);
1467 }
1468 
1469 static inline bool
1470 proto_memory_pressure(struct proto *prot)
1471 {
1472 	if (!prot->memory_pressure)
1473 		return false;
1474 	return !!READ_ONCE(*prot->memory_pressure);
1475 }
1476 
1477 
1478 #ifdef CONFIG_PROC_FS
1479 #define PROTO_INUSE_NR	64	/* should be enough for the first time */
1480 struct prot_inuse {
1481 	int all;
1482 	int val[PROTO_INUSE_NR];
1483 };
1484 
1485 static inline void sock_prot_inuse_add(const struct net *net,
1486 				       const struct proto *prot, int val)
1487 {
1488 	this_cpu_add(net->core.prot_inuse->val[prot->inuse_idx], val);
1489 }
1490 
1491 static inline void sock_inuse_add(const struct net *net, int val)
1492 {
1493 	this_cpu_add(net->core.prot_inuse->all, val);
1494 }
1495 
1496 int sock_prot_inuse_get(struct net *net, struct proto *proto);
1497 int sock_inuse_get(struct net *net);
1498 #else
1499 static inline void sock_prot_inuse_add(const struct net *net,
1500 				       const struct proto *prot, int val)
1501 {
1502 }
1503 
1504 static inline void sock_inuse_add(const struct net *net, int val)
1505 {
1506 }
1507 #endif
1508 
1509 
1510 /* With per-bucket locks this operation is not-atomic, so that
1511  * this version is not worse.
1512  */
1513 static inline int __sk_prot_rehash(struct sock *sk)
1514 {
1515 	sk->sk_prot->unhash(sk);
1516 	return sk->sk_prot->hash(sk);
1517 }
1518 
1519 /* About 10 seconds */
1520 #define SOCK_DESTROY_TIME (10*HZ)
1521 
1522 /* Sockets 0-1023 can't be bound to unless you are superuser */
1523 #define PROT_SOCK	1024
1524 
1525 #define SHUTDOWN_MASK	3
1526 #define RCV_SHUTDOWN	1
1527 #define SEND_SHUTDOWN	2
1528 
1529 #define SOCK_BINDADDR_LOCK	4
1530 #define SOCK_BINDPORT_LOCK	8
1531 
1532 struct socket_alloc {
1533 	struct socket socket;
1534 	struct inode vfs_inode;
1535 };
1536 
1537 static inline struct socket *SOCKET_I(struct inode *inode)
1538 {
1539 	return &container_of(inode, struct socket_alloc, vfs_inode)->socket;
1540 }
1541 
1542 static inline struct inode *SOCK_INODE(struct socket *socket)
1543 {
1544 	return &container_of(socket, struct socket_alloc, socket)->vfs_inode;
1545 }
1546 
1547 /*
1548  * Functions for memory accounting
1549  */
1550 int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind);
1551 int __sk_mem_schedule(struct sock *sk, int size, int kind);
1552 void __sk_mem_reduce_allocated(struct sock *sk, int amount);
1553 void __sk_mem_reclaim(struct sock *sk, int amount);
1554 
1555 #define SK_MEM_SEND	0
1556 #define SK_MEM_RECV	1
1557 
1558 /* sysctl_mem values are in pages */
1559 static inline long sk_prot_mem_limits(const struct sock *sk, int index)
1560 {
1561 	return READ_ONCE(sk->sk_prot->sysctl_mem[index]);
1562 }
1563 
1564 static inline int sk_mem_pages(int amt)
1565 {
1566 	return (amt + PAGE_SIZE - 1) >> PAGE_SHIFT;
1567 }
1568 
1569 static inline bool sk_has_account(struct sock *sk)
1570 {
1571 	/* return true if protocol supports memory accounting */
1572 	return !!sk->sk_prot->memory_allocated;
1573 }
1574 
1575 static inline bool sk_wmem_schedule(struct sock *sk, int size)
1576 {
1577 	int delta;
1578 
1579 	if (!sk_has_account(sk))
1580 		return true;
1581 	delta = size - sk->sk_forward_alloc;
1582 	return delta <= 0 || __sk_mem_schedule(sk, delta, SK_MEM_SEND);
1583 }
1584 
1585 static inline bool
1586 sk_rmem_schedule(struct sock *sk, struct sk_buff *skb, int size)
1587 {
1588 	int delta;
1589 
1590 	if (!sk_has_account(sk))
1591 		return true;
1592 	delta = size - sk->sk_forward_alloc;
1593 	return delta <= 0 || __sk_mem_schedule(sk, delta, SK_MEM_RECV) ||
1594 		skb_pfmemalloc(skb);
1595 }
1596 
1597 static inline int sk_unused_reserved_mem(const struct sock *sk)
1598 {
1599 	int unused_mem;
1600 
1601 	if (likely(!sk->sk_reserved_mem))
1602 		return 0;
1603 
1604 	unused_mem = sk->sk_reserved_mem - sk->sk_wmem_queued -
1605 			atomic_read(&sk->sk_rmem_alloc);
1606 
1607 	return unused_mem > 0 ? unused_mem : 0;
1608 }
1609 
1610 static inline void sk_mem_reclaim(struct sock *sk)
1611 {
1612 	int reclaimable;
1613 
1614 	if (!sk_has_account(sk))
1615 		return;
1616 
1617 	reclaimable = sk->sk_forward_alloc - sk_unused_reserved_mem(sk);
1618 
1619 	if (reclaimable >= (int)PAGE_SIZE)
1620 		__sk_mem_reclaim(sk, reclaimable);
1621 }
1622 
1623 static inline void sk_mem_reclaim_final(struct sock *sk)
1624 {
1625 	sk->sk_reserved_mem = 0;
1626 	sk_mem_reclaim(sk);
1627 }
1628 
1629 static inline void sk_mem_charge(struct sock *sk, int size)
1630 {
1631 	if (!sk_has_account(sk))
1632 		return;
1633 	sk_forward_alloc_add(sk, -size);
1634 }
1635 
1636 static inline void sk_mem_uncharge(struct sock *sk, int size)
1637 {
1638 	if (!sk_has_account(sk))
1639 		return;
1640 	sk_forward_alloc_add(sk, size);
1641 	sk_mem_reclaim(sk);
1642 }
1643 
1644 /*
1645  * Macro so as to not evaluate some arguments when
1646  * lockdep is not enabled.
1647  *
1648  * Mark both the sk_lock and the sk_lock.slock as a
1649  * per-address-family lock class.
1650  */
1651 #define sock_lock_init_class_and_name(sk, sname, skey, name, key)	\
1652 do {									\
1653 	sk->sk_lock.owned = 0;						\
1654 	init_waitqueue_head(&sk->sk_lock.wq);				\
1655 	spin_lock_init(&(sk)->sk_lock.slock);				\
1656 	debug_check_no_locks_freed((void *)&(sk)->sk_lock,		\
1657 			sizeof((sk)->sk_lock));				\
1658 	lockdep_set_class_and_name(&(sk)->sk_lock.slock,		\
1659 				(skey), (sname));				\
1660 	lockdep_init_map(&(sk)->sk_lock.dep_map, (name), (key), 0);	\
1661 } while (0)
1662 
1663 static inline bool lockdep_sock_is_held(const struct sock *sk)
1664 {
1665 	return lockdep_is_held(&sk->sk_lock) ||
1666 	       lockdep_is_held(&sk->sk_lock.slock);
1667 }
1668 
1669 void lock_sock_nested(struct sock *sk, int subclass);
1670 
1671 static inline void lock_sock(struct sock *sk)
1672 {
1673 	lock_sock_nested(sk, 0);
1674 }
1675 
1676 void __lock_sock(struct sock *sk);
1677 void __release_sock(struct sock *sk);
1678 void release_sock(struct sock *sk);
1679 
1680 /* BH context may only use the following locking interface. */
1681 #define bh_lock_sock(__sk)	spin_lock(&((__sk)->sk_lock.slock))
1682 #define bh_lock_sock_nested(__sk) \
1683 				spin_lock_nested(&((__sk)->sk_lock.slock), \
1684 				SINGLE_DEPTH_NESTING)
1685 #define bh_unlock_sock(__sk)	spin_unlock(&((__sk)->sk_lock.slock))
1686 
1687 bool __lock_sock_fast(struct sock *sk) __acquires(&sk->sk_lock.slock);
1688 
1689 /**
1690  * lock_sock_fast - fast version of lock_sock
1691  * @sk: socket
1692  *
1693  * This version should be used for very small section, where process wont block
1694  * return false if fast path is taken:
1695  *
1696  *   sk_lock.slock locked, owned = 0, BH disabled
1697  *
1698  * return true if slow path is taken:
1699  *
1700  *   sk_lock.slock unlocked, owned = 1, BH enabled
1701  */
1702 static inline bool lock_sock_fast(struct sock *sk)
1703 {
1704 	/* The sk_lock has mutex_lock() semantics here. */
1705 	mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_);
1706 
1707 	return __lock_sock_fast(sk);
1708 }
1709 
1710 /* fast socket lock variant for caller already holding a [different] socket lock */
1711 static inline bool lock_sock_fast_nested(struct sock *sk)
1712 {
1713 	mutex_acquire(&sk->sk_lock.dep_map, SINGLE_DEPTH_NESTING, 0, _RET_IP_);
1714 
1715 	return __lock_sock_fast(sk);
1716 }
1717 
1718 /**
1719  * unlock_sock_fast - complement of lock_sock_fast
1720  * @sk: socket
1721  * @slow: slow mode
1722  *
1723  * fast unlock socket for user context.
1724  * If slow mode is on, we call regular release_sock()
1725  */
1726 static inline void unlock_sock_fast(struct sock *sk, bool slow)
1727 	__releases(&sk->sk_lock.slock)
1728 {
1729 	if (slow) {
1730 		release_sock(sk);
1731 		__release(&sk->sk_lock.slock);
1732 	} else {
1733 		mutex_release(&sk->sk_lock.dep_map, _RET_IP_);
1734 		spin_unlock_bh(&sk->sk_lock.slock);
1735 	}
1736 }
1737 
1738 void sockopt_lock_sock(struct sock *sk);
1739 void sockopt_release_sock(struct sock *sk);
1740 bool sockopt_ns_capable(struct user_namespace *ns, int cap);
1741 bool sockopt_capable(int cap);
1742 
1743 /* Used by processes to "lock" a socket state, so that
1744  * interrupts and bottom half handlers won't change it
1745  * from under us. It essentially blocks any incoming
1746  * packets, so that we won't get any new data or any
1747  * packets that change the state of the socket.
1748  *
1749  * While locked, BH processing will add new packets to
1750  * the backlog queue.  This queue is processed by the
1751  * owner of the socket lock right before it is released.
1752  *
1753  * Since ~2.3.5 it is also exclusive sleep lock serializing
1754  * accesses from user process context.
1755  */
1756 
1757 static inline void sock_owned_by_me(const struct sock *sk)
1758 {
1759 #ifdef CONFIG_LOCKDEP
1760 	WARN_ON_ONCE(!lockdep_sock_is_held(sk) && debug_locks);
1761 #endif
1762 }
1763 
1764 static inline void sock_not_owned_by_me(const struct sock *sk)
1765 {
1766 #ifdef CONFIG_LOCKDEP
1767 	WARN_ON_ONCE(lockdep_sock_is_held(sk) && debug_locks);
1768 #endif
1769 }
1770 
1771 static inline bool sock_owned_by_user(const struct sock *sk)
1772 {
1773 	sock_owned_by_me(sk);
1774 	return sk->sk_lock.owned;
1775 }
1776 
1777 static inline bool sock_owned_by_user_nocheck(const struct sock *sk)
1778 {
1779 	return sk->sk_lock.owned;
1780 }
1781 
1782 static inline void sock_release_ownership(struct sock *sk)
1783 {
1784 	DEBUG_NET_WARN_ON_ONCE(!sock_owned_by_user_nocheck(sk));
1785 	sk->sk_lock.owned = 0;
1786 
1787 	/* The sk_lock has mutex_unlock() semantics: */
1788 	mutex_release(&sk->sk_lock.dep_map, _RET_IP_);
1789 }
1790 
1791 /* no reclassification while locks are held */
1792 static inline bool sock_allow_reclassification(const struct sock *csk)
1793 {
1794 	struct sock *sk = (struct sock *)csk;
1795 
1796 	return !sock_owned_by_user_nocheck(sk) &&
1797 		!spin_is_locked(&sk->sk_lock.slock);
1798 }
1799 
1800 struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
1801 		      struct proto *prot, int kern);
1802 void sk_free(struct sock *sk);
1803 void sk_destruct(struct sock *sk);
1804 struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority);
1805 void sk_free_unlock_clone(struct sock *sk);
1806 
1807 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
1808 			     gfp_t priority);
1809 void __sock_wfree(struct sk_buff *skb);
1810 void sock_wfree(struct sk_buff *skb);
1811 struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size,
1812 			     gfp_t priority);
1813 void skb_orphan_partial(struct sk_buff *skb);
1814 void sock_rfree(struct sk_buff *skb);
1815 void sock_efree(struct sk_buff *skb);
1816 #ifdef CONFIG_INET
1817 void sock_edemux(struct sk_buff *skb);
1818 void sock_pfree(struct sk_buff *skb);
1819 #else
1820 #define sock_edemux sock_efree
1821 #endif
1822 
1823 int sk_setsockopt(struct sock *sk, int level, int optname,
1824 		  sockptr_t optval, unsigned int optlen);
1825 int sock_setsockopt(struct socket *sock, int level, int op,
1826 		    sockptr_t optval, unsigned int optlen);
1827 int do_sock_setsockopt(struct socket *sock, bool compat, int level,
1828 		       int optname, sockptr_t optval, int optlen);
1829 int do_sock_getsockopt(struct socket *sock, bool compat, int level,
1830 		       int optname, sockptr_t optval, sockptr_t optlen);
1831 
1832 int sk_getsockopt(struct sock *sk, int level, int optname,
1833 		  sockptr_t optval, sockptr_t optlen);
1834 int sock_gettstamp(struct socket *sock, void __user *userstamp,
1835 		   bool timeval, bool time32);
1836 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
1837 				     unsigned long data_len, int noblock,
1838 				     int *errcode, int max_page_order);
1839 
1840 static inline struct sk_buff *sock_alloc_send_skb(struct sock *sk,
1841 						  unsigned long size,
1842 						  int noblock, int *errcode)
1843 {
1844 	return sock_alloc_send_pskb(sk, size, 0, noblock, errcode, 0);
1845 }
1846 
1847 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority);
1848 void sock_kfree_s(struct sock *sk, void *mem, int size);
1849 void sock_kzfree_s(struct sock *sk, void *mem, int size);
1850 void sk_send_sigurg(struct sock *sk);
1851 
1852 static inline void sock_replace_proto(struct sock *sk, struct proto *proto)
1853 {
1854 	if (sk->sk_socket)
1855 		clear_bit(SOCK_SUPPORT_ZC, &sk->sk_socket->flags);
1856 	WRITE_ONCE(sk->sk_prot, proto);
1857 }
1858 
1859 struct sockcm_cookie {
1860 	u64 transmit_time;
1861 	u32 mark;
1862 	u32 tsflags;
1863 };
1864 
1865 static inline void sockcm_init(struct sockcm_cookie *sockc,
1866 			       const struct sock *sk)
1867 {
1868 	*sockc = (struct sockcm_cookie) {
1869 		.tsflags = READ_ONCE(sk->sk_tsflags)
1870 	};
1871 }
1872 
1873 int __sock_cmsg_send(struct sock *sk, struct cmsghdr *cmsg,
1874 		     struct sockcm_cookie *sockc);
1875 int sock_cmsg_send(struct sock *sk, struct msghdr *msg,
1876 		   struct sockcm_cookie *sockc);
1877 
1878 /*
1879  * Functions to fill in entries in struct proto_ops when a protocol
1880  * does not implement a particular function.
1881  */
1882 int sock_no_bind(struct socket *, struct sockaddr *, int);
1883 int sock_no_connect(struct socket *, struct sockaddr *, int, int);
1884 int sock_no_socketpair(struct socket *, struct socket *);
1885 int sock_no_accept(struct socket *, struct socket *, int, bool);
1886 int sock_no_getname(struct socket *, struct sockaddr *, int);
1887 int sock_no_ioctl(struct socket *, unsigned int, unsigned long);
1888 int sock_no_listen(struct socket *, int);
1889 int sock_no_shutdown(struct socket *, int);
1890 int sock_no_sendmsg(struct socket *, struct msghdr *, size_t);
1891 int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t len);
1892 int sock_no_recvmsg(struct socket *, struct msghdr *, size_t, int);
1893 int sock_no_mmap(struct file *file, struct socket *sock,
1894 		 struct vm_area_struct *vma);
1895 
1896 /*
1897  * Functions to fill in entries in struct proto_ops when a protocol
1898  * uses the inet style.
1899  */
1900 int sock_common_getsockopt(struct socket *sock, int level, int optname,
1901 				  char __user *optval, int __user *optlen);
1902 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
1903 			int flags);
1904 int sock_common_setsockopt(struct socket *sock, int level, int optname,
1905 			   sockptr_t optval, unsigned int optlen);
1906 
1907 void sk_common_release(struct sock *sk);
1908 
1909 /*
1910  *	Default socket callbacks and setup code
1911  */
1912 
1913 /* Initialise core socket variables using an explicit uid. */
1914 void sock_init_data_uid(struct socket *sock, struct sock *sk, kuid_t uid);
1915 
1916 /* Initialise core socket variables.
1917  * Assumes struct socket *sock is embedded in a struct socket_alloc.
1918  */
1919 void sock_init_data(struct socket *sock, struct sock *sk);
1920 
1921 /*
1922  * Socket reference counting postulates.
1923  *
1924  * * Each user of socket SHOULD hold a reference count.
1925  * * Each access point to socket (an hash table bucket, reference from a list,
1926  *   running timer, skb in flight MUST hold a reference count.
1927  * * When reference count hits 0, it means it will never increase back.
1928  * * When reference count hits 0, it means that no references from
1929  *   outside exist to this socket and current process on current CPU
1930  *   is last user and may/should destroy this socket.
1931  * * sk_free is called from any context: process, BH, IRQ. When
1932  *   it is called, socket has no references from outside -> sk_free
1933  *   may release descendant resources allocated by the socket, but
1934  *   to the time when it is called, socket is NOT referenced by any
1935  *   hash tables, lists etc.
1936  * * Packets, delivered from outside (from network or from another process)
1937  *   and enqueued on receive/error queues SHOULD NOT grab reference count,
1938  *   when they sit in queue. Otherwise, packets will leak to hole, when
1939  *   socket is looked up by one cpu and unhasing is made by another CPU.
1940  *   It is true for udp/raw, netlink (leak to receive and error queues), tcp
1941  *   (leak to backlog). Packet socket does all the processing inside
1942  *   BR_NETPROTO_LOCK, so that it has not this race condition. UNIX sockets
1943  *   use separate SMP lock, so that they are prone too.
1944  */
1945 
1946 /* Ungrab socket and destroy it, if it was the last reference. */
1947 static inline void sock_put(struct sock *sk)
1948 {
1949 	if (refcount_dec_and_test(&sk->sk_refcnt))
1950 		sk_free(sk);
1951 }
1952 /* Generic version of sock_put(), dealing with all sockets
1953  * (TCP_TIMEWAIT, TCP_NEW_SYN_RECV, ESTABLISHED...)
1954  */
1955 void sock_gen_put(struct sock *sk);
1956 
1957 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, const int nested,
1958 		     unsigned int trim_cap, bool refcounted);
1959 static inline int sk_receive_skb(struct sock *sk, struct sk_buff *skb,
1960 				 const int nested)
1961 {
1962 	return __sk_receive_skb(sk, skb, nested, 1, true);
1963 }
1964 
1965 static inline void sk_tx_queue_set(struct sock *sk, int tx_queue)
1966 {
1967 	/* sk_tx_queue_mapping accept only upto a 16-bit value */
1968 	if (WARN_ON_ONCE((unsigned short)tx_queue >= USHRT_MAX))
1969 		return;
1970 	/* Paired with READ_ONCE() in sk_tx_queue_get() and
1971 	 * other WRITE_ONCE() because socket lock might be not held.
1972 	 */
1973 	WRITE_ONCE(sk->sk_tx_queue_mapping, tx_queue);
1974 }
1975 
1976 #define NO_QUEUE_MAPPING	USHRT_MAX
1977 
1978 static inline void sk_tx_queue_clear(struct sock *sk)
1979 {
1980 	/* Paired with READ_ONCE() in sk_tx_queue_get() and
1981 	 * other WRITE_ONCE() because socket lock might be not held.
1982 	 */
1983 	WRITE_ONCE(sk->sk_tx_queue_mapping, NO_QUEUE_MAPPING);
1984 }
1985 
1986 static inline int sk_tx_queue_get(const struct sock *sk)
1987 {
1988 	if (sk) {
1989 		/* Paired with WRITE_ONCE() in sk_tx_queue_clear()
1990 		 * and sk_tx_queue_set().
1991 		 */
1992 		int val = READ_ONCE(sk->sk_tx_queue_mapping);
1993 
1994 		if (val != NO_QUEUE_MAPPING)
1995 			return val;
1996 	}
1997 	return -1;
1998 }
1999 
2000 static inline void __sk_rx_queue_set(struct sock *sk,
2001 				     const struct sk_buff *skb,
2002 				     bool force_set)
2003 {
2004 #ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
2005 	if (skb_rx_queue_recorded(skb)) {
2006 		u16 rx_queue = skb_get_rx_queue(skb);
2007 
2008 		if (force_set ||
2009 		    unlikely(READ_ONCE(sk->sk_rx_queue_mapping) != rx_queue))
2010 			WRITE_ONCE(sk->sk_rx_queue_mapping, rx_queue);
2011 	}
2012 #endif
2013 }
2014 
2015 static inline void sk_rx_queue_set(struct sock *sk, const struct sk_buff *skb)
2016 {
2017 	__sk_rx_queue_set(sk, skb, true);
2018 }
2019 
2020 static inline void sk_rx_queue_update(struct sock *sk, const struct sk_buff *skb)
2021 {
2022 	__sk_rx_queue_set(sk, skb, false);
2023 }
2024 
2025 static inline void sk_rx_queue_clear(struct sock *sk)
2026 {
2027 #ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
2028 	WRITE_ONCE(sk->sk_rx_queue_mapping, NO_QUEUE_MAPPING);
2029 #endif
2030 }
2031 
2032 static inline int sk_rx_queue_get(const struct sock *sk)
2033 {
2034 #ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
2035 	if (sk) {
2036 		int res = READ_ONCE(sk->sk_rx_queue_mapping);
2037 
2038 		if (res != NO_QUEUE_MAPPING)
2039 			return res;
2040 	}
2041 #endif
2042 
2043 	return -1;
2044 }
2045 
2046 static inline void sk_set_socket(struct sock *sk, struct socket *sock)
2047 {
2048 	sk->sk_socket = sock;
2049 }
2050 
2051 static inline wait_queue_head_t *sk_sleep(struct sock *sk)
2052 {
2053 	BUILD_BUG_ON(offsetof(struct socket_wq, wait) != 0);
2054 	return &rcu_dereference_raw(sk->sk_wq)->wait;
2055 }
2056 /* Detach socket from process context.
2057  * Announce socket dead, detach it from wait queue and inode.
2058  * Note that parent inode held reference count on this struct sock,
2059  * we do not release it in this function, because protocol
2060  * probably wants some additional cleanups or even continuing
2061  * to work with this socket (TCP).
2062  */
2063 static inline void sock_orphan(struct sock *sk)
2064 {
2065 	write_lock_bh(&sk->sk_callback_lock);
2066 	sock_set_flag(sk, SOCK_DEAD);
2067 	sk_set_socket(sk, NULL);
2068 	sk->sk_wq  = NULL;
2069 	write_unlock_bh(&sk->sk_callback_lock);
2070 }
2071 
2072 static inline void sock_graft(struct sock *sk, struct socket *parent)
2073 {
2074 	WARN_ON(parent->sk);
2075 	write_lock_bh(&sk->sk_callback_lock);
2076 	rcu_assign_pointer(sk->sk_wq, &parent->wq);
2077 	parent->sk = sk;
2078 	sk_set_socket(sk, parent);
2079 	sk->sk_uid = SOCK_INODE(parent)->i_uid;
2080 	security_sock_graft(sk, parent);
2081 	write_unlock_bh(&sk->sk_callback_lock);
2082 }
2083 
2084 kuid_t sock_i_uid(struct sock *sk);
2085 unsigned long __sock_i_ino(struct sock *sk);
2086 unsigned long sock_i_ino(struct sock *sk);
2087 
2088 static inline kuid_t sock_net_uid(const struct net *net, const struct sock *sk)
2089 {
2090 	return sk ? sk->sk_uid : make_kuid(net->user_ns, 0);
2091 }
2092 
2093 static inline u32 net_tx_rndhash(void)
2094 {
2095 	u32 v = get_random_u32();
2096 
2097 	return v ?: 1;
2098 }
2099 
2100 static inline void sk_set_txhash(struct sock *sk)
2101 {
2102 	/* This pairs with READ_ONCE() in skb_set_hash_from_sk() */
2103 	WRITE_ONCE(sk->sk_txhash, net_tx_rndhash());
2104 }
2105 
2106 static inline bool sk_rethink_txhash(struct sock *sk)
2107 {
2108 	if (sk->sk_txhash && sk->sk_txrehash == SOCK_TXREHASH_ENABLED) {
2109 		sk_set_txhash(sk);
2110 		return true;
2111 	}
2112 	return false;
2113 }
2114 
2115 static inline struct dst_entry *
2116 __sk_dst_get(const struct sock *sk)
2117 {
2118 	return rcu_dereference_check(sk->sk_dst_cache,
2119 				     lockdep_sock_is_held(sk));
2120 }
2121 
2122 static inline struct dst_entry *
2123 sk_dst_get(const struct sock *sk)
2124 {
2125 	struct dst_entry *dst;
2126 
2127 	rcu_read_lock();
2128 	dst = rcu_dereference(sk->sk_dst_cache);
2129 	if (dst && !rcuref_get(&dst->__rcuref))
2130 		dst = NULL;
2131 	rcu_read_unlock();
2132 	return dst;
2133 }
2134 
2135 static inline void __dst_negative_advice(struct sock *sk)
2136 {
2137 	struct dst_entry *ndst, *dst = __sk_dst_get(sk);
2138 
2139 	if (dst && dst->ops->negative_advice) {
2140 		ndst = dst->ops->negative_advice(dst);
2141 
2142 		if (ndst != dst) {
2143 			rcu_assign_pointer(sk->sk_dst_cache, ndst);
2144 			sk_tx_queue_clear(sk);
2145 			WRITE_ONCE(sk->sk_dst_pending_confirm, 0);
2146 		}
2147 	}
2148 }
2149 
2150 static inline void dst_negative_advice(struct sock *sk)
2151 {
2152 	sk_rethink_txhash(sk);
2153 	__dst_negative_advice(sk);
2154 }
2155 
2156 static inline void
2157 __sk_dst_set(struct sock *sk, struct dst_entry *dst)
2158 {
2159 	struct dst_entry *old_dst;
2160 
2161 	sk_tx_queue_clear(sk);
2162 	WRITE_ONCE(sk->sk_dst_pending_confirm, 0);
2163 	old_dst = rcu_dereference_protected(sk->sk_dst_cache,
2164 					    lockdep_sock_is_held(sk));
2165 	rcu_assign_pointer(sk->sk_dst_cache, dst);
2166 	dst_release(old_dst);
2167 }
2168 
2169 static inline void
2170 sk_dst_set(struct sock *sk, struct dst_entry *dst)
2171 {
2172 	struct dst_entry *old_dst;
2173 
2174 	sk_tx_queue_clear(sk);
2175 	WRITE_ONCE(sk->sk_dst_pending_confirm, 0);
2176 	old_dst = xchg((__force struct dst_entry **)&sk->sk_dst_cache, dst);
2177 	dst_release(old_dst);
2178 }
2179 
2180 static inline void
2181 __sk_dst_reset(struct sock *sk)
2182 {
2183 	__sk_dst_set(sk, NULL);
2184 }
2185 
2186 static inline void
2187 sk_dst_reset(struct sock *sk)
2188 {
2189 	sk_dst_set(sk, NULL);
2190 }
2191 
2192 struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie);
2193 
2194 struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie);
2195 
2196 static inline void sk_dst_confirm(struct sock *sk)
2197 {
2198 	if (!READ_ONCE(sk->sk_dst_pending_confirm))
2199 		WRITE_ONCE(sk->sk_dst_pending_confirm, 1);
2200 }
2201 
2202 static inline void sock_confirm_neigh(struct sk_buff *skb, struct neighbour *n)
2203 {
2204 	if (skb_get_dst_pending_confirm(skb)) {
2205 		struct sock *sk = skb->sk;
2206 
2207 		if (sk && READ_ONCE(sk->sk_dst_pending_confirm))
2208 			WRITE_ONCE(sk->sk_dst_pending_confirm, 0);
2209 		neigh_confirm(n);
2210 	}
2211 }
2212 
2213 bool sk_mc_loop(const struct sock *sk);
2214 
2215 static inline bool sk_can_gso(const struct sock *sk)
2216 {
2217 	return net_gso_ok(sk->sk_route_caps, sk->sk_gso_type);
2218 }
2219 
2220 void sk_setup_caps(struct sock *sk, struct dst_entry *dst);
2221 
2222 static inline void sk_gso_disable(struct sock *sk)
2223 {
2224 	sk->sk_gso_disabled = 1;
2225 	sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
2226 }
2227 
2228 static inline int skb_do_copy_data_nocache(struct sock *sk, struct sk_buff *skb,
2229 					   struct iov_iter *from, char *to,
2230 					   int copy, int offset)
2231 {
2232 	if (skb->ip_summed == CHECKSUM_NONE) {
2233 		__wsum csum = 0;
2234 		if (!csum_and_copy_from_iter_full(to, copy, &csum, from))
2235 			return -EFAULT;
2236 		skb->csum = csum_block_add(skb->csum, csum, offset);
2237 	} else if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) {
2238 		if (!copy_from_iter_full_nocache(to, copy, from))
2239 			return -EFAULT;
2240 	} else if (!copy_from_iter_full(to, copy, from))
2241 		return -EFAULT;
2242 
2243 	return 0;
2244 }
2245 
2246 static inline int skb_add_data_nocache(struct sock *sk, struct sk_buff *skb,
2247 				       struct iov_iter *from, int copy)
2248 {
2249 	int err, offset = skb->len;
2250 
2251 	err = skb_do_copy_data_nocache(sk, skb, from, skb_put(skb, copy),
2252 				       copy, offset);
2253 	if (err)
2254 		__skb_trim(skb, offset);
2255 
2256 	return err;
2257 }
2258 
2259 static inline int skb_copy_to_page_nocache(struct sock *sk, struct iov_iter *from,
2260 					   struct sk_buff *skb,
2261 					   struct page *page,
2262 					   int off, int copy)
2263 {
2264 	int err;
2265 
2266 	err = skb_do_copy_data_nocache(sk, skb, from, page_address(page) + off,
2267 				       copy, skb->len);
2268 	if (err)
2269 		return err;
2270 
2271 	skb_len_add(skb, copy);
2272 	sk_wmem_queued_add(sk, copy);
2273 	sk_mem_charge(sk, copy);
2274 	return 0;
2275 }
2276 
2277 /**
2278  * sk_wmem_alloc_get - returns write allocations
2279  * @sk: socket
2280  *
2281  * Return: sk_wmem_alloc minus initial offset of one
2282  */
2283 static inline int sk_wmem_alloc_get(const struct sock *sk)
2284 {
2285 	return refcount_read(&sk->sk_wmem_alloc) - 1;
2286 }
2287 
2288 /**
2289  * sk_rmem_alloc_get - returns read allocations
2290  * @sk: socket
2291  *
2292  * Return: sk_rmem_alloc
2293  */
2294 static inline int sk_rmem_alloc_get(const struct sock *sk)
2295 {
2296 	return atomic_read(&sk->sk_rmem_alloc);
2297 }
2298 
2299 /**
2300  * sk_has_allocations - check if allocations are outstanding
2301  * @sk: socket
2302  *
2303  * Return: true if socket has write or read allocations
2304  */
2305 static inline bool sk_has_allocations(const struct sock *sk)
2306 {
2307 	return sk_wmem_alloc_get(sk) || sk_rmem_alloc_get(sk);
2308 }
2309 
2310 /**
2311  * skwq_has_sleeper - check if there are any waiting processes
2312  * @wq: struct socket_wq
2313  *
2314  * Return: true if socket_wq has waiting processes
2315  *
2316  * The purpose of the skwq_has_sleeper and sock_poll_wait is to wrap the memory
2317  * barrier call. They were added due to the race found within the tcp code.
2318  *
2319  * Consider following tcp code paths::
2320  *
2321  *   CPU1                CPU2
2322  *   sys_select          receive packet
2323  *   ...                 ...
2324  *   __add_wait_queue    update tp->rcv_nxt
2325  *   ...                 ...
2326  *   tp->rcv_nxt check   sock_def_readable
2327  *   ...                 {
2328  *   schedule               rcu_read_lock();
2329  *                          wq = rcu_dereference(sk->sk_wq);
2330  *                          if (wq && waitqueue_active(&wq->wait))
2331  *                              wake_up_interruptible(&wq->wait)
2332  *                          ...
2333  *                       }
2334  *
2335  * The race for tcp fires when the __add_wait_queue changes done by CPU1 stay
2336  * in its cache, and so does the tp->rcv_nxt update on CPU2 side.  The CPU1
2337  * could then endup calling schedule and sleep forever if there are no more
2338  * data on the socket.
2339  *
2340  */
2341 static inline bool skwq_has_sleeper(struct socket_wq *wq)
2342 {
2343 	return wq && wq_has_sleeper(&wq->wait);
2344 }
2345 
2346 /**
2347  * sock_poll_wait - place memory barrier behind the poll_wait call.
2348  * @filp:           file
2349  * @sock:           socket to wait on
2350  * @p:              poll_table
2351  *
2352  * See the comments in the wq_has_sleeper function.
2353  */
2354 static inline void sock_poll_wait(struct file *filp, struct socket *sock,
2355 				  poll_table *p)
2356 {
2357 	if (!poll_does_not_wait(p)) {
2358 		poll_wait(filp, &sock->wq.wait, p);
2359 		/* We need to be sure we are in sync with the
2360 		 * socket flags modification.
2361 		 *
2362 		 * This memory barrier is paired in the wq_has_sleeper.
2363 		 */
2364 		smp_mb();
2365 	}
2366 }
2367 
2368 static inline void skb_set_hash_from_sk(struct sk_buff *skb, struct sock *sk)
2369 {
2370 	/* This pairs with WRITE_ONCE() in sk_set_txhash() */
2371 	u32 txhash = READ_ONCE(sk->sk_txhash);
2372 
2373 	if (txhash) {
2374 		skb->l4_hash = 1;
2375 		skb->hash = txhash;
2376 	}
2377 }
2378 
2379 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk);
2380 
2381 /*
2382  *	Queue a received datagram if it will fit. Stream and sequenced
2383  *	protocols can't normally use this as they need to fit buffers in
2384  *	and play with them.
2385  *
2386  *	Inlined as it's very short and called for pretty much every
2387  *	packet ever received.
2388  */
2389 static inline void skb_set_owner_r(struct sk_buff *skb, struct sock *sk)
2390 {
2391 	skb_orphan(skb);
2392 	skb->sk = sk;
2393 	skb->destructor = sock_rfree;
2394 	atomic_add(skb->truesize, &sk->sk_rmem_alloc);
2395 	sk_mem_charge(sk, skb->truesize);
2396 }
2397 
2398 static inline __must_check bool skb_set_owner_sk_safe(struct sk_buff *skb, struct sock *sk)
2399 {
2400 	if (sk && refcount_inc_not_zero(&sk->sk_refcnt)) {
2401 		skb_orphan(skb);
2402 		skb->destructor = sock_efree;
2403 		skb->sk = sk;
2404 		return true;
2405 	}
2406 	return false;
2407 }
2408 
2409 static inline struct sk_buff *skb_clone_and_charge_r(struct sk_buff *skb, struct sock *sk)
2410 {
2411 	skb = skb_clone(skb, sk_gfp_mask(sk, GFP_ATOMIC));
2412 	if (skb) {
2413 		if (sk_rmem_schedule(sk, skb, skb->truesize)) {
2414 			skb_set_owner_r(skb, sk);
2415 			return skb;
2416 		}
2417 		__kfree_skb(skb);
2418 	}
2419 	return NULL;
2420 }
2421 
2422 static inline void skb_prepare_for_gro(struct sk_buff *skb)
2423 {
2424 	if (skb->destructor != sock_wfree) {
2425 		skb_orphan(skb);
2426 		return;
2427 	}
2428 	skb->slow_gro = 1;
2429 }
2430 
2431 void sk_reset_timer(struct sock *sk, struct timer_list *timer,
2432 		    unsigned long expires);
2433 
2434 void sk_stop_timer(struct sock *sk, struct timer_list *timer);
2435 
2436 void sk_stop_timer_sync(struct sock *sk, struct timer_list *timer);
2437 
2438 int __sk_queue_drop_skb(struct sock *sk, struct sk_buff_head *sk_queue,
2439 			struct sk_buff *skb, unsigned int flags,
2440 			void (*destructor)(struct sock *sk,
2441 					   struct sk_buff *skb));
2442 int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
2443 
2444 int sock_queue_rcv_skb_reason(struct sock *sk, struct sk_buff *skb,
2445 			      enum skb_drop_reason *reason);
2446 
2447 static inline int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
2448 {
2449 	return sock_queue_rcv_skb_reason(sk, skb, NULL);
2450 }
2451 
2452 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb);
2453 struct sk_buff *sock_dequeue_err_skb(struct sock *sk);
2454 
2455 /*
2456  *	Recover an error report and clear atomically
2457  */
2458 
2459 static inline int sock_error(struct sock *sk)
2460 {
2461 	int err;
2462 
2463 	/* Avoid an atomic operation for the common case.
2464 	 * This is racy since another cpu/thread can change sk_err under us.
2465 	 */
2466 	if (likely(data_race(!sk->sk_err)))
2467 		return 0;
2468 
2469 	err = xchg(&sk->sk_err, 0);
2470 	return -err;
2471 }
2472 
2473 void sk_error_report(struct sock *sk);
2474 
2475 static inline unsigned long sock_wspace(struct sock *sk)
2476 {
2477 	int amt = 0;
2478 
2479 	if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
2480 		amt = sk->sk_sndbuf - refcount_read(&sk->sk_wmem_alloc);
2481 		if (amt < 0)
2482 			amt = 0;
2483 	}
2484 	return amt;
2485 }
2486 
2487 /* Note:
2488  *  We use sk->sk_wq_raw, from contexts knowing this
2489  *  pointer is not NULL and cannot disappear/change.
2490  */
2491 static inline void sk_set_bit(int nr, struct sock *sk)
2492 {
2493 	if ((nr == SOCKWQ_ASYNC_NOSPACE || nr == SOCKWQ_ASYNC_WAITDATA) &&
2494 	    !sock_flag(sk, SOCK_FASYNC))
2495 		return;
2496 
2497 	set_bit(nr, &sk->sk_wq_raw->flags);
2498 }
2499 
2500 static inline void sk_clear_bit(int nr, struct sock *sk)
2501 {
2502 	if ((nr == SOCKWQ_ASYNC_NOSPACE || nr == SOCKWQ_ASYNC_WAITDATA) &&
2503 	    !sock_flag(sk, SOCK_FASYNC))
2504 		return;
2505 
2506 	clear_bit(nr, &sk->sk_wq_raw->flags);
2507 }
2508 
2509 static inline void sk_wake_async(const struct sock *sk, int how, int band)
2510 {
2511 	if (sock_flag(sk, SOCK_FASYNC)) {
2512 		rcu_read_lock();
2513 		sock_wake_async(rcu_dereference(sk->sk_wq), how, band);
2514 		rcu_read_unlock();
2515 	}
2516 }
2517 
2518 /* Since sk_{r,w}mem_alloc sums skb->truesize, even a small frame might
2519  * need sizeof(sk_buff) + MTU + padding, unless net driver perform copybreak.
2520  * Note: for send buffers, TCP works better if we can build two skbs at
2521  * minimum.
2522  */
2523 #define TCP_SKB_MIN_TRUESIZE	(2048 + SKB_DATA_ALIGN(sizeof(struct sk_buff)))
2524 
2525 #define SOCK_MIN_SNDBUF		(TCP_SKB_MIN_TRUESIZE * 2)
2526 #define SOCK_MIN_RCVBUF		 TCP_SKB_MIN_TRUESIZE
2527 
2528 static inline void sk_stream_moderate_sndbuf(struct sock *sk)
2529 {
2530 	u32 val;
2531 
2532 	if (sk->sk_userlocks & SOCK_SNDBUF_LOCK)
2533 		return;
2534 
2535 	val = min(sk->sk_sndbuf, sk->sk_wmem_queued >> 1);
2536 	val = max_t(u32, val, sk_unused_reserved_mem(sk));
2537 
2538 	WRITE_ONCE(sk->sk_sndbuf, max_t(u32, val, SOCK_MIN_SNDBUF));
2539 }
2540 
2541 /**
2542  * sk_page_frag - return an appropriate page_frag
2543  * @sk: socket
2544  *
2545  * Use the per task page_frag instead of the per socket one for
2546  * optimization when we know that we're in process context and own
2547  * everything that's associated with %current.
2548  *
2549  * Both direct reclaim and page faults can nest inside other
2550  * socket operations and end up recursing into sk_page_frag()
2551  * while it's already in use: explicitly avoid task page_frag
2552  * when users disable sk_use_task_frag.
2553  *
2554  * Return: a per task page_frag if context allows that,
2555  * otherwise a per socket one.
2556  */
2557 static inline struct page_frag *sk_page_frag(struct sock *sk)
2558 {
2559 	if (sk->sk_use_task_frag)
2560 		return &current->task_frag;
2561 
2562 	return &sk->sk_frag;
2563 }
2564 
2565 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag);
2566 
2567 /*
2568  *	Default write policy as shown to user space via poll/select/SIGIO
2569  */
2570 static inline bool sock_writeable(const struct sock *sk)
2571 {
2572 	return refcount_read(&sk->sk_wmem_alloc) < (READ_ONCE(sk->sk_sndbuf) >> 1);
2573 }
2574 
2575 static inline gfp_t gfp_any(void)
2576 {
2577 	return in_softirq() ? GFP_ATOMIC : GFP_KERNEL;
2578 }
2579 
2580 static inline gfp_t gfp_memcg_charge(void)
2581 {
2582 	return in_softirq() ? GFP_ATOMIC : GFP_KERNEL;
2583 }
2584 
2585 static inline long sock_rcvtimeo(const struct sock *sk, bool noblock)
2586 {
2587 	return noblock ? 0 : sk->sk_rcvtimeo;
2588 }
2589 
2590 static inline long sock_sndtimeo(const struct sock *sk, bool noblock)
2591 {
2592 	return noblock ? 0 : sk->sk_sndtimeo;
2593 }
2594 
2595 static inline int sock_rcvlowat(const struct sock *sk, int waitall, int len)
2596 {
2597 	int v = waitall ? len : min_t(int, READ_ONCE(sk->sk_rcvlowat), len);
2598 
2599 	return v ?: 1;
2600 }
2601 
2602 /* Alas, with timeout socket operations are not restartable.
2603  * Compare this to poll().
2604  */
2605 static inline int sock_intr_errno(long timeo)
2606 {
2607 	return timeo == MAX_SCHEDULE_TIMEOUT ? -ERESTARTSYS : -EINTR;
2608 }
2609 
2610 struct sock_skb_cb {
2611 	u32 dropcount;
2612 };
2613 
2614 /* Store sock_skb_cb at the end of skb->cb[] so protocol families
2615  * using skb->cb[] would keep using it directly and utilize its
2616  * alignement guarantee.
2617  */
2618 #define SOCK_SKB_CB_OFFSET ((sizeof_field(struct sk_buff, cb) - \
2619 			    sizeof(struct sock_skb_cb)))
2620 
2621 #define SOCK_SKB_CB(__skb) ((struct sock_skb_cb *)((__skb)->cb + \
2622 			    SOCK_SKB_CB_OFFSET))
2623 
2624 #define sock_skb_cb_check_size(size) \
2625 	BUILD_BUG_ON((size) > SOCK_SKB_CB_OFFSET)
2626 
2627 static inline void
2628 sock_skb_set_dropcount(const struct sock *sk, struct sk_buff *skb)
2629 {
2630 	SOCK_SKB_CB(skb)->dropcount = sock_flag(sk, SOCK_RXQ_OVFL) ?
2631 						atomic_read(&sk->sk_drops) : 0;
2632 }
2633 
2634 static inline void sk_drops_add(struct sock *sk, const struct sk_buff *skb)
2635 {
2636 	int segs = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
2637 
2638 	atomic_add(segs, &sk->sk_drops);
2639 }
2640 
2641 static inline ktime_t sock_read_timestamp(struct sock *sk)
2642 {
2643 #if BITS_PER_LONG==32
2644 	unsigned int seq;
2645 	ktime_t kt;
2646 
2647 	do {
2648 		seq = read_seqbegin(&sk->sk_stamp_seq);
2649 		kt = sk->sk_stamp;
2650 	} while (read_seqretry(&sk->sk_stamp_seq, seq));
2651 
2652 	return kt;
2653 #else
2654 	return READ_ONCE(sk->sk_stamp);
2655 #endif
2656 }
2657 
2658 static inline void sock_write_timestamp(struct sock *sk, ktime_t kt)
2659 {
2660 #if BITS_PER_LONG==32
2661 	write_seqlock(&sk->sk_stamp_seq);
2662 	sk->sk_stamp = kt;
2663 	write_sequnlock(&sk->sk_stamp_seq);
2664 #else
2665 	WRITE_ONCE(sk->sk_stamp, kt);
2666 #endif
2667 }
2668 
2669 void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk,
2670 			   struct sk_buff *skb);
2671 void __sock_recv_wifi_status(struct msghdr *msg, struct sock *sk,
2672 			     struct sk_buff *skb);
2673 
2674 static inline void
2675 sock_recv_timestamp(struct msghdr *msg, struct sock *sk, struct sk_buff *skb)
2676 {
2677 	struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);
2678 	u32 tsflags = READ_ONCE(sk->sk_tsflags);
2679 	ktime_t kt = skb->tstamp;
2680 	/*
2681 	 * generate control messages if
2682 	 * - receive time stamping in software requested
2683 	 * - software time stamp available and wanted
2684 	 * - hardware time stamps available and wanted
2685 	 */
2686 	if (sock_flag(sk, SOCK_RCVTSTAMP) ||
2687 	    (tsflags & SOF_TIMESTAMPING_RX_SOFTWARE) ||
2688 	    (kt && tsflags & SOF_TIMESTAMPING_SOFTWARE) ||
2689 	    (hwtstamps->hwtstamp &&
2690 	     (tsflags & SOF_TIMESTAMPING_RAW_HARDWARE)))
2691 		__sock_recv_timestamp(msg, sk, skb);
2692 	else
2693 		sock_write_timestamp(sk, kt);
2694 
2695 	if (sock_flag(sk, SOCK_WIFI_STATUS) && skb_wifi_acked_valid(skb))
2696 		__sock_recv_wifi_status(msg, sk, skb);
2697 }
2698 
2699 void __sock_recv_cmsgs(struct msghdr *msg, struct sock *sk,
2700 		       struct sk_buff *skb);
2701 
2702 #define SK_DEFAULT_STAMP (-1L * NSEC_PER_SEC)
2703 static inline void sock_recv_cmsgs(struct msghdr *msg, struct sock *sk,
2704 				   struct sk_buff *skb)
2705 {
2706 #define FLAGS_RECV_CMSGS ((1UL << SOCK_RXQ_OVFL)			| \
2707 			   (1UL << SOCK_RCVTSTAMP)			| \
2708 			   (1UL << SOCK_RCVMARK))
2709 #define TSFLAGS_ANY	  (SOF_TIMESTAMPING_SOFTWARE			| \
2710 			   SOF_TIMESTAMPING_RAW_HARDWARE)
2711 
2712 	if (sk->sk_flags & FLAGS_RECV_CMSGS ||
2713 	    READ_ONCE(sk->sk_tsflags) & TSFLAGS_ANY)
2714 		__sock_recv_cmsgs(msg, sk, skb);
2715 	else if (unlikely(sock_flag(sk, SOCK_TIMESTAMP)))
2716 		sock_write_timestamp(sk, skb->tstamp);
2717 	else if (unlikely(sock_read_timestamp(sk) == SK_DEFAULT_STAMP))
2718 		sock_write_timestamp(sk, 0);
2719 }
2720 
2721 void __sock_tx_timestamp(__u16 tsflags, __u8 *tx_flags);
2722 
2723 /**
2724  * _sock_tx_timestamp - checks whether the outgoing packet is to be time stamped
2725  * @sk:		socket sending this packet
2726  * @tsflags:	timestamping flags to use
2727  * @tx_flags:	completed with instructions for time stamping
2728  * @tskey:      filled in with next sk_tskey (not for TCP, which uses seqno)
2729  *
2730  * Note: callers should take care of initial ``*tx_flags`` value (usually 0)
2731  */
2732 static inline void _sock_tx_timestamp(struct sock *sk, __u16 tsflags,
2733 				      __u8 *tx_flags, __u32 *tskey)
2734 {
2735 	if (unlikely(tsflags)) {
2736 		__sock_tx_timestamp(tsflags, tx_flags);
2737 		if (tsflags & SOF_TIMESTAMPING_OPT_ID && tskey &&
2738 		    tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK)
2739 			*tskey = atomic_inc_return(&sk->sk_tskey) - 1;
2740 	}
2741 	if (unlikely(sock_flag(sk, SOCK_WIFI_STATUS)))
2742 		*tx_flags |= SKBTX_WIFI_STATUS;
2743 }
2744 
2745 static inline void sock_tx_timestamp(struct sock *sk, __u16 tsflags,
2746 				     __u8 *tx_flags)
2747 {
2748 	_sock_tx_timestamp(sk, tsflags, tx_flags, NULL);
2749 }
2750 
2751 static inline void skb_setup_tx_timestamp(struct sk_buff *skb, __u16 tsflags)
2752 {
2753 	_sock_tx_timestamp(skb->sk, tsflags, &skb_shinfo(skb)->tx_flags,
2754 			   &skb_shinfo(skb)->tskey);
2755 }
2756 
2757 static inline bool sk_is_inet(const struct sock *sk)
2758 {
2759 	int family = READ_ONCE(sk->sk_family);
2760 
2761 	return family == AF_INET || family == AF_INET6;
2762 }
2763 
2764 static inline bool sk_is_tcp(const struct sock *sk)
2765 {
2766 	return sk_is_inet(sk) &&
2767 	       sk->sk_type == SOCK_STREAM &&
2768 	       sk->sk_protocol == IPPROTO_TCP;
2769 }
2770 
2771 static inline bool sk_is_udp(const struct sock *sk)
2772 {
2773 	return sk_is_inet(sk) &&
2774 	       sk->sk_type == SOCK_DGRAM &&
2775 	       sk->sk_protocol == IPPROTO_UDP;
2776 }
2777 
2778 static inline bool sk_is_stream_unix(const struct sock *sk)
2779 {
2780 	return sk->sk_family == AF_UNIX && sk->sk_type == SOCK_STREAM;
2781 }
2782 
2783 /**
2784  * sk_eat_skb - Release a skb if it is no longer needed
2785  * @sk: socket to eat this skb from
2786  * @skb: socket buffer to eat
2787  *
2788  * This routine must be called with interrupts disabled or with the socket
2789  * locked so that the sk_buff queue operation is ok.
2790 */
2791 static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb)
2792 {
2793 	__skb_unlink(skb, &sk->sk_receive_queue);
2794 	__kfree_skb(skb);
2795 }
2796 
2797 static inline bool
2798 skb_sk_is_prefetched(struct sk_buff *skb)
2799 {
2800 #ifdef CONFIG_INET
2801 	return skb->destructor == sock_pfree;
2802 #else
2803 	return false;
2804 #endif /* CONFIG_INET */
2805 }
2806 
2807 /* This helper checks if a socket is a full socket,
2808  * ie _not_ a timewait or request socket.
2809  */
2810 static inline bool sk_fullsock(const struct sock *sk)
2811 {
2812 	return (1 << sk->sk_state) & ~(TCPF_TIME_WAIT | TCPF_NEW_SYN_RECV);
2813 }
2814 
2815 static inline bool
2816 sk_is_refcounted(struct sock *sk)
2817 {
2818 	/* Only full sockets have sk->sk_flags. */
2819 	return !sk_fullsock(sk) || !sock_flag(sk, SOCK_RCU_FREE);
2820 }
2821 
2822 /* Checks if this SKB belongs to an HW offloaded socket
2823  * and whether any SW fallbacks are required based on dev.
2824  * Check decrypted mark in case skb_orphan() cleared socket.
2825  */
2826 static inline struct sk_buff *sk_validate_xmit_skb(struct sk_buff *skb,
2827 						   struct net_device *dev)
2828 {
2829 #ifdef CONFIG_SOCK_VALIDATE_XMIT
2830 	struct sock *sk = skb->sk;
2831 
2832 	if (sk && sk_fullsock(sk) && sk->sk_validate_xmit_skb) {
2833 		skb = sk->sk_validate_xmit_skb(sk, dev, skb);
2834 #ifdef CONFIG_TLS_DEVICE
2835 	} else if (unlikely(skb->decrypted)) {
2836 		pr_warn_ratelimited("unencrypted skb with no associated socket - dropping\n");
2837 		kfree_skb(skb);
2838 		skb = NULL;
2839 #endif
2840 	}
2841 #endif
2842 
2843 	return skb;
2844 }
2845 
2846 /* This helper checks if a socket is a LISTEN or NEW_SYN_RECV
2847  * SYNACK messages can be attached to either ones (depending on SYNCOOKIE)
2848  */
2849 static inline bool sk_listener(const struct sock *sk)
2850 {
2851 	return (1 << sk->sk_state) & (TCPF_LISTEN | TCPF_NEW_SYN_RECV);
2852 }
2853 
2854 void sock_enable_timestamp(struct sock *sk, enum sock_flags flag);
2855 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, int level,
2856 		       int type);
2857 
2858 bool sk_ns_capable(const struct sock *sk,
2859 		   struct user_namespace *user_ns, int cap);
2860 bool sk_capable(const struct sock *sk, int cap);
2861 bool sk_net_capable(const struct sock *sk, int cap);
2862 
2863 void sk_get_meminfo(const struct sock *sk, u32 *meminfo);
2864 
2865 /* Take into consideration the size of the struct sk_buff overhead in the
2866  * determination of these values, since that is non-constant across
2867  * platforms.  This makes socket queueing behavior and performance
2868  * not depend upon such differences.
2869  */
2870 #define _SK_MEM_PACKETS		256
2871 #define _SK_MEM_OVERHEAD	SKB_TRUESIZE(256)
2872 #define SK_WMEM_MAX		(_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
2873 #define SK_RMEM_MAX		(_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
2874 
2875 extern __u32 sysctl_wmem_max;
2876 extern __u32 sysctl_rmem_max;
2877 
2878 extern int sysctl_tstamp_allow_data;
2879 
2880 extern __u32 sysctl_wmem_default;
2881 extern __u32 sysctl_rmem_default;
2882 
2883 #define SKB_FRAG_PAGE_ORDER	get_order(32768)
2884 DECLARE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key);
2885 
2886 static inline int sk_get_wmem0(const struct sock *sk, const struct proto *proto)
2887 {
2888 	/* Does this proto have per netns sysctl_wmem ? */
2889 	if (proto->sysctl_wmem_offset)
2890 		return READ_ONCE(*(int *)((void *)sock_net(sk) + proto->sysctl_wmem_offset));
2891 
2892 	return READ_ONCE(*proto->sysctl_wmem);
2893 }
2894 
2895 static inline int sk_get_rmem0(const struct sock *sk, const struct proto *proto)
2896 {
2897 	/* Does this proto have per netns sysctl_rmem ? */
2898 	if (proto->sysctl_rmem_offset)
2899 		return READ_ONCE(*(int *)((void *)sock_net(sk) + proto->sysctl_rmem_offset));
2900 
2901 	return READ_ONCE(*proto->sysctl_rmem);
2902 }
2903 
2904 /* Default TCP Small queue budget is ~1 ms of data (1sec >> 10)
2905  * Some wifi drivers need to tweak it to get more chunks.
2906  * They can use this helper from their ndo_start_xmit()
2907  */
2908 static inline void sk_pacing_shift_update(struct sock *sk, int val)
2909 {
2910 	if (!sk || !sk_fullsock(sk) || READ_ONCE(sk->sk_pacing_shift) == val)
2911 		return;
2912 	WRITE_ONCE(sk->sk_pacing_shift, val);
2913 }
2914 
2915 /* if a socket is bound to a device, check that the given device
2916  * index is either the same or that the socket is bound to an L3
2917  * master device and the given device index is also enslaved to
2918  * that L3 master
2919  */
2920 static inline bool sk_dev_equal_l3scope(struct sock *sk, int dif)
2921 {
2922 	int bound_dev_if = READ_ONCE(sk->sk_bound_dev_if);
2923 	int mdif;
2924 
2925 	if (!bound_dev_if || bound_dev_if == dif)
2926 		return true;
2927 
2928 	mdif = l3mdev_master_ifindex_by_index(sock_net(sk), dif);
2929 	if (mdif && mdif == bound_dev_if)
2930 		return true;
2931 
2932 	return false;
2933 }
2934 
2935 void sock_def_readable(struct sock *sk);
2936 
2937 int sock_bindtoindex(struct sock *sk, int ifindex, bool lock_sk);
2938 void sock_set_timestamp(struct sock *sk, int optname, bool valbool);
2939 int sock_set_timestamping(struct sock *sk, int optname,
2940 			  struct so_timestamping timestamping);
2941 
2942 void sock_enable_timestamps(struct sock *sk);
2943 void sock_no_linger(struct sock *sk);
2944 void sock_set_keepalive(struct sock *sk);
2945 void sock_set_priority(struct sock *sk, u32 priority);
2946 void sock_set_rcvbuf(struct sock *sk, int val);
2947 void sock_set_mark(struct sock *sk, u32 val);
2948 void sock_set_reuseaddr(struct sock *sk);
2949 void sock_set_reuseport(struct sock *sk);
2950 void sock_set_sndtimeo(struct sock *sk, s64 secs);
2951 
2952 int sock_bind_add(struct sock *sk, struct sockaddr *addr, int addr_len);
2953 
2954 int sock_get_timeout(long timeo, void *optval, bool old_timeval);
2955 int sock_copy_user_timeval(struct __kernel_sock_timeval *tv,
2956 			   sockptr_t optval, int optlen, bool old_timeval);
2957 
2958 int sock_ioctl_inout(struct sock *sk, unsigned int cmd,
2959 		     void __user *arg, void *karg, size_t size);
2960 int sk_ioctl(struct sock *sk, unsigned int cmd, void __user *arg);
2961 static inline bool sk_is_readable(struct sock *sk)
2962 {
2963 	if (sk->sk_prot->sock_is_readable)
2964 		return sk->sk_prot->sock_is_readable(sk);
2965 	return false;
2966 }
2967 #endif	/* _SOCK_H */
2968