xref: /linux/net/core/sock.c (revision 8012eec76b7e6c0d9fb338fe8e3db6ca1b419da4)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * INET		An implementation of the TCP/IP protocol suite for the LINUX
4  *		operating system.  INET is implemented using the  BSD Socket
5  *		interface as the means of communication with the user level.
6  *
7  *		Generic socket support routines. Memory allocators, socket lock/release
8  *		handler for protocols to use and generic option handler.
9  *
10  * Authors:	Ross Biro
11  *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12  *		Florian La Roche, <flla@stud.uni-sb.de>
13  *		Alan Cox, <A.Cox@swansea.ac.uk>
14  *
15  * Fixes:
16  *		Alan Cox	: 	Numerous verify_area() problems
17  *		Alan Cox	:	Connecting on a connecting socket
18  *					now returns an error for tcp.
19  *		Alan Cox	:	sock->protocol is set correctly.
20  *					and is not sometimes left as 0.
21  *		Alan Cox	:	connect handles icmp errors on a
22  *					connect properly. Unfortunately there
23  *					is a restart syscall nasty there. I
24  *					can't match BSD without hacking the C
25  *					library. Ideas urgently sought!
26  *		Alan Cox	:	Disallow bind() to addresses that are
27  *					not ours - especially broadcast ones!!
28  *		Alan Cox	:	Socket 1024 _IS_ ok for users. (fencepost)
29  *		Alan Cox	:	sock_wfree/sock_rfree don't destroy sockets,
30  *					instead they leave that for the DESTROY timer.
31  *		Alan Cox	:	Clean up error flag in accept
32  *		Alan Cox	:	TCP ack handling is buggy, the DESTROY timer
33  *					was buggy. Put a remove_sock() in the handler
34  *					for memory when we hit 0. Also altered the timer
35  *					code. The ACK stuff can wait and needs major
36  *					TCP layer surgery.
37  *		Alan Cox	:	Fixed TCP ack bug, removed remove sock
38  *					and fixed timer/inet_bh race.
39  *		Alan Cox	:	Added zapped flag for TCP
40  *		Alan Cox	:	Move kfree_skb into skbuff.c and tidied up surplus code
41  *		Alan Cox	:	for new sk_buff allocations wmalloc/rmalloc now call alloc_skb
42  *		Alan Cox	:	kfree_s calls now are kfree_skbmem so we can track skb resources
43  *		Alan Cox	:	Supports socket option broadcast now as does udp. Packet and raw need fixing.
44  *		Alan Cox	:	Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so...
45  *		Rick Sladkey	:	Relaxed UDP rules for matching packets.
46  *		C.E.Hawkins	:	IFF_PROMISC/SIOCGHWADDR support
47  *	Pauline Middelink	:	identd support
48  *		Alan Cox	:	Fixed connect() taking signals I think.
49  *		Alan Cox	:	SO_LINGER supported
50  *		Alan Cox	:	Error reporting fixes
51  *		Anonymous	:	inet_create tidied up (sk->reuse setting)
52  *		Alan Cox	:	inet sockets don't set sk->type!
53  *		Alan Cox	:	Split socket option code
54  *		Alan Cox	:	Callbacks
55  *		Alan Cox	:	Nagle flag for Charles & Johannes stuff
56  *		Alex		:	Removed restriction on inet fioctl
57  *		Alan Cox	:	Splitting INET from NET core
58  *		Alan Cox	:	Fixed bogus SO_TYPE handling in getsockopt()
59  *		Adam Caldwell	:	Missing return in SO_DONTROUTE/SO_DEBUG code
60  *		Alan Cox	:	Split IP from generic code
61  *		Alan Cox	:	New kfree_skbmem()
62  *		Alan Cox	:	Make SO_DEBUG superuser only.
63  *		Alan Cox	:	Allow anyone to clear SO_DEBUG
64  *					(compatibility fix)
65  *		Alan Cox	:	Added optimistic memory grabbing for AF_UNIX throughput.
66  *		Alan Cox	:	Allocator for a socket is settable.
67  *		Alan Cox	:	SO_ERROR includes soft errors.
68  *		Alan Cox	:	Allow NULL arguments on some SO_ opts
69  *		Alan Cox	: 	Generic socket allocation to make hooks
70  *					easier (suggested by Craig Metz).
71  *		Michael Pall	:	SO_ERROR returns positive errno again
72  *              Steve Whitehouse:       Added default destructor to free
73  *                                      protocol private data.
74  *              Steve Whitehouse:       Added various other default routines
75  *                                      common to several socket families.
76  *              Chris Evans     :       Call suser() check last on F_SETOWN
77  *		Jay Schulist	:	Added SO_ATTACH_FILTER and SO_DETACH_FILTER.
78  *		Andi Kleen	:	Add sock_kmalloc()/sock_kfree_s()
79  *		Andi Kleen	:	Fix write_space callback
80  *		Chris Evans	:	Security fixes - signedness again
81  *		Arnaldo C. Melo :       cleanups, use skb_queue_purge
82  *
83  * To Fix:
84  */
85 
86 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
87 
88 #include <linux/unaligned.h>
89 #include <linux/capability.h>
90 #include <linux/errno.h>
91 #include <linux/errqueue.h>
92 #include <linux/types.h>
93 #include <linux/socket.h>
94 #include <linux/in.h>
95 #include <linux/kernel.h>
96 #include <linux/module.h>
97 #include <linux/proc_fs.h>
98 #include <linux/seq_file.h>
99 #include <linux/sched.h>
100 #include <linux/sched/mm.h>
101 #include <linux/timer.h>
102 #include <linux/string.h>
103 #include <linux/sockios.h>
104 #include <linux/net.h>
105 #include <linux/mm.h>
106 #include <linux/slab.h>
107 #include <linux/interrupt.h>
108 #include <linux/poll.h>
109 #include <linux/tcp.h>
110 #include <linux/udp.h>
111 #include <linux/init.h>
112 #include <linux/highmem.h>
113 #include <linux/user_namespace.h>
114 #include <linux/static_key.h>
115 #include <linux/memcontrol.h>
116 #include <linux/prefetch.h>
117 #include <linux/compat.h>
118 #include <linux/mroute.h>
119 #include <linux/mroute6.h>
120 #include <linux/icmpv6.h>
121 
122 #include <linux/uaccess.h>
123 
124 #include <linux/netdevice.h>
125 #include <net/protocol.h>
126 #include <linux/skbuff.h>
127 #include <linux/skbuff_ref.h>
128 #include <net/net_namespace.h>
129 #include <net/request_sock.h>
130 #include <net/sock.h>
131 #include <net/proto_memory.h>
132 #include <net/page_pool/types.h>
133 #include <linux/net_tstamp.h>
134 #include <net/xfrm.h>
135 #include <linux/ipsec.h>
136 #include <net/cls_cgroup.h>
137 #include <net/netprio_cgroup.h>
138 #include <linux/sock_diag.h>
139 
140 #include <linux/filter.h>
141 #include <net/sock_reuseport.h>
142 #include <net/bpf_sk_storage.h>
143 
144 #include <trace/events/sock.h>
145 
146 #include <net/psp.h>
147 #include <net/tcp.h>
148 #include <net/busy_poll.h>
149 #include <net/phonet/phonet.h>
150 
151 #include <linux/ethtool.h>
152 
153 #include <uapi/linux/pidfd.h>
154 
155 #include "dev.h"
156 
157 static DEFINE_MUTEX(proto_list_mutex);
158 static LIST_HEAD(proto_list);
159 
160 static void sock_def_write_space_wfree(struct sock *sk, int wmem_alloc);
161 static void sock_def_write_space(struct sock *sk);
162 
163 /**
164  * sk_ns_capable - General socket capability test
165  * @sk: Socket to use a capability on or through
166  * @user_ns: The user namespace of the capability to use
167  * @cap: The capability to use
168  *
169  * Test to see if the opener of the socket had when the socket was
170  * created and the current process has the capability @cap in the user
171  * namespace @user_ns.
172  */
173 bool sk_ns_capable(const struct sock *sk,
174 		   struct user_namespace *user_ns, int cap)
175 {
176 	return file_ns_capable(sk->sk_socket->file, user_ns, cap) &&
177 		ns_capable(user_ns, cap);
178 }
179 EXPORT_SYMBOL(sk_ns_capable);
180 
181 /**
182  * sk_capable - Socket global capability test
183  * @sk: Socket to use a capability on or through
184  * @cap: The global capability to use
185  *
186  * Test to see if the opener of the socket had when the socket was
187  * created and the current process has the capability @cap in all user
188  * namespaces.
189  */
190 bool sk_capable(const struct sock *sk, int cap)
191 {
192 	return sk_ns_capable(sk, &init_user_ns, cap);
193 }
194 EXPORT_SYMBOL(sk_capable);
195 
196 /**
197  * sk_net_capable - Network namespace socket capability test
198  * @sk: Socket to use a capability on or through
199  * @cap: The capability to use
200  *
201  * Test to see if the opener of the socket had when the socket was created
202  * and the current process has the capability @cap over the network namespace
203  * the socket is a member of.
204  */
205 bool sk_net_capable(const struct sock *sk, int cap)
206 {
207 	return sk_ns_capable(sk, sock_net(sk)->user_ns, cap);
208 }
209 EXPORT_SYMBOL(sk_net_capable);
210 
211 /*
212  * Each address family might have different locking rules, so we have
213  * one slock key per address family and separate keys for internal and
214  * userspace sockets.
215  */
216 static struct lock_class_key af_family_keys[AF_MAX];
217 static struct lock_class_key af_family_kern_keys[AF_MAX];
218 static struct lock_class_key af_family_slock_keys[AF_MAX];
219 static struct lock_class_key af_family_kern_slock_keys[AF_MAX];
220 
221 /*
222  * Make lock validator output more readable. (we pre-construct these
223  * strings build-time, so that runtime initialization of socket
224  * locks is fast):
225  */
226 
227 #define _sock_locks(x)						  \
228   x "AF_UNSPEC",	x "AF_UNIX"     ,	x "AF_INET"     , \
229   x "AF_AX25"  ,	x "AF_IPX"      ,	x "AF_APPLETALK", \
230   x "AF_NETROM",	x "AF_BRIDGE"   ,	x "AF_ATMPVC"   , \
231   x "AF_X25"   ,	x "AF_INET6"    ,	x "AF_ROSE"     , \
232   x "AF_DECnet",	x "AF_NETBEUI"  ,	x "AF_SECURITY" , \
233   x "AF_KEY"   ,	x "AF_NETLINK"  ,	x "AF_PACKET"   , \
234   x "AF_ASH"   ,	x "AF_ECONET"   ,	x "AF_ATMSVC"   , \
235   x "AF_RDS"   ,	x "AF_SNA"      ,	x "AF_IRDA"     , \
236   x "AF_PPPOX" ,	x "AF_WANPIPE"  ,	x "AF_LLC"      , \
237   x "27"       ,	x "28"          ,	x "AF_CAN"      , \
238   x "AF_TIPC"  ,	x "AF_BLUETOOTH",	x "IUCV"        , \
239   x "AF_RXRPC" ,	x "AF_ISDN"     ,	x "AF_PHONET"   , \
240   x "AF_IEEE802154",	x "AF_CAIF"	,	x "AF_ALG"      , \
241   x "AF_NFC"   ,	x "AF_VSOCK"    ,	x "AF_KCM"      , \
242   x "AF_QIPCRTR",	x "AF_SMC"	,	x "AF_XDP"	, \
243   x "AF_MCTP"  , \
244   x "AF_MAX"
245 
246 static const char *const af_family_key_strings[AF_MAX+1] = {
247 	_sock_locks("sk_lock-")
248 };
249 static const char *const af_family_slock_key_strings[AF_MAX+1] = {
250 	_sock_locks("slock-")
251 };
252 static const char *const af_family_clock_key_strings[AF_MAX+1] = {
253 	_sock_locks("clock-")
254 };
255 
256 static const char *const af_family_kern_key_strings[AF_MAX+1] = {
257 	_sock_locks("k-sk_lock-")
258 };
259 static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = {
260 	_sock_locks("k-slock-")
261 };
262 static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = {
263 	_sock_locks("k-clock-")
264 };
265 static const char *const af_family_rlock_key_strings[AF_MAX+1] = {
266 	_sock_locks("rlock-")
267 };
268 static const char *const af_family_wlock_key_strings[AF_MAX+1] = {
269 	_sock_locks("wlock-")
270 };
271 static const char *const af_family_elock_key_strings[AF_MAX+1] = {
272 	_sock_locks("elock-")
273 };
274 
275 /*
276  * sk_callback_lock and sk queues locking rules are per-address-family,
277  * so split the lock classes by using a per-AF key:
278  */
279 static struct lock_class_key af_callback_keys[AF_MAX];
280 static struct lock_class_key af_rlock_keys[AF_MAX];
281 static struct lock_class_key af_wlock_keys[AF_MAX];
282 static struct lock_class_key af_elock_keys[AF_MAX];
283 static struct lock_class_key af_kern_callback_keys[AF_MAX];
284 
285 /* Run time adjustable parameters. */
286 __u32 sysctl_wmem_max __read_mostly = 4 << 20;
287 EXPORT_SYMBOL(sysctl_wmem_max);
288 __u32 sysctl_rmem_max __read_mostly = 4 << 20;
289 EXPORT_SYMBOL(sysctl_rmem_max);
290 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_DEFAULT;
291 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_DEFAULT;
292 
293 DEFINE_STATIC_KEY_FALSE(memalloc_socks_key);
294 EXPORT_SYMBOL_GPL(memalloc_socks_key);
295 
296 /**
297  * sk_set_memalloc - sets %SOCK_MEMALLOC
298  * @sk: socket to set it on
299  *
300  * Set %SOCK_MEMALLOC on a socket for access to emergency reserves.
301  * It's the responsibility of the admin to adjust min_free_kbytes
302  * to meet the requirements
303  */
304 void sk_set_memalloc(struct sock *sk)
305 {
306 	sock_set_flag(sk, SOCK_MEMALLOC);
307 	sk->sk_allocation |= __GFP_MEMALLOC;
308 	static_branch_inc(&memalloc_socks_key);
309 }
310 EXPORT_SYMBOL_GPL(sk_set_memalloc);
311 
312 void sk_clear_memalloc(struct sock *sk)
313 {
314 	sock_reset_flag(sk, SOCK_MEMALLOC);
315 	sk->sk_allocation &= ~__GFP_MEMALLOC;
316 	static_branch_dec(&memalloc_socks_key);
317 
318 	/*
319 	 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward
320 	 * progress of swapping. SOCK_MEMALLOC may be cleared while
321 	 * it has rmem allocations due to the last swapfile being deactivated
322 	 * but there is a risk that the socket is unusable due to exceeding
323 	 * the rmem limits. Reclaim the reserves and obey rmem limits again.
324 	 */
325 	sk_mem_reclaim(sk);
326 }
327 EXPORT_SYMBOL_GPL(sk_clear_memalloc);
328 
329 int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
330 {
331 	int ret;
332 	unsigned int noreclaim_flag;
333 
334 	/* these should have been dropped before queueing */
335 	BUG_ON(!sock_flag(sk, SOCK_MEMALLOC));
336 
337 	noreclaim_flag = memalloc_noreclaim_save();
338 	ret = INDIRECT_CALL_INET(sk->sk_backlog_rcv,
339 				 tcp_v6_do_rcv,
340 				 tcp_v4_do_rcv,
341 				 sk, skb);
342 	memalloc_noreclaim_restore(noreclaim_flag);
343 
344 	return ret;
345 }
346 EXPORT_SYMBOL(__sk_backlog_rcv);
347 
348 void sk_error_report(struct sock *sk)
349 {
350 	sk->sk_error_report(sk);
351 
352 	switch (sk->sk_family) {
353 	case AF_INET:
354 		fallthrough;
355 	case AF_INET6:
356 		trace_inet_sk_error_report(sk);
357 		break;
358 	default:
359 		break;
360 	}
361 }
362 EXPORT_SYMBOL(sk_error_report);
363 
364 int sock_get_timeout(long timeo, void *optval, bool old_timeval)
365 {
366 	struct __kernel_sock_timeval tv;
367 
368 	if (timeo == MAX_SCHEDULE_TIMEOUT) {
369 		tv.tv_sec = 0;
370 		tv.tv_usec = 0;
371 	} else {
372 		tv.tv_sec = timeo / HZ;
373 		tv.tv_usec = ((timeo % HZ) * USEC_PER_SEC) / HZ;
374 	}
375 
376 	if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) {
377 		struct old_timeval32 tv32 = { tv.tv_sec, tv.tv_usec };
378 		*(struct old_timeval32 *)optval = tv32;
379 		return sizeof(tv32);
380 	}
381 
382 	if (old_timeval) {
383 		struct __kernel_old_timeval old_tv;
384 		old_tv.tv_sec = tv.tv_sec;
385 		old_tv.tv_usec = tv.tv_usec;
386 		*(struct __kernel_old_timeval *)optval = old_tv;
387 		return sizeof(old_tv);
388 	}
389 
390 	*(struct __kernel_sock_timeval *)optval = tv;
391 	return sizeof(tv);
392 }
393 EXPORT_SYMBOL(sock_get_timeout);
394 
395 int sock_copy_user_timeval(struct __kernel_sock_timeval *tv,
396 			   sockptr_t optval, int optlen, bool old_timeval)
397 {
398 	if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) {
399 		struct old_timeval32 tv32;
400 
401 		if (optlen < sizeof(tv32))
402 			return -EINVAL;
403 
404 		if (copy_from_sockptr(&tv32, optval, sizeof(tv32)))
405 			return -EFAULT;
406 		tv->tv_sec = tv32.tv_sec;
407 		tv->tv_usec = tv32.tv_usec;
408 	} else if (old_timeval) {
409 		struct __kernel_old_timeval old_tv;
410 
411 		if (optlen < sizeof(old_tv))
412 			return -EINVAL;
413 		if (copy_from_sockptr(&old_tv, optval, sizeof(old_tv)))
414 			return -EFAULT;
415 		tv->tv_sec = old_tv.tv_sec;
416 		tv->tv_usec = old_tv.tv_usec;
417 	} else {
418 		if (optlen < sizeof(*tv))
419 			return -EINVAL;
420 		if (copy_from_sockptr(tv, optval, sizeof(*tv)))
421 			return -EFAULT;
422 	}
423 
424 	return 0;
425 }
426 EXPORT_SYMBOL(sock_copy_user_timeval);
427 
428 static int sock_set_timeout(long *timeo_p, sockptr_t optval, int optlen,
429 			    bool old_timeval)
430 {
431 	struct __kernel_sock_timeval tv;
432 	int err = sock_copy_user_timeval(&tv, optval, optlen, old_timeval);
433 	long val;
434 
435 	if (err)
436 		return err;
437 
438 	if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC)
439 		return -EDOM;
440 
441 	if (tv.tv_sec < 0) {
442 		static int warned __read_mostly;
443 
444 		WRITE_ONCE(*timeo_p, 0);
445 		if (warned < 10 && net_ratelimit()) {
446 			warned++;
447 			pr_info("%s: `%s' (pid %d) tries to set negative timeout\n",
448 				__func__, current->comm, task_pid_nr(current));
449 		}
450 		return 0;
451 	}
452 	val = MAX_SCHEDULE_TIMEOUT;
453 	if ((tv.tv_sec || tv.tv_usec) &&
454 	    (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT / HZ - 1)))
455 		val = tv.tv_sec * HZ + DIV_ROUND_UP((unsigned long)tv.tv_usec,
456 						    USEC_PER_SEC / HZ);
457 	WRITE_ONCE(*timeo_p, val);
458 	return 0;
459 }
460 
461 static bool sk_set_prio_allowed(const struct sock *sk, int val)
462 {
463 	return ((val >= TC_PRIO_BESTEFFORT && val <= TC_PRIO_INTERACTIVE) ||
464 		sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) ||
465 		sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN));
466 }
467 
468 static bool sock_needs_netstamp(const struct sock *sk)
469 {
470 	switch (sk->sk_family) {
471 	case AF_UNSPEC:
472 	case AF_UNIX:
473 		return false;
474 	default:
475 		return true;
476 	}
477 }
478 
479 static void sock_disable_timestamp(struct sock *sk, unsigned long flags)
480 {
481 	if (sk->sk_flags & flags) {
482 		sk->sk_flags &= ~flags;
483 		if (sock_needs_netstamp(sk) &&
484 		    !(sk->sk_flags & SK_FLAGS_TIMESTAMP))
485 			net_disable_timestamp();
486 	}
487 }
488 
489 
490 int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
491 {
492 	unsigned long flags;
493 	struct sk_buff_head *list = &sk->sk_receive_queue;
494 
495 	if (atomic_read(&sk->sk_rmem_alloc) >= READ_ONCE(sk->sk_rcvbuf)) {
496 		sk_drops_inc(sk);
497 		trace_sock_rcvqueue_full(sk, skb);
498 		return -ENOMEM;
499 	}
500 
501 	if (!sk_rmem_schedule(sk, skb, skb->truesize)) {
502 		sk_drops_inc(sk);
503 		return -ENOBUFS;
504 	}
505 
506 	skb->dev = NULL;
507 	skb_set_owner_r(skb, sk);
508 
509 	/* we escape from rcu protected region, make sure we dont leak
510 	 * a norefcounted dst
511 	 */
512 	skb_dst_force(skb);
513 
514 	spin_lock_irqsave(&list->lock, flags);
515 	sock_skb_set_dropcount(sk, skb);
516 	__skb_queue_tail(list, skb);
517 	spin_unlock_irqrestore(&list->lock, flags);
518 
519 	if (!sock_flag(sk, SOCK_DEAD))
520 		sk->sk_data_ready(sk);
521 	return 0;
522 }
523 EXPORT_SYMBOL(__sock_queue_rcv_skb);
524 
525 enum skb_drop_reason
526 sock_queue_rcv_skb_reason(struct sock *sk, struct sk_buff *skb)
527 {
528 	enum skb_drop_reason drop_reason;
529 	int err;
530 
531 	drop_reason = sk_filter_reason(sk, skb);
532 	if (drop_reason)
533 		return drop_reason;
534 
535 	err = __sock_queue_rcv_skb(sk, skb);
536 	switch (err) {
537 	case -ENOMEM:
538 		return SKB_DROP_REASON_SOCKET_RCVBUFF;
539 	case -ENOBUFS:
540 		return SKB_DROP_REASON_PROTO_MEM;
541 	}
542 	return SKB_NOT_DROPPED_YET;
543 }
544 EXPORT_SYMBOL(sock_queue_rcv_skb_reason);
545 
546 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb,
547 		     const int nested, unsigned int trim_cap, bool refcounted)
548 {
549 	enum skb_drop_reason reason;
550 	int rc = NET_RX_SUCCESS;
551 	int err;
552 
553 	reason = sk_filter_trim_cap(sk, skb, trim_cap);
554 	if (reason)
555 		goto discard_and_relse;
556 
557 	skb->dev = NULL;
558 
559 	if (sk_rcvqueues_full(sk, READ_ONCE(sk->sk_rcvbuf))) {
560 		sk_drops_inc(sk);
561 		reason = SKB_DROP_REASON_SOCKET_RCVBUFF;
562 		goto discard_and_relse;
563 	}
564 	if (nested)
565 		bh_lock_sock_nested(sk);
566 	else
567 		bh_lock_sock(sk);
568 	if (!sock_owned_by_user(sk)) {
569 		/*
570 		 * trylock + unlock semantics:
571 		 */
572 		mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_);
573 
574 		rc = sk_backlog_rcv(sk, skb);
575 
576 		mutex_release(&sk->sk_lock.dep_map, _RET_IP_);
577 	} else if ((err = sk_add_backlog(sk, skb, READ_ONCE(sk->sk_rcvbuf)))) {
578 		bh_unlock_sock(sk);
579 		if (err == -ENOMEM)
580 			reason = SKB_DROP_REASON_PFMEMALLOC;
581 		if (err == -ENOBUFS)
582 			reason = SKB_DROP_REASON_SOCKET_BACKLOG;
583 		sk_drops_inc(sk);
584 		goto discard_and_relse;
585 	}
586 
587 	bh_unlock_sock(sk);
588 out:
589 	if (refcounted)
590 		sock_put(sk);
591 	return rc;
592 discard_and_relse:
593 	sk_skb_reason_drop(sk, skb, reason);
594 	goto out;
595 }
596 EXPORT_SYMBOL(__sk_receive_skb);
597 
598 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ip6_dst_check(struct dst_entry *,
599 							  u32));
600 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
601 							   u32));
602 struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie)
603 {
604 	struct dst_entry *dst = __sk_dst_get(sk);
605 
606 	if (dst && READ_ONCE(dst->obsolete) &&
607 	    INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check,
608 			       dst, cookie) == NULL) {
609 		sk_tx_queue_clear(sk);
610 		WRITE_ONCE(sk->sk_dst_pending_confirm, 0);
611 		RCU_INIT_POINTER(sk->sk_dst_cache, NULL);
612 		dst_release(dst);
613 		return NULL;
614 	}
615 
616 	return dst;
617 }
618 EXPORT_SYMBOL(__sk_dst_check);
619 
620 struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie)
621 {
622 	struct dst_entry *dst = sk_dst_get(sk);
623 
624 	if (dst && READ_ONCE(dst->obsolete) &&
625 	    INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check,
626 			       dst, cookie) == NULL) {
627 		sk_dst_reset(sk);
628 		dst_release(dst);
629 		return NULL;
630 	}
631 
632 	return dst;
633 }
634 EXPORT_SYMBOL(sk_dst_check);
635 
636 static int sock_bindtoindex_locked(struct sock *sk, int ifindex)
637 {
638 	int ret = -ENOPROTOOPT;
639 #ifdef CONFIG_NETDEVICES
640 	struct net *net = sock_net(sk);
641 
642 	/* Sorry... */
643 	ret = -EPERM;
644 	if (sk->sk_bound_dev_if && !ns_capable(net->user_ns, CAP_NET_RAW))
645 		goto out;
646 
647 	ret = -EINVAL;
648 	if (ifindex < 0)
649 		goto out;
650 
651 	/* Paired with all READ_ONCE() done locklessly. */
652 	WRITE_ONCE(sk->sk_bound_dev_if, ifindex);
653 
654 	if (sk->sk_prot->rehash)
655 		sk->sk_prot->rehash(sk);
656 	sk_dst_reset(sk);
657 
658 	ret = 0;
659 
660 out:
661 #endif
662 
663 	return ret;
664 }
665 
666 int sock_bindtoindex(struct sock *sk, int ifindex, bool lock_sk)
667 {
668 	int ret;
669 
670 	if (lock_sk)
671 		lock_sock(sk);
672 	ret = sock_bindtoindex_locked(sk, ifindex);
673 	if (lock_sk)
674 		release_sock(sk);
675 
676 	return ret;
677 }
678 EXPORT_SYMBOL(sock_bindtoindex);
679 
680 static int sock_setbindtodevice(struct sock *sk, sockptr_t optval, int optlen)
681 {
682 	int ret = -ENOPROTOOPT;
683 #ifdef CONFIG_NETDEVICES
684 	struct net *net = sock_net(sk);
685 	char devname[IFNAMSIZ];
686 	int index;
687 
688 	ret = -EINVAL;
689 	if (optlen < 0)
690 		goto out;
691 
692 	/* Bind this socket to a particular device like "eth0",
693 	 * as specified in the passed interface name. If the
694 	 * name is "" or the option length is zero the socket
695 	 * is not bound.
696 	 */
697 	if (optlen > IFNAMSIZ - 1)
698 		optlen = IFNAMSIZ - 1;
699 	memset(devname, 0, sizeof(devname));
700 
701 	ret = -EFAULT;
702 	if (copy_from_sockptr(devname, optval, optlen))
703 		goto out;
704 
705 	index = 0;
706 	if (devname[0] != '\0') {
707 		struct net_device *dev;
708 
709 		rcu_read_lock();
710 		dev = dev_get_by_name_rcu(net, devname);
711 		if (dev)
712 			index = dev->ifindex;
713 		rcu_read_unlock();
714 		ret = -ENODEV;
715 		if (!dev)
716 			goto out;
717 	}
718 
719 	sockopt_lock_sock(sk);
720 	ret = sock_bindtoindex_locked(sk, index);
721 	sockopt_release_sock(sk);
722 out:
723 #endif
724 
725 	return ret;
726 }
727 
728 static int sock_getbindtodevice(struct sock *sk, sockptr_t optval,
729 				sockptr_t optlen, int len)
730 {
731 	int ret = -ENOPROTOOPT;
732 #ifdef CONFIG_NETDEVICES
733 	int bound_dev_if = READ_ONCE(sk->sk_bound_dev_if);
734 	struct net *net = sock_net(sk);
735 	char devname[IFNAMSIZ];
736 
737 	if (bound_dev_if == 0) {
738 		len = 0;
739 		goto zero;
740 	}
741 
742 	ret = -EINVAL;
743 	if (len < IFNAMSIZ)
744 		goto out;
745 
746 	ret = netdev_get_name(net, devname, bound_dev_if);
747 	if (ret)
748 		goto out;
749 
750 	len = strlen(devname) + 1;
751 
752 	ret = -EFAULT;
753 	if (copy_to_sockptr(optval, devname, len))
754 		goto out;
755 
756 zero:
757 	ret = -EFAULT;
758 	if (copy_to_sockptr(optlen, &len, sizeof(int)))
759 		goto out;
760 
761 	ret = 0;
762 
763 out:
764 #endif
765 
766 	return ret;
767 }
768 
769 bool sk_mc_loop(const struct sock *sk)
770 {
771 	if (dev_recursion_level())
772 		return false;
773 	sk = sk_const_to_full_sk(sk);
774 	if (!sk)
775 		return true;
776 
777 	switch (READ_ONCE(sk->sk_family)) {
778 	case AF_INET:
779 		return inet_test_bit(MC_LOOP, sk);
780 #if IS_ENABLED(CONFIG_IPV6)
781 	case AF_INET6:
782 		return inet6_test_bit(MC6_LOOP, sk);
783 #endif
784 	}
785 	return true;
786 }
787 EXPORT_SYMBOL(sk_mc_loop);
788 
789 void sock_set_reuseaddr(struct sock *sk)
790 {
791 	lock_sock(sk);
792 	sk->sk_reuse = SK_CAN_REUSE;
793 	release_sock(sk);
794 }
795 EXPORT_SYMBOL(sock_set_reuseaddr);
796 
797 void sock_set_reuseport(struct sock *sk)
798 {
799 	lock_sock(sk);
800 	sk->sk_reuseport = true;
801 	release_sock(sk);
802 }
803 EXPORT_SYMBOL(sock_set_reuseport);
804 
805 void sock_no_linger(struct sock *sk)
806 {
807 	lock_sock(sk);
808 	WRITE_ONCE(sk->sk_lingertime, 0);
809 	sock_set_flag(sk, SOCK_LINGER);
810 	release_sock(sk);
811 }
812 EXPORT_SYMBOL(sock_no_linger);
813 
814 void sock_set_priority(struct sock *sk, u32 priority)
815 {
816 	WRITE_ONCE(sk->sk_priority, priority);
817 }
818 EXPORT_SYMBOL(sock_set_priority);
819 
820 void sock_set_sndtimeo(struct sock *sk, s64 secs)
821 {
822 	if (secs && secs < MAX_SCHEDULE_TIMEOUT / HZ - 1)
823 		WRITE_ONCE(sk->sk_sndtimeo, secs * HZ);
824 	else
825 		WRITE_ONCE(sk->sk_sndtimeo, MAX_SCHEDULE_TIMEOUT);
826 }
827 EXPORT_SYMBOL(sock_set_sndtimeo);
828 
829 static void __sock_set_timestamps(struct sock *sk, bool val, bool new, bool ns)
830 {
831 	sock_valbool_flag(sk, SOCK_RCVTSTAMP, val);
832 	sock_valbool_flag(sk, SOCK_RCVTSTAMPNS, val && ns);
833 	if (val)  {
834 		sock_valbool_flag(sk, SOCK_TSTAMP_NEW, new);
835 		sock_enable_timestamp(sk, SOCK_TIMESTAMP);
836 	}
837 }
838 
839 void sock_set_timestamp(struct sock *sk, int optname, bool valbool)
840 {
841 	switch (optname) {
842 	case SO_TIMESTAMP_OLD:
843 		__sock_set_timestamps(sk, valbool, false, false);
844 		break;
845 	case SO_TIMESTAMP_NEW:
846 		__sock_set_timestamps(sk, valbool, true, false);
847 		break;
848 	case SO_TIMESTAMPNS_OLD:
849 		__sock_set_timestamps(sk, valbool, false, true);
850 		break;
851 	case SO_TIMESTAMPNS_NEW:
852 		__sock_set_timestamps(sk, valbool, true, true);
853 		break;
854 	}
855 }
856 
857 static int sock_timestamping_bind_phc(struct sock *sk, int phc_index)
858 {
859 	struct net *net = sock_net(sk);
860 	struct net_device *dev = NULL;
861 	bool match = false;
862 	int *vclock_index;
863 	int i, num;
864 
865 	if (sk->sk_bound_dev_if)
866 		dev = dev_get_by_index(net, sk->sk_bound_dev_if);
867 
868 	if (!dev) {
869 		pr_err("%s: sock not bind to device\n", __func__);
870 		return -EOPNOTSUPP;
871 	}
872 
873 	num = ethtool_get_phc_vclocks(dev, &vclock_index);
874 	dev_put(dev);
875 
876 	for (i = 0; i < num; i++) {
877 		if (*(vclock_index + i) == phc_index) {
878 			match = true;
879 			break;
880 		}
881 	}
882 
883 	if (num > 0)
884 		kfree(vclock_index);
885 
886 	if (!match)
887 		return -EINVAL;
888 
889 	WRITE_ONCE(sk->sk_bind_phc, phc_index);
890 
891 	return 0;
892 }
893 
894 int sock_set_timestamping(struct sock *sk, int optname,
895 			  struct so_timestamping timestamping)
896 {
897 	int val = timestamping.flags;
898 	int ret;
899 
900 	if (val & ~SOF_TIMESTAMPING_MASK)
901 		return -EINVAL;
902 
903 	if (val & SOF_TIMESTAMPING_OPT_ID_TCP &&
904 	    !(val & SOF_TIMESTAMPING_OPT_ID))
905 		return -EINVAL;
906 
907 	if (val & SOF_TIMESTAMPING_OPT_ID &&
908 	    !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) {
909 		if (sk_is_tcp(sk)) {
910 			if ((1 << sk->sk_state) &
911 			    (TCPF_CLOSE | TCPF_LISTEN))
912 				return -EINVAL;
913 			if (val & SOF_TIMESTAMPING_OPT_ID_TCP)
914 				atomic_set(&sk->sk_tskey, tcp_sk(sk)->write_seq);
915 			else
916 				atomic_set(&sk->sk_tskey, tcp_sk(sk)->snd_una);
917 		} else {
918 			atomic_set(&sk->sk_tskey, 0);
919 		}
920 	}
921 
922 	if (val & SOF_TIMESTAMPING_OPT_STATS &&
923 	    !(val & SOF_TIMESTAMPING_OPT_TSONLY))
924 		return -EINVAL;
925 
926 	if (val & SOF_TIMESTAMPING_BIND_PHC) {
927 		ret = sock_timestamping_bind_phc(sk, timestamping.bind_phc);
928 		if (ret)
929 			return ret;
930 	}
931 
932 	WRITE_ONCE(sk->sk_tsflags, val);
933 	sock_valbool_flag(sk, SOCK_TSTAMP_NEW, optname == SO_TIMESTAMPING_NEW);
934 	sock_valbool_flag(sk, SOCK_TIMESTAMPING_ANY, !!(val & TSFLAGS_ANY));
935 
936 	if (val & SOF_TIMESTAMPING_RX_SOFTWARE)
937 		sock_enable_timestamp(sk,
938 				      SOCK_TIMESTAMPING_RX_SOFTWARE);
939 	else
940 		sock_disable_timestamp(sk,
941 				       (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE));
942 	return 0;
943 }
944 
945 #if defined(CONFIG_CGROUP_BPF)
946 void bpf_skops_tx_timestamping(struct sock *sk, struct sk_buff *skb, int op)
947 {
948 	struct bpf_sock_ops_kern sock_ops;
949 
950 	memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
951 	sock_ops.op = op;
952 	sock_ops.is_fullsock = 1;
953 	sock_ops.sk = sk;
954 	bpf_skops_init_skb(&sock_ops, skb, 0);
955 	__cgroup_bpf_run_filter_sock_ops(sk, &sock_ops, CGROUP_SOCK_OPS);
956 }
957 #endif
958 
959 void sock_set_keepalive(struct sock *sk)
960 {
961 	lock_sock(sk);
962 	if (sk->sk_prot->keepalive)
963 		sk->sk_prot->keepalive(sk, true);
964 	sock_valbool_flag(sk, SOCK_KEEPOPEN, true);
965 	release_sock(sk);
966 }
967 EXPORT_SYMBOL(sock_set_keepalive);
968 
969 static void __sock_set_rcvbuf(struct sock *sk, int val)
970 {
971 	struct socket *sock = sk->sk_socket;
972 
973 	/* Ensure val * 2 fits into an int, to prevent max_t() from treating it
974 	 * as a negative value.
975 	 */
976 	val = min_t(int, val, INT_MAX / 2);
977 	sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
978 
979 	/* We double it on the way in to account for "struct sk_buff" etc.
980 	 * overhead.   Applications assume that the SO_RCVBUF setting they make
981 	 * will allow that much actual data to be received on that socket.
982 	 *
983 	 * Applications are unaware that "struct sk_buff" and other overheads
984 	 * allocate from the receive buffer during socket buffer allocation.
985 	 *
986 	 * And after considering the possible alternatives, returning the value
987 	 * we actually used in getsockopt is the most desirable behavior.
988 	 */
989 	WRITE_ONCE(sk->sk_rcvbuf, max_t(int, val * 2, SOCK_MIN_RCVBUF));
990 
991 	if (sock) {
992 		const struct proto_ops *ops = READ_ONCE(sock->ops);
993 
994 		if (ops->set_rcvbuf)
995 			ops->set_rcvbuf(sk, sk->sk_rcvbuf);
996 	}
997 }
998 
999 void sock_set_rcvbuf(struct sock *sk, int val)
1000 {
1001 	lock_sock(sk);
1002 	__sock_set_rcvbuf(sk, val);
1003 	release_sock(sk);
1004 }
1005 EXPORT_SYMBOL(sock_set_rcvbuf);
1006 
1007 static void __sock_set_mark(struct sock *sk, u32 val)
1008 {
1009 	if (val != sk->sk_mark) {
1010 		WRITE_ONCE(sk->sk_mark, val);
1011 		sk_dst_reset(sk);
1012 	}
1013 }
1014 
1015 void sock_set_mark(struct sock *sk, u32 val)
1016 {
1017 	lock_sock(sk);
1018 	__sock_set_mark(sk, val);
1019 	release_sock(sk);
1020 }
1021 EXPORT_SYMBOL(sock_set_mark);
1022 
1023 static void sock_release_reserved_memory(struct sock *sk, int bytes)
1024 {
1025 	/* Round down bytes to multiple of pages */
1026 	bytes = round_down(bytes, PAGE_SIZE);
1027 
1028 	WARN_ON(bytes > sk->sk_reserved_mem);
1029 	WRITE_ONCE(sk->sk_reserved_mem, sk->sk_reserved_mem - bytes);
1030 	sk_mem_reclaim(sk);
1031 }
1032 
1033 static int sock_reserve_memory(struct sock *sk, int bytes)
1034 {
1035 	long allocated;
1036 	bool charged;
1037 	int pages;
1038 
1039 	if (!mem_cgroup_sk_enabled(sk) || !sk_is_tcp(sk))
1040 		return -EOPNOTSUPP;
1041 
1042 	if (!bytes)
1043 		return 0;
1044 
1045 	pages = sk_mem_pages(bytes);
1046 
1047 	/* pre-charge to memcg */
1048 	charged = mem_cgroup_sk_charge(sk, pages,
1049 				       GFP_KERNEL | __GFP_RETRY_MAYFAIL);
1050 	if (!charged)
1051 		return -ENOMEM;
1052 
1053 	if (sk->sk_bypass_prot_mem)
1054 		goto success;
1055 
1056 	/* pre-charge to forward_alloc */
1057 	sk_memory_allocated_add(sk, pages);
1058 	allocated = sk_memory_allocated(sk);
1059 
1060 	/* If the system goes into memory pressure with this
1061 	 * precharge, give up and return error.
1062 	 */
1063 	if (allocated > sk_prot_mem_limits(sk, 1)) {
1064 		sk_memory_allocated_sub(sk, pages);
1065 		mem_cgroup_sk_uncharge(sk, pages);
1066 		return -ENOMEM;
1067 	}
1068 
1069 success:
1070 	sk_forward_alloc_add(sk, pages << PAGE_SHIFT);
1071 
1072 	WRITE_ONCE(sk->sk_reserved_mem,
1073 		   sk->sk_reserved_mem + (pages << PAGE_SHIFT));
1074 
1075 	return 0;
1076 }
1077 
1078 #ifdef CONFIG_PAGE_POOL
1079 
1080 /* This is the number of tokens and frags that the user can SO_DEVMEM_DONTNEED
1081  * in 1 syscall. The limit exists to limit the amount of memory the kernel
1082  * allocates to copy these tokens, and to prevent looping over the frags for
1083  * too long.
1084  */
1085 #define MAX_DONTNEED_TOKENS 128
1086 #define MAX_DONTNEED_FRAGS 1024
1087 
1088 static noinline_for_stack int
1089 sock_devmem_dontneed(struct sock *sk, sockptr_t optval, unsigned int optlen)
1090 {
1091 	unsigned int num_tokens, i, j, netmem_num = 0;
1092 	struct dmabuf_token *tokens;
1093 	int ret = 0, num_frags = 0;
1094 	netmem_ref netmems[16];
1095 
1096 	if (!sk_is_tcp(sk))
1097 		return -EBADF;
1098 
1099 	if (optlen % sizeof(*tokens) ||
1100 	    optlen > sizeof(*tokens) * MAX_DONTNEED_TOKENS)
1101 		return -EINVAL;
1102 
1103 	num_tokens = optlen / sizeof(*tokens);
1104 	tokens = kvmalloc_objs(*tokens, num_tokens);
1105 	if (!tokens)
1106 		return -ENOMEM;
1107 
1108 	if (copy_from_sockptr(tokens, optval, optlen)) {
1109 		kvfree(tokens);
1110 		return -EFAULT;
1111 	}
1112 
1113 	xa_lock_bh(&sk->sk_user_frags);
1114 	for (i = 0; i < num_tokens; i++) {
1115 		for (j = 0; j < tokens[i].token_count; j++) {
1116 			if (++num_frags > MAX_DONTNEED_FRAGS)
1117 				goto frag_limit_reached;
1118 
1119 			netmem_ref netmem = (__force netmem_ref)__xa_erase(
1120 				&sk->sk_user_frags, tokens[i].token_start + j);
1121 
1122 			if (!netmem || WARN_ON_ONCE(!netmem_is_net_iov(netmem)))
1123 				continue;
1124 
1125 			netmems[netmem_num++] = netmem;
1126 			if (netmem_num == ARRAY_SIZE(netmems)) {
1127 				xa_unlock_bh(&sk->sk_user_frags);
1128 				page_pool_put_netmem_bulk(netmems,
1129 							  ARRAY_SIZE(netmems));
1130 				netmem_num = 0;
1131 				xa_lock_bh(&sk->sk_user_frags);
1132 			}
1133 			ret++;
1134 		}
1135 	}
1136 
1137 frag_limit_reached:
1138 	xa_unlock_bh(&sk->sk_user_frags);
1139 	page_pool_put_netmem_bulk(netmems, netmem_num);
1140 
1141 	kvfree(tokens);
1142 	return ret;
1143 }
1144 #endif
1145 
1146 void sockopt_lock_sock(struct sock *sk)
1147 {
1148 	/* When current->bpf_ctx is set, the setsockopt is called from
1149 	 * a bpf prog.  bpf has ensured the sk lock has been
1150 	 * acquired before calling setsockopt().
1151 	 */
1152 	if (has_current_bpf_ctx())
1153 		return;
1154 
1155 	lock_sock(sk);
1156 }
1157 EXPORT_SYMBOL(sockopt_lock_sock);
1158 
1159 void sockopt_release_sock(struct sock *sk)
1160 {
1161 	if (has_current_bpf_ctx())
1162 		return;
1163 
1164 	release_sock(sk);
1165 }
1166 EXPORT_SYMBOL(sockopt_release_sock);
1167 
1168 bool sockopt_ns_capable(struct user_namespace *ns, int cap)
1169 {
1170 	return has_current_bpf_ctx() || ns_capable(ns, cap);
1171 }
1172 EXPORT_SYMBOL(sockopt_ns_capable);
1173 
1174 bool sockopt_capable(int cap)
1175 {
1176 	return has_current_bpf_ctx() || capable(cap);
1177 }
1178 EXPORT_SYMBOL(sockopt_capable);
1179 
1180 static int sockopt_validate_clockid(__kernel_clockid_t value)
1181 {
1182 	switch (value) {
1183 	case CLOCK_REALTIME:
1184 	case CLOCK_MONOTONIC:
1185 	case CLOCK_TAI:
1186 		return 0;
1187 	}
1188 	return -EINVAL;
1189 }
1190 
1191 /*
1192  *	This is meant for all protocols to use and covers goings on
1193  *	at the socket level. Everything here is generic.
1194  */
1195 
1196 int sk_setsockopt(struct sock *sk, int level, int optname,
1197 		  sockptr_t optval, unsigned int optlen)
1198 {
1199 	struct so_timestamping timestamping;
1200 	struct socket *sock = sk->sk_socket;
1201 	struct sock_txtime sk_txtime;
1202 	int val;
1203 	int valbool;
1204 	struct linger ling;
1205 	int ret = 0;
1206 
1207 	/*
1208 	 *	Options without arguments
1209 	 */
1210 
1211 	if (optname == SO_BINDTODEVICE)
1212 		return sock_setbindtodevice(sk, optval, optlen);
1213 
1214 	if (optlen < sizeof(int))
1215 		return -EINVAL;
1216 
1217 	if (copy_from_sockptr(&val, optval, sizeof(val)))
1218 		return -EFAULT;
1219 
1220 	valbool = val ? 1 : 0;
1221 
1222 	/* handle options which do not require locking the socket. */
1223 	switch (optname) {
1224 	case SO_PRIORITY:
1225 		if (sk_set_prio_allowed(sk, val)) {
1226 			sock_set_priority(sk, val);
1227 			return 0;
1228 		}
1229 		return -EPERM;
1230 	case SO_TYPE:
1231 	case SO_PROTOCOL:
1232 	case SO_DOMAIN:
1233 	case SO_ERROR:
1234 		return -ENOPROTOOPT;
1235 #ifdef CONFIG_NET_RX_BUSY_POLL
1236 	case SO_BUSY_POLL:
1237 		if (val < 0)
1238 			return -EINVAL;
1239 		WRITE_ONCE(sk->sk_ll_usec, val);
1240 		return 0;
1241 	case SO_PREFER_BUSY_POLL:
1242 		if (valbool && !sockopt_capable(CAP_NET_ADMIN))
1243 			return -EPERM;
1244 		WRITE_ONCE(sk->sk_prefer_busy_poll, valbool);
1245 		return 0;
1246 	case SO_BUSY_POLL_BUDGET:
1247 		if (val > READ_ONCE(sk->sk_busy_poll_budget) &&
1248 		    !sockopt_capable(CAP_NET_ADMIN))
1249 			return -EPERM;
1250 		if (val < 0 || val > U16_MAX)
1251 			return -EINVAL;
1252 		WRITE_ONCE(sk->sk_busy_poll_budget, val);
1253 		return 0;
1254 #endif
1255 	case SO_MAX_PACING_RATE:
1256 		{
1257 		unsigned long ulval = (val == ~0U) ? ~0UL : (unsigned int)val;
1258 		unsigned long pacing_rate;
1259 
1260 		if (sizeof(ulval) != sizeof(val) &&
1261 		    optlen >= sizeof(ulval) &&
1262 		    copy_from_sockptr(&ulval, optval, sizeof(ulval))) {
1263 			return -EFAULT;
1264 		}
1265 		if (ulval != ~0UL)
1266 			cmpxchg(&sk->sk_pacing_status,
1267 				SK_PACING_NONE,
1268 				SK_PACING_NEEDED);
1269 		/* Pairs with READ_ONCE() from sk_getsockopt() */
1270 		WRITE_ONCE(sk->sk_max_pacing_rate, ulval);
1271 		pacing_rate = READ_ONCE(sk->sk_pacing_rate);
1272 		if (ulval < pacing_rate)
1273 			WRITE_ONCE(sk->sk_pacing_rate, ulval);
1274 		return 0;
1275 		}
1276 	case SO_TXREHASH:
1277 		if (!sk_is_tcp(sk))
1278 			return -EOPNOTSUPP;
1279 		if (val < -1 || val > 1)
1280 			return -EINVAL;
1281 		if ((u8)val == SOCK_TXREHASH_DEFAULT)
1282 			val = READ_ONCE(sock_net(sk)->core.sysctl_txrehash);
1283 		/* Paired with READ_ONCE() in tcp_rtx_synack()
1284 		 * and sk_getsockopt().
1285 		 */
1286 		WRITE_ONCE(sk->sk_txrehash, (u8)val);
1287 		return 0;
1288 	case SO_PEEK_OFF:
1289 		{
1290 		int (*set_peek_off)(struct sock *sk, int val);
1291 
1292 		set_peek_off = READ_ONCE(sock->ops)->set_peek_off;
1293 		if (set_peek_off)
1294 			ret = set_peek_off(sk, val);
1295 		else
1296 			ret = -EOPNOTSUPP;
1297 		return ret;
1298 		}
1299 #ifdef CONFIG_PAGE_POOL
1300 	case SO_DEVMEM_DONTNEED:
1301 		return sock_devmem_dontneed(sk, optval, optlen);
1302 #endif
1303 	case SO_SNDTIMEO_OLD:
1304 	case SO_SNDTIMEO_NEW:
1305 		return sock_set_timeout(&sk->sk_sndtimeo, optval,
1306 					optlen, optname == SO_SNDTIMEO_OLD);
1307 	case SO_RCVTIMEO_OLD:
1308 	case SO_RCVTIMEO_NEW:
1309 		return sock_set_timeout(&sk->sk_rcvtimeo, optval,
1310 					optlen, optname == SO_RCVTIMEO_OLD);
1311 	}
1312 
1313 	sockopt_lock_sock(sk);
1314 
1315 	switch (optname) {
1316 	case SO_DEBUG:
1317 		if (val && !sockopt_capable(CAP_NET_ADMIN))
1318 			ret = -EACCES;
1319 		else
1320 			sock_valbool_flag(sk, SOCK_DBG, valbool);
1321 		break;
1322 	case SO_REUSEADDR:
1323 		sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE);
1324 		break;
1325 	case SO_REUSEPORT:
1326 		if (valbool && !sk_is_inet(sk))
1327 			ret = -EOPNOTSUPP;
1328 		else
1329 			sk->sk_reuseport = valbool;
1330 		break;
1331 	case SO_DONTROUTE:
1332 		sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool);
1333 		sk_dst_reset(sk);
1334 		break;
1335 	case SO_BROADCAST:
1336 		sock_valbool_flag(sk, SOCK_BROADCAST, valbool);
1337 		break;
1338 	case SO_SNDBUF:
1339 		/* Don't error on this BSD doesn't and if you think
1340 		 * about it this is right. Otherwise apps have to
1341 		 * play 'guess the biggest size' games. RCVBUF/SNDBUF
1342 		 * are treated in BSD as hints
1343 		 */
1344 		val = min_t(u32, val, READ_ONCE(sysctl_wmem_max));
1345 set_sndbuf:
1346 		/* Ensure val * 2 fits into an int, to prevent max_t()
1347 		 * from treating it as a negative value.
1348 		 */
1349 		val = min_t(int, val, INT_MAX / 2);
1350 		sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
1351 		WRITE_ONCE(sk->sk_sndbuf,
1352 			   max_t(int, val * 2, SOCK_MIN_SNDBUF));
1353 		/* Wake up sending tasks if we upped the value. */
1354 		sk->sk_write_space(sk);
1355 		break;
1356 
1357 	case SO_SNDBUFFORCE:
1358 		if (!sockopt_capable(CAP_NET_ADMIN)) {
1359 			ret = -EPERM;
1360 			break;
1361 		}
1362 
1363 		/* No negative values (to prevent underflow, as val will be
1364 		 * multiplied by 2).
1365 		 */
1366 		if (val < 0)
1367 			val = 0;
1368 		goto set_sndbuf;
1369 
1370 	case SO_RCVBUF:
1371 		/* Don't error on this BSD doesn't and if you think
1372 		 * about it this is right. Otherwise apps have to
1373 		 * play 'guess the biggest size' games. RCVBUF/SNDBUF
1374 		 * are treated in BSD as hints
1375 		 */
1376 		__sock_set_rcvbuf(sk, min_t(u32, val, READ_ONCE(sysctl_rmem_max)));
1377 		break;
1378 
1379 	case SO_RCVBUFFORCE:
1380 		if (!sockopt_capable(CAP_NET_ADMIN)) {
1381 			ret = -EPERM;
1382 			break;
1383 		}
1384 
1385 		/* No negative values (to prevent underflow, as val will be
1386 		 * multiplied by 2).
1387 		 */
1388 		__sock_set_rcvbuf(sk, max(val, 0));
1389 		break;
1390 
1391 	case SO_KEEPALIVE:
1392 		if (sk->sk_prot->keepalive)
1393 			sk->sk_prot->keepalive(sk, valbool);
1394 		sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool);
1395 		break;
1396 
1397 	case SO_OOBINLINE:
1398 		sock_valbool_flag(sk, SOCK_URGINLINE, valbool);
1399 		break;
1400 
1401 	case SO_NO_CHECK:
1402 		sk->sk_no_check_tx = valbool;
1403 		break;
1404 
1405 	case SO_LINGER:
1406 		if (optlen < sizeof(ling)) {
1407 			ret = -EINVAL;	/* 1003.1g */
1408 			break;
1409 		}
1410 		if (copy_from_sockptr(&ling, optval, sizeof(ling))) {
1411 			ret = -EFAULT;
1412 			break;
1413 		}
1414 		if (!ling.l_onoff) {
1415 			sock_reset_flag(sk, SOCK_LINGER);
1416 		} else {
1417 			unsigned long t_sec = ling.l_linger;
1418 
1419 			if (t_sec >= MAX_SCHEDULE_TIMEOUT / HZ)
1420 				WRITE_ONCE(sk->sk_lingertime, MAX_SCHEDULE_TIMEOUT);
1421 			else
1422 				WRITE_ONCE(sk->sk_lingertime, t_sec * HZ);
1423 			sock_set_flag(sk, SOCK_LINGER);
1424 		}
1425 		break;
1426 
1427 	case SO_BSDCOMPAT:
1428 		break;
1429 
1430 	case SO_TIMESTAMP_OLD:
1431 	case SO_TIMESTAMP_NEW:
1432 	case SO_TIMESTAMPNS_OLD:
1433 	case SO_TIMESTAMPNS_NEW:
1434 		sock_set_timestamp(sk, optname, valbool);
1435 		break;
1436 
1437 	case SO_TIMESTAMPING_NEW:
1438 	case SO_TIMESTAMPING_OLD:
1439 		if (optlen == sizeof(timestamping)) {
1440 			if (copy_from_sockptr(&timestamping, optval,
1441 					      sizeof(timestamping))) {
1442 				ret = -EFAULT;
1443 				break;
1444 			}
1445 		} else {
1446 			memset(&timestamping, 0, sizeof(timestamping));
1447 			timestamping.flags = val;
1448 		}
1449 		ret = sock_set_timestamping(sk, optname, timestamping);
1450 		break;
1451 
1452 	case SO_RCVLOWAT:
1453 		{
1454 		int (*set_rcvlowat)(struct sock *sk, int val) = NULL;
1455 
1456 		if (val < 0)
1457 			val = INT_MAX;
1458 		if (sock)
1459 			set_rcvlowat = READ_ONCE(sock->ops)->set_rcvlowat;
1460 		if (set_rcvlowat)
1461 			ret = set_rcvlowat(sk, val);
1462 		else
1463 			WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
1464 		break;
1465 		}
1466 	case SO_ATTACH_FILTER: {
1467 		struct sock_fprog fprog;
1468 
1469 		if (sk_is_tcp(sk) &&
1470 		    !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
1471 			ret = -EPERM;
1472 			break;
1473 		}
1474 		ret = copy_bpf_fprog_from_user(&fprog, optval, optlen);
1475 		if (!ret)
1476 			ret = sk_attach_filter(&fprog, sk);
1477 		break;
1478 	}
1479 	case SO_ATTACH_BPF:
1480 		ret = -EINVAL;
1481 		if (optlen == sizeof(u32)) {
1482 			u32 ufd;
1483 
1484 			ret = -EFAULT;
1485 			if (copy_from_sockptr(&ufd, optval, sizeof(ufd)))
1486 				break;
1487 
1488 			ret = sk_attach_bpf(ufd, sk);
1489 		}
1490 		break;
1491 
1492 	case SO_ATTACH_REUSEPORT_CBPF: {
1493 		struct sock_fprog fprog;
1494 
1495 		ret = copy_bpf_fprog_from_user(&fprog, optval, optlen);
1496 		if (!ret)
1497 			ret = sk_reuseport_attach_filter(&fprog, sk);
1498 		break;
1499 	}
1500 	case SO_ATTACH_REUSEPORT_EBPF:
1501 		ret = -EINVAL;
1502 		if (optlen == sizeof(u32)) {
1503 			u32 ufd;
1504 
1505 			ret = -EFAULT;
1506 			if (copy_from_sockptr(&ufd, optval, sizeof(ufd)))
1507 				break;
1508 
1509 			ret = sk_reuseport_attach_bpf(ufd, sk);
1510 		}
1511 		break;
1512 
1513 	case SO_DETACH_REUSEPORT_BPF:
1514 		ret = reuseport_detach_prog(sk);
1515 		break;
1516 
1517 	case SO_DETACH_FILTER:
1518 		ret = sk_detach_filter(sk);
1519 		break;
1520 
1521 	case SO_LOCK_FILTER:
1522 		if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool)
1523 			ret = -EPERM;
1524 		else
1525 			sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool);
1526 		break;
1527 
1528 	case SO_MARK:
1529 		if (!sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) &&
1530 		    !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
1531 			ret = -EPERM;
1532 			break;
1533 		}
1534 
1535 		__sock_set_mark(sk, val);
1536 		break;
1537 	case SO_RCVMARK:
1538 		sock_valbool_flag(sk, SOCK_RCVMARK, valbool);
1539 		break;
1540 
1541 	case SO_RCVPRIORITY:
1542 		sock_valbool_flag(sk, SOCK_RCVPRIORITY, valbool);
1543 		break;
1544 
1545 	case SO_RXQ_OVFL:
1546 		sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool);
1547 		break;
1548 
1549 	case SO_WIFI_STATUS:
1550 		sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool);
1551 		break;
1552 
1553 	case SO_NOFCS:
1554 		sock_valbool_flag(sk, SOCK_NOFCS, valbool);
1555 		break;
1556 
1557 	case SO_SELECT_ERR_QUEUE:
1558 		sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool);
1559 		break;
1560 
1561 	case SO_PASSCRED:
1562 		if (sk_may_scm_recv(sk))
1563 			sk->sk_scm_credentials = valbool;
1564 		else
1565 			ret = -EOPNOTSUPP;
1566 		break;
1567 
1568 	case SO_PASSSEC:
1569 		if (IS_ENABLED(CONFIG_SECURITY_NETWORK) && sk_may_scm_recv(sk))
1570 			sk->sk_scm_security = valbool;
1571 		else
1572 			ret = -EOPNOTSUPP;
1573 		break;
1574 
1575 	case SO_PASSPIDFD:
1576 		if (sk_is_unix(sk))
1577 			sk->sk_scm_pidfd = valbool;
1578 		else
1579 			ret = -EOPNOTSUPP;
1580 		break;
1581 
1582 	case SO_PASSPIDFD_THREAD:
1583 		if (sk_is_unix(sk))
1584 			sk->sk_scm_pidfd_thread = valbool;
1585 		else
1586 			ret = -EOPNOTSUPP;
1587 		break;
1588 
1589 	case SO_PASSRIGHTS:
1590 		if (sk_is_unix(sk))
1591 			sk->sk_scm_rights = valbool;
1592 		else
1593 			ret = -EOPNOTSUPP;
1594 		break;
1595 
1596 	case SO_INCOMING_CPU:
1597 		reuseport_update_incoming_cpu(sk, val);
1598 		break;
1599 
1600 	case SO_CNX_ADVICE:
1601 		if (val == 1)
1602 			dst_negative_advice(sk);
1603 		break;
1604 
1605 	case SO_ZEROCOPY:
1606 		if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) {
1607 			if (!(sk_is_tcp(sk) ||
1608 			      (sk->sk_type == SOCK_DGRAM &&
1609 			       sk->sk_protocol == IPPROTO_UDP)))
1610 				ret = -EOPNOTSUPP;
1611 		} else if (sk->sk_family != PF_RDS) {
1612 			ret = -EOPNOTSUPP;
1613 		}
1614 		if (!ret) {
1615 			if (val < 0 || val > 1)
1616 				ret = -EINVAL;
1617 			else
1618 				sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool);
1619 		}
1620 		break;
1621 
1622 	case SO_TXTIME:
1623 		if (optlen != sizeof(struct sock_txtime)) {
1624 			ret = -EINVAL;
1625 			break;
1626 		} else if (copy_from_sockptr(&sk_txtime, optval,
1627 			   sizeof(struct sock_txtime))) {
1628 			ret = -EFAULT;
1629 			break;
1630 		} else if (sk_txtime.flags & ~SOF_TXTIME_FLAGS_MASK) {
1631 			ret = -EINVAL;
1632 			break;
1633 		}
1634 		/* CLOCK_MONOTONIC is only used by sch_fq, and this packet
1635 		 * scheduler has enough safe guards.
1636 		 */
1637 		if (sk_txtime.clockid != CLOCK_MONOTONIC &&
1638 		    !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
1639 			ret = -EPERM;
1640 			break;
1641 		}
1642 
1643 		ret = sockopt_validate_clockid(sk_txtime.clockid);
1644 		if (ret)
1645 			break;
1646 
1647 		sock_valbool_flag(sk, SOCK_TXTIME, true);
1648 		sk->sk_clockid = sk_txtime.clockid;
1649 		sk->sk_txtime_deadline_mode =
1650 			!!(sk_txtime.flags & SOF_TXTIME_DEADLINE_MODE);
1651 		sk->sk_txtime_report_errors =
1652 			!!(sk_txtime.flags & SOF_TXTIME_REPORT_ERRORS);
1653 		break;
1654 
1655 	case SO_BINDTOIFINDEX:
1656 		ret = sock_bindtoindex_locked(sk, val);
1657 		break;
1658 
1659 	case SO_BUF_LOCK:
1660 		if (val & ~SOCK_BUF_LOCK_MASK) {
1661 			ret = -EINVAL;
1662 			break;
1663 		}
1664 		sk->sk_userlocks = val | (sk->sk_userlocks &
1665 					  ~SOCK_BUF_LOCK_MASK);
1666 		break;
1667 
1668 	case SO_RESERVE_MEM:
1669 	{
1670 		int delta;
1671 
1672 		if (val < 0 || val > SZ_1G) {
1673 			ret = -EINVAL;
1674 			break;
1675 		}
1676 
1677 		delta = val - sk->sk_reserved_mem;
1678 		if (delta < 0)
1679 			sock_release_reserved_memory(sk, -delta);
1680 		else
1681 			ret = sock_reserve_memory(sk, delta);
1682 		break;
1683 	}
1684 
1685 	default:
1686 		ret = -ENOPROTOOPT;
1687 		break;
1688 	}
1689 	sockopt_release_sock(sk);
1690 	return ret;
1691 }
1692 
1693 int sock_setsockopt(struct socket *sock, int level, int optname,
1694 		    sockptr_t optval, unsigned int optlen)
1695 {
1696 	return sk_setsockopt(sock->sk, level, optname,
1697 			     optval, optlen);
1698 }
1699 EXPORT_SYMBOL(sock_setsockopt);
1700 
1701 static const struct cred *sk_get_peer_cred(struct sock *sk)
1702 {
1703 	const struct cred *cred;
1704 
1705 	spin_lock(&sk->sk_peer_lock);
1706 	cred = get_cred(sk->sk_peer_cred);
1707 	spin_unlock(&sk->sk_peer_lock);
1708 
1709 	return cred;
1710 }
1711 
1712 static void cred_to_ucred(struct pid *pid, const struct cred *cred,
1713 			  struct ucred *ucred)
1714 {
1715 	ucred->pid = pid_vnr(pid);
1716 	ucred->uid = ucred->gid = -1;
1717 	if (cred) {
1718 		struct user_namespace *current_ns = current_user_ns();
1719 
1720 		ucred->uid = from_kuid_munged(current_ns, cred->euid);
1721 		ucred->gid = from_kgid_munged(current_ns, cred->egid);
1722 	}
1723 }
1724 
1725 static int groups_to_user(sockptr_t dst, const struct group_info *src)
1726 {
1727 	struct user_namespace *user_ns = current_user_ns();
1728 	int i;
1729 
1730 	for (i = 0; i < src->ngroups; i++) {
1731 		gid_t gid = from_kgid_munged(user_ns, src->gid[i]);
1732 
1733 		if (copy_to_sockptr_offset(dst, i * sizeof(gid), &gid, sizeof(gid)))
1734 			return -EFAULT;
1735 	}
1736 
1737 	return 0;
1738 }
1739 
1740 /* Hand out a pidfd for @type of the socket's peer via SO_PEERPIDFD*. */
1741 static int sk_getsockopt_peerpidfd(struct sock *sk, enum pid_type type,
1742 				   sockptr_t optval, sockptr_t optlen, int len)
1743 {
1744 	struct file *pidfd_file = NULL;
1745 	unsigned int flags = 0;
1746 	struct pid *peer_pid;
1747 	int pidfd;
1748 
1749 	if (len > sizeof(pidfd))
1750 		len = sizeof(pidfd);
1751 
1752 	spin_lock(&sk->sk_peer_lock);
1753 	peer_pid = get_pid(sk->sk_peer_pid[type]);
1754 	spin_unlock(&sk->sk_peer_lock);
1755 
1756 	if (!peer_pid)
1757 		return -ENODATA;
1758 
1759 	/* The use of PIDFD_STALE requires stashing of struct pid on pidfs
1760 	 * with pidfs_register_pid() and only AF_UNIX is prepared for this.
1761 	 */
1762 	if (sk_is_unix(sk))
1763 		flags |= PIDFD_STALE;
1764 	if (type == PIDTYPE_PID)
1765 		flags |= PIDFD_THREAD;
1766 
1767 	pidfd = pidfd_prepare(peer_pid, flags, &pidfd_file);
1768 	put_pid(peer_pid);
1769 	if (pidfd < 0)
1770 		return pidfd;
1771 
1772 	if (copy_to_sockptr(optval, &pidfd, len) ||
1773 	    copy_to_sockptr(optlen, &len, sizeof(int))) {
1774 		put_unused_fd(pidfd);
1775 		fput(pidfd_file);
1776 
1777 		return -EFAULT;
1778 	}
1779 
1780 	fd_install(pidfd, pidfd_file);
1781 	return 0;
1782 }
1783 
1784 int sk_getsockopt(struct sock *sk, int level, int optname,
1785 		  sockptr_t optval, sockptr_t optlen)
1786 {
1787 	struct socket *sock = sk->sk_socket;
1788 
1789 	union {
1790 		int val;
1791 		u64 val64;
1792 		unsigned long ulval;
1793 		struct linger ling;
1794 		struct old_timeval32 tm32;
1795 		struct __kernel_old_timeval tm;
1796 		struct  __kernel_sock_timeval stm;
1797 		struct sock_txtime txtime;
1798 		struct so_timestamping timestamping;
1799 	} v;
1800 
1801 	int lv = sizeof(int);
1802 	int len;
1803 
1804 	if (copy_from_sockptr(&len, optlen, sizeof(int)))
1805 		return -EFAULT;
1806 	if (len < 0)
1807 		return -EINVAL;
1808 
1809 	memset(&v, 0, sizeof(v));
1810 
1811 	switch (optname) {
1812 	case SO_DEBUG:
1813 		v.val = sock_flag(sk, SOCK_DBG);
1814 		break;
1815 
1816 	case SO_DONTROUTE:
1817 		v.val = sock_flag(sk, SOCK_LOCALROUTE);
1818 		break;
1819 
1820 	case SO_BROADCAST:
1821 		v.val = sock_flag(sk, SOCK_BROADCAST);
1822 		break;
1823 
1824 	case SO_SNDBUF:
1825 		v.val = READ_ONCE(sk->sk_sndbuf);
1826 		break;
1827 
1828 	case SO_RCVBUF:
1829 		v.val = READ_ONCE(sk->sk_rcvbuf);
1830 		break;
1831 
1832 	case SO_REUSEADDR:
1833 		v.val = sk->sk_reuse;
1834 		break;
1835 
1836 	case SO_REUSEPORT:
1837 		v.val = sk->sk_reuseport;
1838 		break;
1839 
1840 	case SO_KEEPALIVE:
1841 		v.val = sock_flag(sk, SOCK_KEEPOPEN);
1842 		break;
1843 
1844 	case SO_TYPE:
1845 		v.val = sk->sk_type;
1846 		break;
1847 
1848 	case SO_PROTOCOL:
1849 		v.val = sk->sk_protocol;
1850 		break;
1851 
1852 	case SO_DOMAIN:
1853 		v.val = sk->sk_family;
1854 		break;
1855 
1856 	case SO_ERROR:
1857 		v.val = -sock_error(sk);
1858 		if (v.val == 0)
1859 			v.val = xchg(&sk->sk_err_soft, 0);
1860 		break;
1861 
1862 	case SO_OOBINLINE:
1863 		v.val = sock_flag(sk, SOCK_URGINLINE);
1864 		break;
1865 
1866 	case SO_NO_CHECK:
1867 		v.val = sk->sk_no_check_tx;
1868 		break;
1869 
1870 	case SO_PRIORITY:
1871 		v.val = READ_ONCE(sk->sk_priority);
1872 		break;
1873 
1874 	case SO_LINGER:
1875 		lv		= sizeof(v.ling);
1876 		v.ling.l_onoff	= sock_flag(sk, SOCK_LINGER);
1877 		v.ling.l_linger	= READ_ONCE(sk->sk_lingertime) / HZ;
1878 		break;
1879 
1880 	case SO_BSDCOMPAT:
1881 		break;
1882 
1883 	case SO_TIMESTAMP_OLD:
1884 		v.val = sock_flag(sk, SOCK_RCVTSTAMP) &&
1885 				!sock_flag(sk, SOCK_TSTAMP_NEW) &&
1886 				!sock_flag(sk, SOCK_RCVTSTAMPNS);
1887 		break;
1888 
1889 	case SO_TIMESTAMPNS_OLD:
1890 		v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && !sock_flag(sk, SOCK_TSTAMP_NEW);
1891 		break;
1892 
1893 	case SO_TIMESTAMP_NEW:
1894 		v.val = sock_flag(sk, SOCK_RCVTSTAMP) && sock_flag(sk, SOCK_TSTAMP_NEW);
1895 		break;
1896 
1897 	case SO_TIMESTAMPNS_NEW:
1898 		v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && sock_flag(sk, SOCK_TSTAMP_NEW);
1899 		break;
1900 
1901 	case SO_TIMESTAMPING_OLD:
1902 	case SO_TIMESTAMPING_NEW:
1903 		lv = sizeof(v.timestamping);
1904 		/* For the later-added case SO_TIMESTAMPING_NEW: Be strict about only
1905 		 * returning the flags when they were set through the same option.
1906 		 * Don't change the beviour for the old case SO_TIMESTAMPING_OLD.
1907 		 */
1908 		if (optname == SO_TIMESTAMPING_OLD || sock_flag(sk, SOCK_TSTAMP_NEW)) {
1909 			v.timestamping.flags = READ_ONCE(sk->sk_tsflags);
1910 			v.timestamping.bind_phc = READ_ONCE(sk->sk_bind_phc);
1911 		}
1912 		break;
1913 
1914 	case SO_RCVTIMEO_OLD:
1915 	case SO_RCVTIMEO_NEW:
1916 		lv = sock_get_timeout(READ_ONCE(sk->sk_rcvtimeo), &v,
1917 				      SO_RCVTIMEO_OLD == optname);
1918 		break;
1919 
1920 	case SO_SNDTIMEO_OLD:
1921 	case SO_SNDTIMEO_NEW:
1922 		lv = sock_get_timeout(READ_ONCE(sk->sk_sndtimeo), &v,
1923 				      SO_SNDTIMEO_OLD == optname);
1924 		break;
1925 
1926 	case SO_RCVLOWAT:
1927 		v.val = READ_ONCE(sk->sk_rcvlowat);
1928 		break;
1929 
1930 	case SO_SNDLOWAT:
1931 		v.val = 1;
1932 		break;
1933 
1934 	case SO_PASSCRED:
1935 		if (!sk_may_scm_recv(sk))
1936 			return -EOPNOTSUPP;
1937 
1938 		v.val = sk->sk_scm_credentials;
1939 		break;
1940 
1941 	case SO_PASSPIDFD:
1942 		if (!sk_is_unix(sk))
1943 			return -EOPNOTSUPP;
1944 
1945 		v.val = sk->sk_scm_pidfd;
1946 		break;
1947 
1948 	case SO_PASSPIDFD_THREAD:
1949 		if (!sk_is_unix(sk))
1950 			return -EOPNOTSUPP;
1951 
1952 		v.val = sk->sk_scm_pidfd_thread;
1953 		break;
1954 
1955 	case SO_PASSRIGHTS:
1956 		if (!sk_is_unix(sk))
1957 			return -EOPNOTSUPP;
1958 
1959 		v.val = sk->sk_scm_rights;
1960 		break;
1961 
1962 	case SO_PEERCRED:
1963 	{
1964 		struct ucred peercred;
1965 		if (len > sizeof(peercred))
1966 			len = sizeof(peercred);
1967 
1968 		spin_lock(&sk->sk_peer_lock);
1969 		cred_to_ucred(sk->sk_peer_pid[PIDTYPE_TGID], sk->sk_peer_cred,
1970 			      &peercred);
1971 		spin_unlock(&sk->sk_peer_lock);
1972 
1973 		if (copy_to_sockptr(optval, &peercred, len))
1974 			return -EFAULT;
1975 		goto lenout;
1976 	}
1977 
1978 	case SO_PEERPIDFD:
1979 		return sk_getsockopt_peerpidfd(sk, PIDTYPE_TGID, optval, optlen, len);
1980 
1981 	case SO_PEERPIDFD_THREAD:
1982 		/* Only AF_UNIX records the peer's connecting thread. */
1983 		if (!sk_is_unix(sk))
1984 			return -EOPNOTSUPP;
1985 
1986 		return sk_getsockopt_peerpidfd(sk, PIDTYPE_PID, optval, optlen, len);
1987 
1988 	case SO_PEERGROUPS:
1989 	{
1990 		const struct cred *cred;
1991 		int ret, n;
1992 
1993 		cred = sk_get_peer_cred(sk);
1994 		if (!cred)
1995 			return -ENODATA;
1996 
1997 		n = cred->group_info->ngroups;
1998 		if (len < n * sizeof(gid_t)) {
1999 			len = n * sizeof(gid_t);
2000 			put_cred(cred);
2001 			return copy_to_sockptr(optlen, &len, sizeof(int)) ? -EFAULT : -ERANGE;
2002 		}
2003 		len = n * sizeof(gid_t);
2004 
2005 		ret = groups_to_user(optval, cred->group_info);
2006 		put_cred(cred);
2007 		if (ret)
2008 			return ret;
2009 		goto lenout;
2010 	}
2011 
2012 	case SO_PEERNAME:
2013 	{
2014 		struct sockaddr_storage address;
2015 
2016 		lv = READ_ONCE(sock->ops)->getname(sock, (struct sockaddr *)&address, 2);
2017 		if (lv < 0)
2018 			return -ENOTCONN;
2019 		if (lv < len)
2020 			return -EINVAL;
2021 		if (copy_to_sockptr(optval, &address, len))
2022 			return -EFAULT;
2023 		goto lenout;
2024 	}
2025 
2026 	/* Dubious BSD thing... Probably nobody even uses it, but
2027 	 * the UNIX standard wants it for whatever reason... -DaveM
2028 	 */
2029 	case SO_ACCEPTCONN:
2030 		v.val = sk->sk_state == TCP_LISTEN;
2031 		break;
2032 
2033 	case SO_PASSSEC:
2034 		if (!IS_ENABLED(CONFIG_SECURITY_NETWORK) || !sk_may_scm_recv(sk))
2035 			return -EOPNOTSUPP;
2036 
2037 		v.val = sk->sk_scm_security;
2038 		break;
2039 
2040 	case SO_PEERSEC:
2041 		return security_socket_getpeersec_stream(sock,
2042 							 optval, optlen, len);
2043 
2044 	case SO_MARK:
2045 		v.val = READ_ONCE(sk->sk_mark);
2046 		break;
2047 
2048 	case SO_RCVMARK:
2049 		v.val = sock_flag(sk, SOCK_RCVMARK);
2050 		break;
2051 
2052 	case SO_RCVPRIORITY:
2053 		v.val = sock_flag(sk, SOCK_RCVPRIORITY);
2054 		break;
2055 
2056 	case SO_RXQ_OVFL:
2057 		v.val = sock_flag(sk, SOCK_RXQ_OVFL);
2058 		break;
2059 
2060 	case SO_WIFI_STATUS:
2061 		v.val = sock_flag(sk, SOCK_WIFI_STATUS);
2062 		break;
2063 
2064 	case SO_PEEK_OFF:
2065 		if (!READ_ONCE(sock->ops)->set_peek_off)
2066 			return -EOPNOTSUPP;
2067 
2068 		v.val = READ_ONCE(sk->sk_peek_off);
2069 		break;
2070 	case SO_NOFCS:
2071 		v.val = sock_flag(sk, SOCK_NOFCS);
2072 		break;
2073 
2074 	case SO_BINDTODEVICE:
2075 		return sock_getbindtodevice(sk, optval, optlen, len);
2076 
2077 	case SO_GET_FILTER:
2078 		len = sk_get_filter(sk, optval, len);
2079 		if (len < 0)
2080 			return len;
2081 
2082 		goto lenout;
2083 
2084 	case SO_LOCK_FILTER:
2085 		v.val = sock_flag(sk, SOCK_FILTER_LOCKED);
2086 		break;
2087 
2088 	case SO_BPF_EXTENSIONS:
2089 		v.val = bpf_tell_extensions();
2090 		break;
2091 
2092 	case SO_SELECT_ERR_QUEUE:
2093 		v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE);
2094 		break;
2095 
2096 #ifdef CONFIG_NET_RX_BUSY_POLL
2097 	case SO_BUSY_POLL:
2098 		v.val = READ_ONCE(sk->sk_ll_usec);
2099 		break;
2100 	case SO_PREFER_BUSY_POLL:
2101 		v.val = READ_ONCE(sk->sk_prefer_busy_poll);
2102 		break;
2103 #endif
2104 
2105 	case SO_MAX_PACING_RATE:
2106 		/* The READ_ONCE() pair with the WRITE_ONCE() in sk_setsockopt() */
2107 		if (sizeof(v.ulval) != sizeof(v.val) && len >= sizeof(v.ulval)) {
2108 			lv = sizeof(v.ulval);
2109 			v.ulval = READ_ONCE(sk->sk_max_pacing_rate);
2110 		} else {
2111 			/* 32bit version */
2112 			v.val = min_t(unsigned long, ~0U,
2113 				      READ_ONCE(sk->sk_max_pacing_rate));
2114 		}
2115 		break;
2116 
2117 	case SO_INCOMING_CPU:
2118 		v.val = READ_ONCE(sk->sk_incoming_cpu);
2119 		break;
2120 
2121 	case SO_MEMINFO:
2122 	{
2123 		u32 meminfo[SK_MEMINFO_VARS];
2124 
2125 		sk_get_meminfo(sk, meminfo);
2126 
2127 		len = min_t(unsigned int, len, sizeof(meminfo));
2128 		if (copy_to_sockptr(optval, &meminfo, len))
2129 			return -EFAULT;
2130 
2131 		goto lenout;
2132 	}
2133 
2134 #ifdef CONFIG_NET_RX_BUSY_POLL
2135 	case SO_INCOMING_NAPI_ID:
2136 		v.val = READ_ONCE(sk->sk_napi_id);
2137 
2138 		/* aggregate non-NAPI IDs down to 0 */
2139 		if (!napi_id_valid(v.val))
2140 			v.val = 0;
2141 
2142 		break;
2143 #endif
2144 
2145 	case SO_COOKIE:
2146 		lv = sizeof(u64);
2147 		if (len < lv)
2148 			return -EINVAL;
2149 		v.val64 = sock_gen_cookie(sk);
2150 		break;
2151 
2152 	case SO_ZEROCOPY:
2153 		v.val = sock_flag(sk, SOCK_ZEROCOPY);
2154 		break;
2155 
2156 	case SO_TXTIME:
2157 		lv = sizeof(v.txtime);
2158 		v.txtime.clockid = sk->sk_clockid;
2159 		v.txtime.flags |= sk->sk_txtime_deadline_mode ?
2160 				  SOF_TXTIME_DEADLINE_MODE : 0;
2161 		v.txtime.flags |= sk->sk_txtime_report_errors ?
2162 				  SOF_TXTIME_REPORT_ERRORS : 0;
2163 		break;
2164 
2165 	case SO_BINDTOIFINDEX:
2166 		v.val = READ_ONCE(sk->sk_bound_dev_if);
2167 		break;
2168 
2169 	case SO_NETNS_COOKIE:
2170 		lv = sizeof(u64);
2171 		if (len != lv)
2172 			return -EINVAL;
2173 		v.val64 = sock_net(sk)->net_cookie;
2174 		break;
2175 
2176 	case SO_BUF_LOCK:
2177 		v.val = sk->sk_userlocks & SOCK_BUF_LOCK_MASK;
2178 		break;
2179 
2180 	case SO_RESERVE_MEM:
2181 		v.val = READ_ONCE(sk->sk_reserved_mem);
2182 		break;
2183 
2184 	case SO_TXREHASH:
2185 		if (!sk_is_tcp(sk))
2186 			return -EOPNOTSUPP;
2187 
2188 		/* Paired with WRITE_ONCE() in sk_setsockopt() */
2189 		v.val = READ_ONCE(sk->sk_txrehash);
2190 		break;
2191 
2192 	default:
2193 		/* We implement the SO_SNDLOWAT etc to not be settable
2194 		 * (1003.1g 7).
2195 		 */
2196 		return -ENOPROTOOPT;
2197 	}
2198 
2199 	if (len > lv)
2200 		len = lv;
2201 	if (copy_to_sockptr(optval, &v, len))
2202 		return -EFAULT;
2203 lenout:
2204 	if (copy_to_sockptr(optlen, &len, sizeof(int)))
2205 		return -EFAULT;
2206 	return 0;
2207 }
2208 
2209 /*
2210  * Initialize an sk_lock.
2211  *
2212  * (We also register the sk_lock with the lock validator.)
2213  */
2214 static inline void sock_lock_init(struct sock *sk)
2215 {
2216 	sk_owner_clear(sk);
2217 
2218 	if (sk->sk_kern_sock)
2219 		sock_lock_init_class_and_name(
2220 			sk,
2221 			af_family_kern_slock_key_strings[sk->sk_family],
2222 			af_family_kern_slock_keys + sk->sk_family,
2223 			af_family_kern_key_strings[sk->sk_family],
2224 			af_family_kern_keys + sk->sk_family);
2225 	else
2226 		sock_lock_init_class_and_name(
2227 			sk,
2228 			af_family_slock_key_strings[sk->sk_family],
2229 			af_family_slock_keys + sk->sk_family,
2230 			af_family_key_strings[sk->sk_family],
2231 			af_family_keys + sk->sk_family);
2232 }
2233 
2234 /*
2235  * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet,
2236  * even temporarily, because of RCU lookups. sk_node should also be left as is.
2237  * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end
2238  */
2239 static void sock_copy(struct sock *nsk, const struct sock *osk)
2240 {
2241 	const struct proto *prot = READ_ONCE(osk->sk_prot);
2242 #ifdef CONFIG_SECURITY_NETWORK
2243 	void *sptr = nsk->sk_security;
2244 #endif
2245 
2246 	/* If we move sk_tx_queue_mapping out of the private section,
2247 	 * we must check if sk_tx_queue_clear() is called after
2248 	 * sock_copy() in sk_clone_lock().
2249 	 */
2250 	BUILD_BUG_ON(offsetof(struct sock, sk_tx_queue_mapping) <
2251 		     offsetof(struct sock, sk_dontcopy_begin) ||
2252 		     offsetof(struct sock, sk_tx_queue_mapping) >=
2253 		     offsetof(struct sock, sk_dontcopy_end));
2254 
2255 	memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin));
2256 
2257 	unsafe_memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end,
2258 		      prot->obj_size - offsetof(struct sock, sk_dontcopy_end),
2259 		      /* alloc is larger than struct, see sk_prot_alloc() */);
2260 
2261 #ifdef CONFIG_SECURITY_NETWORK
2262 	nsk->sk_security = sptr;
2263 	security_sk_clone(osk, nsk);
2264 #endif
2265 }
2266 
2267 static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority,
2268 		int family)
2269 {
2270 	struct sock *sk;
2271 	struct kmem_cache *slab;
2272 
2273 	slab = prot->slab;
2274 	if (slab != NULL) {
2275 		sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO);
2276 		if (!sk)
2277 			return sk;
2278 		if (want_init_on_alloc(priority))
2279 			sk_prot_clear_nulls(sk, prot->obj_size);
2280 	} else
2281 		sk = kmalloc(prot->obj_size, priority);
2282 
2283 	if (sk != NULL) {
2284 		if (security_sk_alloc(sk, family, priority))
2285 			goto out_free;
2286 
2287 		if (!try_module_get(prot->owner))
2288 			goto out_free_sec;
2289 	}
2290 
2291 	return sk;
2292 
2293 out_free_sec:
2294 	security_sk_free(sk);
2295 out_free:
2296 	if (slab != NULL)
2297 		kmem_cache_free(slab, sk);
2298 	else
2299 		kfree(sk);
2300 	return NULL;
2301 }
2302 
2303 static void sk_prot_free(struct proto *prot, struct sock *sk)
2304 {
2305 	struct kmem_cache *slab;
2306 	struct module *owner;
2307 
2308 	owner = prot->owner;
2309 	slab = prot->slab;
2310 
2311 	cgroup_sk_free(&sk->sk_cgrp_data);
2312 	mem_cgroup_sk_free(sk);
2313 	security_sk_free(sk);
2314 
2315 	sk_owner_put(sk);
2316 
2317 	if (slab != NULL)
2318 		kmem_cache_free(slab, sk);
2319 	else
2320 		kfree(sk);
2321 	module_put(owner);
2322 }
2323 
2324 /**
2325  *	sk_alloc - All socket objects are allocated here
2326  *	@net: the applicable net namespace
2327  *	@family: protocol family
2328  *	@priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
2329  *	@prot: struct proto associated with this new sock instance
2330  *	@kern: is this to be a kernel socket?
2331  */
2332 struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
2333 		      struct proto *prot, int kern)
2334 {
2335 	struct sock *sk;
2336 
2337 	sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family);
2338 	if (sk) {
2339 		sk->sk_family = family;
2340 		/*
2341 		 * See comment in struct sock definition to understand
2342 		 * why we need sk_prot_creator -acme
2343 		 */
2344 		sk->sk_prot = sk->sk_prot_creator = prot;
2345 
2346 		if (READ_ONCE(net->core.sysctl_bypass_prot_mem))
2347 			sk->sk_bypass_prot_mem = 1;
2348 
2349 		sk->sk_kern_sock = kern;
2350 		sock_lock_init(sk);
2351 
2352 		sk->sk_net_refcnt = kern ? 0 : 1;
2353 		if (likely(sk->sk_net_refcnt)) {
2354 			get_net_track(net, &sk->ns_tracker, priority);
2355 			sock_inuse_add(net, 1);
2356 		} else {
2357 			net_passive_inc(net);
2358 			__netns_tracker_alloc(net, &sk->ns_tracker,
2359 					      false, priority);
2360 		}
2361 
2362 		sock_net_set(sk, net);
2363 		refcount_set(&sk->sk_wmem_alloc, SK_WMEM_ALLOC_BIAS);
2364 
2365 		mem_cgroup_sk_alloc(sk);
2366 		cgroup_sk_alloc(&sk->sk_cgrp_data);
2367 		sock_update_classid(&sk->sk_cgrp_data);
2368 		sock_update_netprioidx(&sk->sk_cgrp_data);
2369 		sk_tx_queue_clear(sk);
2370 	}
2371 
2372 	return sk;
2373 }
2374 EXPORT_SYMBOL(sk_alloc);
2375 
2376 /* Sockets having SOCK_RCU_FREE will call this function after one RCU
2377  * grace period. This is the case for UDP sockets and TCP listeners.
2378  */
2379 static void __sk_destruct(struct rcu_head *head)
2380 {
2381 	struct sock *sk = container_of(head, struct sock, sk_rcu);
2382 	struct net *net = sock_net(sk);
2383 	struct sk_filter *filter;
2384 
2385 	if (sk->sk_destruct)
2386 		sk->sk_destruct(sk);
2387 
2388 	filter = rcu_dereference_check(sk->sk_filter,
2389 				       refcount_read(&sk->sk_wmem_alloc) == 0);
2390 	if (filter) {
2391 		sk_filter_uncharge(sk, filter);
2392 		RCU_INIT_POINTER(sk->sk_filter, NULL);
2393 	}
2394 
2395 	sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP);
2396 
2397 #ifdef CONFIG_BPF_SYSCALL
2398 	bpf_sk_storage_free(sk);
2399 #endif
2400 
2401 	if (atomic_read(&sk->sk_omem_alloc))
2402 		pr_debug("%s: optmem leakage (%d bytes) detected\n",
2403 			 __func__, atomic_read(&sk->sk_omem_alloc));
2404 
2405 	if (sk->sk_frag.page) {
2406 		put_page(sk->sk_frag.page);
2407 		sk->sk_frag.page = NULL;
2408 	}
2409 
2410 	/* We do not need to acquire sk->sk_peer_lock, we are the last user. */
2411 	put_cred(sk->sk_peer_cred);
2412 	put_pids(sk->sk_peer_pid);
2413 
2414 	if (likely(sk->sk_net_refcnt)) {
2415 		put_net_track(net, &sk->ns_tracker);
2416 	} else {
2417 		__netns_tracker_free(net, &sk->ns_tracker, false);
2418 		net_passive_dec(net);
2419 	}
2420 	sk_prot_free(sk->sk_prot_creator, sk);
2421 }
2422 
2423 void sk_net_refcnt_upgrade(struct sock *sk)
2424 {
2425 	struct net *net = sock_net(sk);
2426 
2427 	WARN_ON_ONCE(sk->sk_net_refcnt);
2428 	__netns_tracker_free(net, &sk->ns_tracker, false);
2429 	net_passive_dec(net);
2430 	sk->sk_net_refcnt = 1;
2431 	get_net_track(net, &sk->ns_tracker, GFP_KERNEL);
2432 	sock_inuse_add(net, 1);
2433 }
2434 EXPORT_SYMBOL_GPL(sk_net_refcnt_upgrade);
2435 
2436 void sk_destruct(struct sock *sk)
2437 {
2438 	bool use_call_rcu = sock_flag(sk, SOCK_RCU_FREE);
2439 
2440 	if (rcu_access_pointer(sk->sk_reuseport_cb)) {
2441 		reuseport_detach_sock(sk);
2442 		use_call_rcu = true;
2443 	}
2444 
2445 	if (use_call_rcu)
2446 		call_rcu(&sk->sk_rcu, __sk_destruct);
2447 	else
2448 		__sk_destruct(&sk->sk_rcu);
2449 }
2450 
2451 static void __sk_free(struct sock *sk)
2452 {
2453 	if (likely(sk->sk_net_refcnt))
2454 		sock_inuse_add(sock_net(sk), -1);
2455 
2456 	if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk)))
2457 		sock_diag_broadcast_destroy(sk);
2458 	else
2459 		sk_destruct(sk);
2460 }
2461 
2462 void sk_free(struct sock *sk)
2463 {
2464 	/*
2465 	 * We subtract one from sk_wmem_alloc and can know if
2466 	 * some packets are still in some tx queue.
2467 	 * If not null, sock_wfree() will call __sk_free(sk) later
2468 	 */
2469 	if (refcount_dec_and_test(&sk->sk_wmem_alloc))
2470 		__sk_free(sk);
2471 }
2472 EXPORT_SYMBOL(sk_free);
2473 
2474 static void sk_init_common(struct sock *sk)
2475 {
2476 	skb_queue_head_init(&sk->sk_receive_queue);
2477 	skb_queue_head_init(&sk->sk_write_queue);
2478 	skb_queue_head_init(&sk->sk_error_queue);
2479 
2480 	rwlock_init(&sk->sk_callback_lock);
2481 	lockdep_set_class_and_name(&sk->sk_receive_queue.lock,
2482 			af_rlock_keys + sk->sk_family,
2483 			af_family_rlock_key_strings[sk->sk_family]);
2484 	lockdep_set_class_and_name(&sk->sk_write_queue.lock,
2485 			af_wlock_keys + sk->sk_family,
2486 			af_family_wlock_key_strings[sk->sk_family]);
2487 	lockdep_set_class_and_name(&sk->sk_error_queue.lock,
2488 			af_elock_keys + sk->sk_family,
2489 			af_family_elock_key_strings[sk->sk_family]);
2490 	if (sk->sk_kern_sock)
2491 		lockdep_set_class_and_name(&sk->sk_callback_lock,
2492 			af_kern_callback_keys + sk->sk_family,
2493 			af_family_kern_clock_key_strings[sk->sk_family]);
2494 	else
2495 		lockdep_set_class_and_name(&sk->sk_callback_lock,
2496 			af_callback_keys + sk->sk_family,
2497 			af_family_clock_key_strings[sk->sk_family]);
2498 }
2499 
2500 /**
2501  * sk_clone - clone a socket
2502  * @sk: the socket to clone
2503  * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
2504  * @lock: if true, lock the cloned sk
2505  *
2506  * If @lock is true, the clone is locked by bh_lock_sock(), and
2507  * caller must unlock socket even in error path by bh_unlock_sock().
2508  */
2509 struct sock *sk_clone(const struct sock *sk, const gfp_t priority,
2510 		      bool lock)
2511 {
2512 	struct proto *prot = READ_ONCE(sk->sk_prot);
2513 	struct sk_filter *filter;
2514 	bool is_charged = true;
2515 	struct sock *newsk;
2516 
2517 	newsk = sk_prot_alloc(prot, priority, sk->sk_family);
2518 	if (!newsk)
2519 		goto out;
2520 
2521 	sock_copy(newsk, sk);
2522 
2523 	newsk->sk_prot_creator = prot;
2524 #ifdef CONFIG_BPF_SYSCALL
2525 	RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL);
2526 #endif
2527 #if IS_ENABLED(CONFIG_INET_PSP)
2528 	DEBUG_NET_WARN_ON_ONCE(rcu_access_pointer(sk->psp_assoc));
2529 #endif
2530 
2531 	/* SANITY */
2532 	if (likely(newsk->sk_net_refcnt)) {
2533 		get_net_track(sock_net(newsk), &newsk->ns_tracker, priority);
2534 		sock_inuse_add(sock_net(newsk), 1);
2535 	} else {
2536 		/* Kernel sockets are not elevating the struct net refcount.
2537 		 * Instead, use a tracker to more easily detect if a layer
2538 		 * is not properly dismantling its kernel sockets at netns
2539 		 * destroy time.
2540 		 */
2541 		net_passive_inc(sock_net(newsk));
2542 		__netns_tracker_alloc(sock_net(newsk), &newsk->ns_tracker,
2543 				      false, priority);
2544 	}
2545 
2546 	sk_node_init(&newsk->sk_node);
2547 	sock_lock_init(newsk);
2548 
2549 	if (lock)
2550 		bh_lock_sock(newsk);
2551 
2552 	newsk->sk_backlog.head	= newsk->sk_backlog.tail = NULL;
2553 	newsk->sk_backlog.len = 0;
2554 
2555 	atomic_set(&newsk->sk_rmem_alloc, 0);
2556 
2557 	refcount_set(&newsk->sk_wmem_alloc, SK_WMEM_ALLOC_BIAS);
2558 
2559 	atomic_set(&newsk->sk_omem_alloc, 0);
2560 	sk_init_common(newsk);
2561 
2562 	newsk->sk_dst_cache	= NULL;
2563 	newsk->sk_dst_pending_confirm = 0;
2564 	newsk->sk_wmem_queued	= 0;
2565 	newsk->sk_forward_alloc = 0;
2566 	newsk->sk_reserved_mem  = 0;
2567 	DEBUG_NET_WARN_ON_ONCE(newsk->sk_drop_counters);
2568 	sk_drops_reset(newsk);
2569 	newsk->sk_send_head	= NULL;
2570 	newsk->sk_userlocks	= sk->sk_userlocks & ~SOCK_BINDPORT_LOCK;
2571 	atomic_set(&newsk->sk_zckey, 0);
2572 
2573 	sock_reset_flag(newsk, SOCK_DONE);
2574 
2575 #ifdef CONFIG_MEMCG
2576 	/* sk->sk_memcg will be populated at accept() time */
2577 	newsk->sk_memcg = NULL;
2578 #endif
2579 
2580 	cgroup_sk_clone(&newsk->sk_cgrp_data);
2581 
2582 	RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL);
2583 
2584 	if (sock_needs_netstamp(sk) && newsk->sk_flags & SK_FLAGS_TIMESTAMP)
2585 		net_enable_timestamp();
2586 
2587 	rcu_read_lock();
2588 	filter = rcu_dereference(sk->sk_filter);
2589 	if (filter != NULL)
2590 		/* though it's an empty new sock, the charging may fail
2591 		 * if sysctl_optmem_max was changed between creation of
2592 		 * original socket and cloning
2593 		 */
2594 		is_charged = sk_filter_charge(newsk, filter);
2595 	RCU_INIT_POINTER(newsk->sk_filter, filter);
2596 	rcu_read_unlock();
2597 
2598 	if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) {
2599 		/* We need to make sure that we don't uncharge the new
2600 		 * socket if we couldn't charge it in the first place
2601 		 * as otherwise we uncharge the parent's filter.
2602 		 */
2603 		if (!is_charged)
2604 			RCU_INIT_POINTER(newsk->sk_filter, NULL);
2605 
2606 		goto free;
2607 	}
2608 
2609 	if (bpf_sk_storage_clone(sk, newsk))
2610 		goto free;
2611 
2612 	/* Clear sk_user_data if parent had the pointer tagged
2613 	 * as not suitable for copying when cloning.
2614 	 */
2615 	if (sk_user_data_is_nocopy(newsk))
2616 		newsk->sk_user_data = NULL;
2617 
2618 	newsk->sk_err	   = 0;
2619 	newsk->sk_err_soft = 0;
2620 	newsk->sk_priority = 0;
2621 	newsk->sk_incoming_cpu = raw_smp_processor_id();
2622 
2623 	/* Before updating sk_refcnt, we must commit prior changes to memory
2624 	 * (Documentation/RCU/rculist_nulls.rst for details)
2625 	 */
2626 	smp_wmb();
2627 	refcount_set(&newsk->sk_refcnt, 2);
2628 
2629 	sk_set_socket(newsk, NULL);
2630 	sk_tx_queue_clear(newsk);
2631 	sk_rx_queue_clear(newsk);
2632 	RCU_INIT_POINTER(newsk->sk_wq, NULL);
2633 
2634 	if (newsk->sk_prot->sockets_allocated)
2635 		sk_sockets_allocated_inc(newsk);
2636 out:
2637 	return newsk;
2638 free:
2639 	/* It is still raw copy of parent, so invalidate
2640 	 * destructor and make plain sk_free()
2641 	 */
2642 	newsk->sk_destruct = NULL;
2643 	if (lock)
2644 		bh_unlock_sock(newsk);
2645 	sk_free(newsk);
2646 	newsk = NULL;
2647 	goto out;
2648 }
2649 EXPORT_SYMBOL_GPL(sk_clone);
2650 
2651 static u32 sk_dst_gso_max_size(struct sock *sk, const struct net_device *dev)
2652 {
2653 	bool is_ipv6 = false;
2654 	u32 max_size;
2655 
2656 #if IS_ENABLED(CONFIG_IPV6)
2657 	is_ipv6 = (sk->sk_family == AF_INET6 &&
2658 		   !ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr));
2659 #endif
2660 	/* pairs with the WRITE_ONCE() in netif_set_gso(_ipv4)_max_size() */
2661 	max_size = is_ipv6 ? READ_ONCE(dev->gso_max_size) :
2662 			READ_ONCE(dev->gso_ipv4_max_size);
2663 	if (max_size > GSO_LEGACY_MAX_SIZE && !sk_is_tcp(sk))
2664 		max_size = GSO_LEGACY_MAX_SIZE;
2665 
2666 	return max_size - (MAX_TCP_HEADER + 1);
2667 }
2668 
2669 void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
2670 {
2671 	const struct net_device *dev;
2672 	u32 max_segs = 1;
2673 
2674 	rcu_read_lock();
2675 	dev = dst_dev_rcu(dst);
2676 	sk->sk_route_caps = dev->features;
2677 	if (sk_is_tcp(sk)) {
2678 		struct inet_connection_sock *icsk = inet_csk(sk);
2679 
2680 		sk->sk_route_caps |= NETIF_F_GSO;
2681 		icsk->icsk_ack.dst_quick_ack = dst_metric(dst, RTAX_QUICKACK);
2682 	}
2683 	if (sk->sk_route_caps & NETIF_F_GSO)
2684 		sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE;
2685 	if (unlikely(sk->sk_gso_disabled))
2686 		sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
2687 	if (sk_can_gso(sk)) {
2688 		if (dst->header_len && !xfrm_dst_offload_ok(dst)) {
2689 			sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
2690 		} else {
2691 			sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;
2692 			sk->sk_gso_max_size = sk_dst_gso_max_size(sk, dev);
2693 			/* pairs with the WRITE_ONCE() in netif_set_gso_max_segs() */
2694 			max_segs = max_t(u32, READ_ONCE(dev->gso_max_segs), 1);
2695 		}
2696 	}
2697 	sk->sk_gso_max_segs = max_segs;
2698 	sk_dst_set(sk, dst);
2699 	rcu_read_unlock();
2700 }
2701 EXPORT_SYMBOL_GPL(sk_setup_caps);
2702 
2703 bool sk_has_decrypt_user(const struct sock *sk)
2704 {
2705 	return psp_sk_assoc(sk) ||
2706 	       (sk_is_inet(sk) && inet_csk_has_ulp(sk)); /* for tls */
2707 }
2708 
2709 /*
2710  *	Simple resource managers for sockets.
2711  */
2712 
2713 
2714 /*
2715  * Write buffer destructor automatically called from kfree_skb.
2716  */
2717 void sock_wfree(struct sk_buff *skb)
2718 {
2719 	unsigned int len = skb->truesize;
2720 	struct sock *sk = skb->sk;
2721 	bool free;
2722 	int old;
2723 
2724 	if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) {
2725 		void (*sk_write_space)(struct sock *sk);
2726 
2727 		sk_write_space = READ_ONCE(sk->sk_write_space);
2728 
2729 		if (sock_flag(sk, SOCK_RCU_FREE) &&
2730 		    sk_write_space == sock_def_write_space) {
2731 			rcu_read_lock();
2732 			free = __refcount_sub_and_test(len, &sk->sk_wmem_alloc,
2733 						       &old);
2734 			sock_def_write_space_wfree(sk, old - len);
2735 			rcu_read_unlock();
2736 			if (unlikely(free))
2737 				__sk_free(sk);
2738 			return;
2739 		}
2740 
2741 		/*
2742 		 * Keep a reference on sk_wmem_alloc, this will be released
2743 		 * after sk_write_space() call
2744 		 */
2745 		WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc));
2746 		sk_write_space(sk);
2747 		len = 1;
2748 	}
2749 	/*
2750 	 * if sk_wmem_alloc reaches 0, we must finish what sk_free()
2751 	 * could not do because of in-flight packets
2752 	 */
2753 	if (refcount_sub_and_test(len, &sk->sk_wmem_alloc))
2754 		__sk_free(sk);
2755 }
2756 EXPORT_SYMBOL(sock_wfree);
2757 
2758 /* This variant of sock_wfree() is used by TCP,
2759  * since it sets SOCK_USE_WRITE_QUEUE.
2760  */
2761 #ifdef CONFIG_INET
2762 void __sock_wfree(struct sk_buff *skb)
2763 {
2764 	struct sock *sk = skb->sk;
2765 
2766 	if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc))
2767 		__sk_free(sk);
2768 }
2769 EXPORT_SYMBOL_GPL(__sock_wfree);
2770 #endif
2771 
2772 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
2773 {
2774 	int old_wmem;
2775 
2776 	skb_orphan(skb);
2777 #ifdef CONFIG_INET
2778 	if (unlikely(!sk_fullsock(sk)))
2779 		return skb_set_owner_edemux(skb, sk);
2780 #endif
2781 	skb->sk = sk;
2782 	skb->destructor = sock_wfree;
2783 	skb_set_hash_from_sk(skb, sk);
2784 	/*
2785 	 * We used to take a refcount on sk, but following operation
2786 	 * is enough to guarantee sk_free() won't free this sock until
2787 	 * all in-flight packets are completed
2788 	 */
2789 	__refcount_add(skb->truesize, &sk->sk_wmem_alloc, &old_wmem);
2790 
2791 	/* (old_wmem == SK_WMEM_ALLOC_BIAS) if no other TX packet for this socket
2792 	 * is in a host queue (qdisc, NIC queue).
2793 	 * Set skb->ooo_okay so that netdev_pick_tx() can choose a TX queue
2794 	 * based on XPS for better performance.
2795 	 * Otherwise clear ooo_okay to not risk Out Of Order delivery.
2796 	 */
2797 	skb->ooo_okay = (old_wmem == SK_WMEM_ALLOC_BIAS);
2798 }
2799 EXPORT_SYMBOL(skb_set_owner_w);
2800 
2801 static bool can_skb_orphan_partial(const struct sk_buff *skb)
2802 {
2803 	/* Drivers depend on in-order delivery for crypto offload,
2804 	 * partial orphan breaks out-of-order-OK logic.
2805 	 */
2806 	if (skb_is_decrypted(skb))
2807 		return false;
2808 
2809 	return (skb->destructor == sock_wfree ||
2810 		(IS_ENABLED(CONFIG_INET) && skb->destructor == tcp_wfree));
2811 }
2812 
2813 /* This helper is used by netem, as it can hold packets in its
2814  * delay queue. We want to allow the owner socket to send more
2815  * packets, as if they were already TX completed by a typical driver.
2816  * But we also want to keep skb->sk set because some packet schedulers
2817  * rely on it (sch_fq for example).
2818  */
2819 void skb_orphan_partial(struct sk_buff *skb)
2820 {
2821 	if (skb_is_tcp_pure_ack(skb))
2822 		return;
2823 
2824 	if (can_skb_orphan_partial(skb) && skb_set_owner_sk_safe(skb, skb->sk))
2825 		return;
2826 
2827 	skb_orphan(skb);
2828 }
2829 EXPORT_SYMBOL(skb_orphan_partial);
2830 
2831 /*
2832  * Read buffer destructor automatically called from kfree_skb.
2833  */
2834 void sock_rfree(struct sk_buff *skb)
2835 {
2836 	struct sock *sk = skb->sk;
2837 	unsigned int len = skb->truesize;
2838 
2839 	atomic_sub(len, &sk->sk_rmem_alloc);
2840 	sk_mem_uncharge(sk, len);
2841 }
2842 EXPORT_SYMBOL(sock_rfree);
2843 
2844 /*
2845  * Buffer destructor for skbs that are not used directly in read or write
2846  * path, e.g. for error handler skbs. Automatically called from kfree_skb.
2847  */
2848 void sock_efree(struct sk_buff *skb)
2849 {
2850 	sock_put(skb->sk);
2851 }
2852 EXPORT_SYMBOL(sock_efree);
2853 
2854 /* Buffer destructor for prefetch/receive path where reference count may
2855  * not be held, e.g. for listen sockets.
2856  */
2857 #ifdef CONFIG_INET
2858 void sock_pfree(struct sk_buff *skb)
2859 {
2860 	struct sock *sk = skb->sk;
2861 
2862 	if (!sk_is_refcounted(sk))
2863 		return;
2864 
2865 	if (sk->sk_state == TCP_NEW_SYN_RECV && inet_reqsk(sk)->syncookie) {
2866 		inet_reqsk(sk)->rsk_listener = NULL;
2867 		reqsk_free(inet_reqsk(sk));
2868 		return;
2869 	}
2870 
2871 	sock_gen_put(sk);
2872 }
2873 EXPORT_SYMBOL(sock_pfree);
2874 #endif /* CONFIG_INET */
2875 
2876 /*
2877  * Allocate a skb from the socket's send buffer.
2878  */
2879 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
2880 			     gfp_t priority)
2881 {
2882 	if (force ||
2883 	    refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) {
2884 		struct sk_buff *skb = alloc_skb(size, priority);
2885 
2886 		if (skb) {
2887 			skb_set_owner_w(skb, sk);
2888 			return skb;
2889 		}
2890 	}
2891 	return NULL;
2892 }
2893 EXPORT_SYMBOL(sock_wmalloc);
2894 
2895 static void sock_ofree(struct sk_buff *skb)
2896 {
2897 	struct sock *sk = skb->sk;
2898 
2899 	atomic_sub(skb->truesize, &sk->sk_omem_alloc);
2900 }
2901 
2902 struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size,
2903 			     gfp_t priority)
2904 {
2905 	struct sk_buff *skb;
2906 
2907 	/* small safe race: SKB_TRUESIZE may differ from final skb->truesize */
2908 	if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) >
2909 	    READ_ONCE(sock_net(sk)->core.sysctl_optmem_max))
2910 		return NULL;
2911 
2912 	skb = alloc_skb(size, priority);
2913 	if (!skb)
2914 		return NULL;
2915 
2916 	atomic_add(skb->truesize, &sk->sk_omem_alloc);
2917 	skb->sk = sk;
2918 	skb->destructor = sock_ofree;
2919 	return skb;
2920 }
2921 
2922 /*
2923  * Allocate a memory block from the socket's option memory buffer.
2924  */
2925 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority)
2926 {
2927 	int optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max);
2928 
2929 	if ((unsigned int)size <= optmem_max &&
2930 	    atomic_read(&sk->sk_omem_alloc) + size < optmem_max) {
2931 		void *mem;
2932 		/* First do the add, to avoid the race if kmalloc
2933 		 * might sleep.
2934 		 */
2935 		atomic_add(size, &sk->sk_omem_alloc);
2936 		mem = kmalloc(size, priority);
2937 		if (mem)
2938 			return mem;
2939 		atomic_sub(size, &sk->sk_omem_alloc);
2940 	}
2941 	return NULL;
2942 }
2943 EXPORT_SYMBOL(sock_kmalloc);
2944 
2945 /*
2946  * Duplicate the input "src" memory block using the socket's
2947  * option memory buffer.
2948  */
2949 void *sock_kmemdup(struct sock *sk, const void *src,
2950 		   int size, gfp_t priority)
2951 {
2952 	void *mem;
2953 
2954 	mem = sock_kmalloc(sk, size, priority);
2955 	if (mem)
2956 		memcpy(mem, src, size);
2957 	return mem;
2958 }
2959 EXPORT_SYMBOL(sock_kmemdup);
2960 
2961 /* Free an option memory block. Note, we actually want the inline
2962  * here as this allows gcc to detect the nullify and fold away the
2963  * condition entirely.
2964  */
2965 static inline void __sock_kfree_s(struct sock *sk, void *mem, int size,
2966 				  const bool nullify)
2967 {
2968 	if (WARN_ON_ONCE(!mem))
2969 		return;
2970 	if (nullify)
2971 		kfree_sensitive(mem);
2972 	else
2973 		kfree(mem);
2974 	atomic_sub(size, &sk->sk_omem_alloc);
2975 }
2976 
2977 void sock_kfree_s(struct sock *sk, void *mem, int size)
2978 {
2979 	__sock_kfree_s(sk, mem, size, false);
2980 }
2981 EXPORT_SYMBOL(sock_kfree_s);
2982 
2983 void sock_kzfree_s(struct sock *sk, void *mem, int size)
2984 {
2985 	__sock_kfree_s(sk, mem, size, true);
2986 }
2987 EXPORT_SYMBOL(sock_kzfree_s);
2988 
2989 /**
2990  *	sk_set_nospace - tell the transport a writer is waiting for space
2991  *	@sk: socket
2992  *
2993  *	Must be called before the final check of the available send space,
2994  *	so that the transport can not miss the request and forget to call
2995  *	sk->sk_write_space() once space is available again.
2996  */
2997 void sk_set_nospace(struct sock *sk)
2998 {
2999 	struct socket *sock = sk->sk_socket;
3000 
3001 	if (!sock)
3002 		return;
3003 	/* Set SOCK_NOSPACE before tp->tcp_nospace (paired with
3004 	 * sk_clear_nospace() clearing tp->tcp_nospace before SOCK_NOSPACE)
3005 	 * so a concurrent clear cannot leave SOCK_NOSPACE set with
3006 	 * tp->tcp_nospace cleared.
3007 	 */
3008 	set_bit(SOCK_NOSPACE, &sock->flags);
3009 	tcp_set_nospace(sk);
3010 }
3011 EXPORT_SYMBOL(sk_set_nospace);
3012 
3013 /**
3014  *	sk_clear_nospace - tell the transport no writer is waiting for space
3015  *	@sk: socket
3016  *
3017  *	Called from ->sk_write_space() handlers, once send space has been
3018  *	made available to writers.
3019  */
3020 void sk_clear_nospace(struct sock *sk)
3021 {
3022 	struct socket *sock = sk->sk_socket;
3023 
3024 	if (!sock)
3025 		return;
3026 	tcp_clear_nospace(sk);
3027 	clear_bit(SOCK_NOSPACE, &sock->flags);
3028 }
3029 EXPORT_SYMBOL(sk_clear_nospace);
3030 
3031 /* It is almost wait_for_tcp_memory minus release_sock/lock_sock.
3032    I think, these locks should be removed for datagram sockets.
3033  */
3034 static long sock_wait_for_wmem(struct sock *sk, long timeo)
3035 {
3036 	DEFINE_WAIT(wait);
3037 
3038 	sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
3039 	for (;;) {
3040 		if (!timeo)
3041 			break;
3042 		if (signal_pending(current))
3043 			break;
3044 		sk_set_nospace(sk);
3045 		prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
3046 		if (refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf))
3047 			break;
3048 		if (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN)
3049 			break;
3050 		if (READ_ONCE(sk->sk_err))
3051 			break;
3052 		timeo = schedule_timeout(timeo);
3053 	}
3054 	finish_wait(sk_sleep(sk), &wait);
3055 	return timeo;
3056 }
3057 
3058 
3059 /*
3060  *	Generic send/receive buffer handlers
3061  */
3062 
3063 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
3064 				     unsigned long data_len, int noblock,
3065 				     int *errcode, int max_page_order)
3066 {
3067 	struct sk_buff *skb;
3068 	long timeo;
3069 	int err;
3070 
3071 	timeo = sock_sndtimeo(sk, noblock);
3072 	for (;;) {
3073 		err = sock_error(sk);
3074 		if (err != 0)
3075 			goto failure;
3076 
3077 		err = -EPIPE;
3078 		if (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN)
3079 			goto failure;
3080 
3081 		if (sk_wmem_alloc_get(sk) < READ_ONCE(sk->sk_sndbuf))
3082 			break;
3083 
3084 		sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
3085 		sk_set_nospace(sk);
3086 		err = -EAGAIN;
3087 		if (!timeo)
3088 			goto failure;
3089 		if (signal_pending(current))
3090 			goto interrupted;
3091 		timeo = sock_wait_for_wmem(sk, timeo);
3092 	}
3093 	skb = alloc_skb_with_frags(header_len, data_len, max_page_order,
3094 				   errcode, sk->sk_allocation);
3095 	if (skb)
3096 		skb_set_owner_w(skb, sk);
3097 	return skb;
3098 
3099 interrupted:
3100 	err = sock_intr_errno(timeo);
3101 failure:
3102 	*errcode = err;
3103 	return NULL;
3104 }
3105 EXPORT_SYMBOL(sock_alloc_send_pskb);
3106 
3107 int __sock_cmsg_send(struct sock *sk, struct cmsghdr *cmsg,
3108 		     struct sockcm_cookie *sockc)
3109 {
3110 	u32 tsflags;
3111 
3112 	BUILD_BUG_ON(SOF_TIMESTAMPING_LAST == (1 << 31));
3113 
3114 	switch (cmsg->cmsg_type) {
3115 	case SO_MARK:
3116 		if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) &&
3117 		    !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
3118 			return -EPERM;
3119 		if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
3120 			return -EINVAL;
3121 		sockc->mark = *(u32 *)CMSG_DATA(cmsg);
3122 		break;
3123 	case SO_TIMESTAMPING_OLD:
3124 	case SO_TIMESTAMPING_NEW:
3125 		if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
3126 			return -EINVAL;
3127 
3128 		tsflags = *(u32 *)CMSG_DATA(cmsg);
3129 		if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK)
3130 			return -EINVAL;
3131 
3132 		sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK;
3133 		sockc->tsflags |= tsflags;
3134 		break;
3135 	case SCM_TXTIME:
3136 	{
3137 		ktime_t tmin;
3138 		u64 txtime;
3139 
3140 		if (!sock_flag(sk, SOCK_TXTIME))
3141 			return -EINVAL;
3142 		if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64)))
3143 			return -EINVAL;
3144 
3145 		txtime = get_unaligned((u64 *)CMSG_DATA(cmsg));
3146 
3147 		/* Allow sending without a delivery time: zero special case */
3148 		if (!txtime) {
3149 			sockc->transmit_time = 0;
3150 			break;
3151 		}
3152 
3153 		switch (sk->sk_clockid) {
3154 		case CLOCK_MONOTONIC:
3155 			tmin = 1;
3156 			break;
3157 		case CLOCK_REALTIME:
3158 			tmin = max(ktime_mono_to_real(0), 1);
3159 			break;
3160 		case CLOCK_TAI:
3161 			tmin = max(ktime_mono_to_any(0, TK_OFFS_TAI), 1);
3162 			break;
3163 		default:
3164 			tmin = 1;
3165 			WARN_ON_ONCE(1);
3166 			break;
3167 		}
3168 
3169 		sockc->transmit_time = max_t(ktime_t, txtime, tmin);
3170 		break;
3171 	}
3172 	case SCM_TS_OPT_ID:
3173 		if (sk_is_tcp(sk))
3174 			return -EINVAL;
3175 		tsflags = READ_ONCE(sk->sk_tsflags);
3176 		if (!(tsflags & SOF_TIMESTAMPING_OPT_ID))
3177 			return -EINVAL;
3178 		if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
3179 			return -EINVAL;
3180 		sockc->ts_opt_id = *(u32 *)CMSG_DATA(cmsg);
3181 		sockc->tsflags |= SOCKCM_FLAG_TS_OPT_ID;
3182 		break;
3183 	/* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */
3184 	case SCM_RIGHTS:
3185 	case SCM_CREDENTIALS:
3186 		break;
3187 	case SO_PRIORITY:
3188 		if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
3189 			return -EINVAL;
3190 		if (!sk_set_prio_allowed(sk, *(u32 *)CMSG_DATA(cmsg)))
3191 			return -EPERM;
3192 		sockc->priority = *(u32 *)CMSG_DATA(cmsg);
3193 		break;
3194 	case SCM_DEVMEM_DMABUF:
3195 		if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
3196 			return -EINVAL;
3197 		sockc->dmabuf_id = *(u32 *)CMSG_DATA(cmsg);
3198 		break;
3199 	default:
3200 		return -EINVAL;
3201 	}
3202 	return 0;
3203 }
3204 EXPORT_SYMBOL(__sock_cmsg_send);
3205 
3206 int sock_cmsg_send(struct sock *sk, struct msghdr *msg,
3207 		   struct sockcm_cookie *sockc)
3208 {
3209 	struct cmsghdr *cmsg;
3210 	int ret;
3211 
3212 	for_each_cmsghdr(cmsg, msg) {
3213 		if (!CMSG_OK(msg, cmsg))
3214 			return -EINVAL;
3215 		if (cmsg->cmsg_level != SOL_SOCKET)
3216 			continue;
3217 		ret = __sock_cmsg_send(sk, cmsg, sockc);
3218 		if (ret)
3219 			return ret;
3220 	}
3221 	return 0;
3222 }
3223 EXPORT_SYMBOL(sock_cmsg_send);
3224 
3225 static void sk_enter_memory_pressure(struct sock *sk)
3226 {
3227 	if (!sk->sk_prot->enter_memory_pressure)
3228 		return;
3229 
3230 	sk->sk_prot->enter_memory_pressure(sk);
3231 }
3232 
3233 static void sk_leave_memory_pressure(struct sock *sk)
3234 {
3235 	if (sk->sk_prot->leave_memory_pressure) {
3236 		INDIRECT_CALL_INET_1(sk->sk_prot->leave_memory_pressure,
3237 				     tcp_leave_memory_pressure, sk);
3238 	} else {
3239 		unsigned long *memory_pressure = sk->sk_prot->memory_pressure;
3240 
3241 		if (memory_pressure && READ_ONCE(*memory_pressure))
3242 			WRITE_ONCE(*memory_pressure, 0);
3243 	}
3244 }
3245 
3246 DEFINE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key);
3247 
3248 /**
3249  * skb_page_frag_refill - check that a page_frag contains enough room
3250  * @sz: minimum size of the fragment we want to get
3251  * @pfrag: pointer to page_frag
3252  * @gfp: priority for memory allocation
3253  *
3254  * Note: While this allocator tries to use high order pages, there is
3255  * no guarantee that allocations succeed. Therefore, @sz MUST be
3256  * less or equal than PAGE_SIZE.
3257  */
3258 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp)
3259 {
3260 	if (pfrag->page) {
3261 		if (page_ref_count(pfrag->page) == 1) {
3262 			pfrag->offset = 0;
3263 			return true;
3264 		}
3265 		if (pfrag->offset + sz <= pfrag->size)
3266 			return true;
3267 		put_page(pfrag->page);
3268 	}
3269 
3270 	pfrag->offset = 0;
3271 	if (SKB_FRAG_PAGE_ORDER &&
3272 	    !static_branch_unlikely(&net_high_order_alloc_disable_key)) {
3273 		/* Avoid direct reclaim but allow kswapd to wake */
3274 		pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) |
3275 					  __GFP_COMP | __GFP_NOWARN |
3276 					  __GFP_NORETRY,
3277 					  SKB_FRAG_PAGE_ORDER);
3278 		if (likely(pfrag->page)) {
3279 			pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER;
3280 			return true;
3281 		}
3282 	}
3283 	pfrag->page = alloc_page(gfp);
3284 	if (likely(pfrag->page)) {
3285 		pfrag->size = PAGE_SIZE;
3286 		return true;
3287 	}
3288 	return false;
3289 }
3290 EXPORT_SYMBOL(skb_page_frag_refill);
3291 
3292 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
3293 {
3294 	if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation)))
3295 		return true;
3296 
3297 	if (!sk->sk_bypass_prot_mem)
3298 		sk_enter_memory_pressure(sk);
3299 
3300 	sk_stream_moderate_sndbuf(sk);
3301 
3302 	return false;
3303 }
3304 EXPORT_SYMBOL(sk_page_frag_refill);
3305 
3306 static void __lock_sock(struct sock *sk)
3307 	__releases(&sk->sk_lock.slock)
3308 	__acquires(&sk->sk_lock.slock)
3309 {
3310 	DEFINE_WAIT(wait);
3311 
3312 	for (;;) {
3313 		prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait,
3314 					TASK_UNINTERRUPTIBLE);
3315 		spin_unlock_bh(&sk->sk_lock.slock);
3316 		schedule();
3317 		spin_lock_bh(&sk->sk_lock.slock);
3318 		if (!sock_owned_by_user(sk))
3319 			break;
3320 	}
3321 	finish_wait(&sk->sk_lock.wq, &wait);
3322 }
3323 
3324 void __release_sock(struct sock *sk)
3325 	__releases(&sk->sk_lock.slock)
3326 	__acquires(&sk->sk_lock.slock)
3327 {
3328 	struct sk_buff *skb, *next;
3329 	int nb = 0;
3330 
3331 	while ((skb = sk->sk_backlog.head) != NULL) {
3332 		sk->sk_backlog.head = sk->sk_backlog.tail = NULL;
3333 
3334 		spin_unlock_bh(&sk->sk_lock.slock);
3335 
3336 		do {
3337 			next = skb->next;
3338 			prefetch(next);
3339 			DEBUG_NET_WARN_ON_ONCE(skb_dst_is_noref(skb));
3340 			skb_mark_not_on_list(skb);
3341 			sk_backlog_rcv(sk, skb);
3342 
3343 			skb = next;
3344 
3345 			if (!(++nb & 15))
3346 				cond_resched();
3347 		} while (skb);
3348 
3349 		spin_lock_bh(&sk->sk_lock.slock);
3350 	}
3351 
3352 	/*
3353 	 * Doing the zeroing here guarantee we can not loop forever
3354 	 * while a wild producer attempts to flood us.
3355 	 */
3356 	sk->sk_backlog.len = 0;
3357 }
3358 
3359 void __sk_flush_backlog(struct sock *sk)
3360 {
3361 	spin_lock_bh(&sk->sk_lock.slock);
3362 	__release_sock(sk);
3363 
3364 	if (sk->sk_prot->release_cb)
3365 		INDIRECT_CALL_INET_1(sk->sk_prot->release_cb,
3366 				     tcp_release_cb, sk);
3367 
3368 	spin_unlock_bh(&sk->sk_lock.slock);
3369 }
3370 EXPORT_SYMBOL_GPL(__sk_flush_backlog);
3371 
3372 /**
3373  * sk_wait_data - wait for data to arrive at sk_receive_queue
3374  * @sk:    sock to wait on
3375  * @timeo: for how long
3376  * @skb:   last skb seen on sk_receive_queue
3377  *
3378  * Now socket state including sk->sk_err is changed only under lock,
3379  * hence we may omit checks after joining wait queue.
3380  * We check receive queue before schedule() only as optimization;
3381  * it is very likely that release_sock() added new data.
3382  */
3383 int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb)
3384 {
3385 	DEFINE_WAIT_FUNC(wait, woken_wake_function);
3386 	int rc;
3387 
3388 	add_wait_queue(sk_sleep(sk), &wait);
3389 	sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
3390 	rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait);
3391 	sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
3392 	remove_wait_queue(sk_sleep(sk), &wait);
3393 	return rc;
3394 }
3395 EXPORT_SYMBOL(sk_wait_data);
3396 
3397 /**
3398  *	__sk_mem_raise_allocated - increase memory_allocated
3399  *	@sk: socket
3400  *	@size: memory size to allocate
3401  *	@amt: pages to allocate
3402  *	@kind: allocation type
3403  *
3404  *	Similar to __sk_mem_schedule(), but does not update sk_forward_alloc.
3405  *
3406  *	Unlike the globally shared limits among the sockets under same protocol,
3407  *	consuming the budget of a memcg won't have direct effect on other ones.
3408  *	So be optimistic about memcg's tolerance, and leave the callers to decide
3409  *	whether or not to raise allocated through sk_under_memory_pressure() or
3410  *	its variants.
3411  */
3412 int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind)
3413 {
3414 	bool memcg_enabled = false, charged = false;
3415 	struct proto *prot = sk->sk_prot;
3416 	long allocated = 0;
3417 
3418 	if (!sk->sk_bypass_prot_mem) {
3419 		sk_memory_allocated_add(sk, amt);
3420 		allocated = sk_memory_allocated(sk);
3421 	}
3422 
3423 	if (mem_cgroup_sk_enabled(sk)) {
3424 		memcg_enabled = true;
3425 		charged = mem_cgroup_sk_charge(sk, amt, gfp_memcg_charge());
3426 		if (!charged)
3427 			goto suppress_allocation;
3428 	}
3429 
3430 	if (sk->sk_bypass_prot_mem)
3431 		return 1;
3432 
3433 	/* Under limit. */
3434 	if (allocated <= sk_prot_mem_limits(sk, 0)) {
3435 		sk_leave_memory_pressure(sk);
3436 		return 1;
3437 	}
3438 
3439 	/* Under pressure. */
3440 	if (allocated > sk_prot_mem_limits(sk, 1))
3441 		sk_enter_memory_pressure(sk);
3442 
3443 	/* Over hard limit. */
3444 	if (allocated > sk_prot_mem_limits(sk, 2))
3445 		goto suppress_allocation;
3446 
3447 	/* Guarantee minimum buffer size under pressure (either global
3448 	 * or memcg) to make sure features described in RFC 7323 (TCP
3449 	 * Extensions for High Performance) work properly.
3450 	 *
3451 	 * This rule does NOT stand when exceeds global or memcg's hard
3452 	 * limit, or else a DoS attack can be taken place by spawning
3453 	 * lots of sockets whose usage are under minimum buffer size.
3454 	 */
3455 	if (kind == SK_MEM_RECV) {
3456 		if (atomic_read(&sk->sk_rmem_alloc) < sk_get_rmem0(sk, prot))
3457 			return 1;
3458 
3459 	} else { /* SK_MEM_SEND */
3460 		int wmem0 = sk_get_wmem0(sk, prot);
3461 
3462 		if (sk->sk_type == SOCK_STREAM) {
3463 			if (sk->sk_wmem_queued < wmem0)
3464 				return 1;
3465 		} else if (refcount_read(&sk->sk_wmem_alloc) < wmem0) {
3466 				return 1;
3467 		}
3468 	}
3469 
3470 	if (sk_has_memory_pressure(sk)) {
3471 		u64 alloc;
3472 
3473 		/* The following 'average' heuristic is within the
3474 		 * scope of global accounting, so it only makes
3475 		 * sense for global memory pressure.
3476 		 */
3477 		if (!sk_under_global_memory_pressure(sk))
3478 			return 1;
3479 
3480 		/* Try to be fair among all the sockets under global
3481 		 * pressure by allowing the ones that below average
3482 		 * usage to raise.
3483 		 */
3484 		alloc = sk_sockets_allocated_read_positive(sk);
3485 		if (sk_prot_mem_limits(sk, 2) > alloc *
3486 		    sk_mem_pages(sk->sk_wmem_queued +
3487 				 atomic_read(&sk->sk_rmem_alloc) +
3488 				 sk->sk_forward_alloc))
3489 			return 1;
3490 	}
3491 
3492 suppress_allocation:
3493 
3494 	if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) {
3495 		sk_stream_moderate_sndbuf(sk);
3496 
3497 		/* Fail only if socket is _under_ its sndbuf.
3498 		 * In this case we cannot block, so that we have to fail.
3499 		 */
3500 		if (sk->sk_wmem_queued + size >= sk->sk_sndbuf) {
3501 			/* Force charge with __GFP_NOFAIL */
3502 			if (memcg_enabled && !charged)
3503 				mem_cgroup_sk_charge(sk, amt,
3504 						     gfp_memcg_charge() | __GFP_NOFAIL);
3505 			return 1;
3506 		}
3507 	}
3508 
3509 	trace_sock_exceed_buf_limit(sk, prot, allocated, kind);
3510 
3511 	if (!sk->sk_bypass_prot_mem)
3512 		sk_memory_allocated_sub(sk, amt);
3513 
3514 	if (charged)
3515 		mem_cgroup_sk_uncharge(sk, amt);
3516 
3517 	return 0;
3518 }
3519 
3520 /**
3521  *	__sk_mem_schedule - increase sk_forward_alloc and memory_allocated
3522  *	@sk: socket
3523  *	@size: memory size to allocate
3524  *	@kind: allocation type
3525  *
3526  *	If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means
3527  *	rmem allocation. This function assumes that protocols which have
3528  *	memory_pressure use sk_wmem_queued as write buffer accounting.
3529  */
3530 int __sk_mem_schedule(struct sock *sk, int size, int kind)
3531 {
3532 	int ret, amt = sk_mem_pages(size);
3533 
3534 	sk_forward_alloc_add(sk, amt << PAGE_SHIFT);
3535 	ret = __sk_mem_raise_allocated(sk, size, amt, kind);
3536 	if (!ret)
3537 		sk_forward_alloc_add(sk, -(amt << PAGE_SHIFT));
3538 	return ret;
3539 }
3540 EXPORT_SYMBOL(__sk_mem_schedule);
3541 
3542 /**
3543  *	__sk_mem_reduce_allocated - reclaim memory_allocated
3544  *	@sk: socket
3545  *	@amount: number of quanta
3546  *
3547  *	Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc
3548  */
3549 void __sk_mem_reduce_allocated(struct sock *sk, int amount)
3550 {
3551 	if (mem_cgroup_sk_enabled(sk))
3552 		mem_cgroup_sk_uncharge(sk, amount);
3553 
3554 	if (sk->sk_bypass_prot_mem)
3555 		return;
3556 
3557 	sk_memory_allocated_sub(sk, amount);
3558 
3559 	if (sk_under_global_memory_pressure(sk) &&
3560 	    (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)))
3561 		sk_leave_memory_pressure(sk);
3562 }
3563 
3564 /**
3565  *	__sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated
3566  *	@sk: socket
3567  *	@amount: number of bytes (rounded down to a PAGE_SIZE multiple)
3568  */
3569 void __sk_mem_reclaim(struct sock *sk, int amount)
3570 {
3571 	amount >>= PAGE_SHIFT;
3572 	sk_forward_alloc_add(sk, -(amount << PAGE_SHIFT));
3573 	__sk_mem_reduce_allocated(sk, amount);
3574 }
3575 EXPORT_SYMBOL(__sk_mem_reclaim);
3576 
3577 void __sk_charge(struct sock *sk, gfp_t gfp)
3578 {
3579 	int amt;
3580 
3581 	gfp |= __GFP_NOFAIL;
3582 	if (mem_cgroup_from_sk(sk)) {
3583 		/* The socket has not been accepted yet, no need
3584 		 * to look at newsk->sk_wmem_queued.
3585 		 */
3586 		amt = sk_mem_pages(sk->sk_forward_alloc +
3587 				   atomic_read(&sk->sk_rmem_alloc));
3588 		if (amt)
3589 			mem_cgroup_sk_charge(sk, amt, gfp);
3590 	}
3591 
3592 	kmem_cache_charge(sk, gfp);
3593 }
3594 
3595 int sk_set_peek_off(struct sock *sk, int val)
3596 {
3597 	WRITE_ONCE(sk->sk_peek_off, val);
3598 	return 0;
3599 }
3600 EXPORT_SYMBOL_GPL(sk_set_peek_off);
3601 
3602 /*
3603  * Set of default routines for initialising struct proto_ops when
3604  * the protocol does not support a particular function. In certain
3605  * cases where it makes no sense for a protocol to have a "do nothing"
3606  * function, some default processing is provided.
3607  */
3608 
3609 int sock_no_bind(struct socket *sock, struct sockaddr_unsized *saddr, int len)
3610 {
3611 	return -EOPNOTSUPP;
3612 }
3613 EXPORT_SYMBOL(sock_no_bind);
3614 
3615 int sock_no_connect(struct socket *sock, struct sockaddr_unsized *saddr,
3616 		    int len, int flags)
3617 {
3618 	return -EOPNOTSUPP;
3619 }
3620 EXPORT_SYMBOL(sock_no_connect);
3621 
3622 int sock_no_socketpair(struct socket *sock1, struct socket *sock2)
3623 {
3624 	return -EOPNOTSUPP;
3625 }
3626 EXPORT_SYMBOL(sock_no_socketpair);
3627 
3628 int sock_no_accept(struct socket *sock, struct socket *newsock,
3629 		   struct proto_accept_arg *arg)
3630 {
3631 	return -EOPNOTSUPP;
3632 }
3633 EXPORT_SYMBOL(sock_no_accept);
3634 
3635 int sock_no_getname(struct socket *sock, struct sockaddr *saddr,
3636 		    int peer)
3637 {
3638 	return -EOPNOTSUPP;
3639 }
3640 EXPORT_SYMBOL(sock_no_getname);
3641 
3642 int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
3643 {
3644 	return -EOPNOTSUPP;
3645 }
3646 EXPORT_SYMBOL(sock_no_ioctl);
3647 
3648 int sock_no_listen(struct socket *sock, int backlog)
3649 {
3650 	return -EOPNOTSUPP;
3651 }
3652 EXPORT_SYMBOL(sock_no_listen);
3653 
3654 int sock_no_shutdown(struct socket *sock, int how)
3655 {
3656 	return -EOPNOTSUPP;
3657 }
3658 EXPORT_SYMBOL(sock_no_shutdown);
3659 
3660 int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len)
3661 {
3662 	return -EOPNOTSUPP;
3663 }
3664 EXPORT_SYMBOL(sock_no_sendmsg);
3665 
3666 int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len)
3667 {
3668 	return -EOPNOTSUPP;
3669 }
3670 EXPORT_SYMBOL(sock_no_sendmsg_locked);
3671 
3672 int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len,
3673 		    int flags)
3674 {
3675 	return -EOPNOTSUPP;
3676 }
3677 EXPORT_SYMBOL(sock_no_recvmsg);
3678 
3679 int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma)
3680 {
3681 	/* Mirror missing mmap method error code */
3682 	return -ENODEV;
3683 }
3684 EXPORT_SYMBOL(sock_no_mmap);
3685 
3686 /*
3687  * When a file is received (via SCM_RIGHTS, etc), we must bump the
3688  * various sock-based usage counts.
3689  */
3690 void __receive_sock(struct file *file)
3691 {
3692 	struct socket *sock;
3693 
3694 	sock = sock_from_file(file);
3695 	if (sock) {
3696 		sock_update_netprioidx(&sock->sk->sk_cgrp_data);
3697 		sock_update_classid(&sock->sk->sk_cgrp_data);
3698 	}
3699 }
3700 
3701 /*
3702  *	Default Socket Callbacks
3703  */
3704 
3705 static void sock_def_wakeup(struct sock *sk)
3706 {
3707 	struct socket_wq *wq;
3708 
3709 	rcu_read_lock();
3710 	wq = rcu_dereference(sk->sk_wq);
3711 	if (skwq_has_sleeper(wq))
3712 		wake_up_interruptible_all(&wq->wait);
3713 	rcu_read_unlock();
3714 }
3715 
3716 static void sock_def_error_report(struct sock *sk)
3717 {
3718 	struct socket_wq *wq;
3719 
3720 	rcu_read_lock();
3721 	wq = rcu_dereference(sk->sk_wq);
3722 	if (skwq_has_sleeper(wq))
3723 		wake_up_interruptible_poll(&wq->wait, EPOLLERR);
3724 	sk_wake_async_rcu(sk, SOCK_WAKE_IO, POLL_ERR);
3725 	rcu_read_unlock();
3726 }
3727 
3728 void sock_def_readable(struct sock *sk)
3729 {
3730 	struct socket_wq *wq;
3731 
3732 	trace_sk_data_ready(sk);
3733 
3734 	rcu_read_lock();
3735 	wq = rcu_dereference(sk->sk_wq);
3736 	if (skwq_has_sleeper(wq))
3737 		wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI |
3738 						EPOLLRDNORM | EPOLLRDBAND);
3739 	sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_IN);
3740 	rcu_read_unlock();
3741 }
3742 
3743 static void sock_def_write_space(struct sock *sk)
3744 {
3745 	struct socket_wq *wq;
3746 
3747 	rcu_read_lock();
3748 
3749 	/* Do not wake up a writer until he can make "significant"
3750 	 * progress.  --DaveM
3751 	 */
3752 	if (sock_writeable(sk)) {
3753 		wq = rcu_dereference(sk->sk_wq);
3754 		if (skwq_has_sleeper(wq))
3755 			wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
3756 						EPOLLWRNORM | EPOLLWRBAND);
3757 
3758 		/* Should agree with poll, otherwise some programs break */
3759 		sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
3760 	}
3761 
3762 	rcu_read_unlock();
3763 }
3764 
3765 /* An optimised version of sock_def_write_space(), should only be called
3766  * for SOCK_RCU_FREE sockets under RCU read section and after putting
3767  * ->sk_wmem_alloc.
3768  */
3769 static void sock_def_write_space_wfree(struct sock *sk, int wmem_alloc)
3770 {
3771 	/* Do not wake up a writer until he can make "significant"
3772 	 * progress.  --DaveM
3773 	 */
3774 	if (__sock_writeable(sk, wmem_alloc)) {
3775 		struct socket_wq *wq = rcu_dereference(sk->sk_wq);
3776 
3777 		/* rely on refcount_sub from sock_wfree() */
3778 		smp_mb__after_atomic();
3779 		if (wq && waitqueue_active(&wq->wait))
3780 			wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
3781 						EPOLLWRNORM | EPOLLWRBAND);
3782 
3783 		/* Should agree with poll, otherwise some programs break */
3784 		sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
3785 	}
3786 }
3787 
3788 static void sock_def_destruct(struct sock *sk)
3789 {
3790 }
3791 
3792 void sk_send_sigurg(struct sock *sk)
3793 {
3794 	if (sk->sk_socket && sk->sk_socket->file)
3795 		if (send_sigurg(sk->sk_socket->file))
3796 			sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI);
3797 }
3798 EXPORT_SYMBOL(sk_send_sigurg);
3799 
3800 void sk_reset_timer(struct sock *sk, struct timer_list* timer,
3801 		    unsigned long expires)
3802 {
3803 	if (!mod_timer(timer, expires))
3804 		sock_hold(sk);
3805 }
3806 EXPORT_SYMBOL(sk_reset_timer);
3807 
3808 void sk_stop_timer(struct sock *sk, struct timer_list* timer)
3809 {
3810 	if (timer_delete(timer))
3811 		__sock_put(sk);
3812 }
3813 EXPORT_SYMBOL(sk_stop_timer);
3814 
3815 void sk_stop_timer_sync(struct sock *sk, struct timer_list *timer)
3816 {
3817 	if (timer_delete_sync(timer))
3818 		__sock_put(sk);
3819 }
3820 EXPORT_SYMBOL(sk_stop_timer_sync);
3821 
3822 void sock_init_data_uid(struct socket *sock, struct sock *sk, kuid_t uid)
3823 {
3824 	sk_init_common(sk);
3825 	sk->sk_send_head	=	NULL;
3826 
3827 	timer_setup(&sk->sk_timer, NULL, 0);
3828 
3829 	sk->sk_allocation	=	GFP_KERNEL;
3830 	sk->sk_rcvbuf		=	READ_ONCE(sysctl_rmem_default);
3831 	sk->sk_sndbuf		=	READ_ONCE(sysctl_wmem_default);
3832 	sk->sk_state		=	TCP_CLOSE;
3833 	sk->sk_use_task_frag	=	true;
3834 	sk_set_socket(sk, sock);
3835 
3836 	sock_set_flag(sk, SOCK_ZAPPED);
3837 
3838 	if (sock) {
3839 		sk->sk_type	=	sock->type;
3840 		RCU_INIT_POINTER(sk->sk_wq, &sock->wq);
3841 		sock->sk	=	sk;
3842 	} else {
3843 		RCU_INIT_POINTER(sk->sk_wq, NULL);
3844 	}
3845 	sk->sk_uid	=	uid;
3846 
3847 	sk->sk_state_change	=	sock_def_wakeup;
3848 	sk->sk_data_ready	=	sock_def_readable;
3849 	sk->sk_write_space	=	sock_def_write_space;
3850 	sk->sk_error_report	=	sock_def_error_report;
3851 	sk->sk_destruct		=	sock_def_destruct;
3852 
3853 	sk->sk_frag.page	=	NULL;
3854 	sk->sk_frag.offset	=	0;
3855 	sk->sk_peek_off		=	-1;
3856 
3857 	memset(sk->sk_peer_pid, 0, sizeof(sk->sk_peer_pid));
3858 	sk->sk_peer_cred	=	NULL;
3859 	spin_lock_init(&sk->sk_peer_lock);
3860 
3861 	sk->sk_write_pending	=	0;
3862 	sk->sk_rcvlowat		=	1;
3863 	sk->sk_rcvtimeo		=	MAX_SCHEDULE_TIMEOUT;
3864 	sk->sk_sndtimeo		=	MAX_SCHEDULE_TIMEOUT;
3865 
3866 	sk->sk_stamp = SK_DEFAULT_STAMP;
3867 #if BITS_PER_LONG==32
3868 	seqlock_init(&sk->sk_stamp_seq);
3869 #endif
3870 	atomic_set(&sk->sk_zckey, 0);
3871 
3872 #ifdef CONFIG_NET_RX_BUSY_POLL
3873 	sk->sk_napi_id		=	0;
3874 	sk->sk_ll_usec		=	READ_ONCE(sysctl_net_busy_read);
3875 #endif
3876 
3877 	sk->sk_max_pacing_rate = ~0UL;
3878 	sk->sk_pacing_rate = ~0UL;
3879 	WRITE_ONCE(sk->sk_pacing_shift, 10);
3880 	sk->sk_incoming_cpu = -1;
3881 
3882 	sk_rx_queue_clear(sk);
3883 	/*
3884 	 * Before updating sk_refcnt, we must commit prior changes to memory
3885 	 * (Documentation/RCU/rculist_nulls.rst for details)
3886 	 */
3887 	smp_wmb();
3888 	refcount_set(&sk->sk_refcnt, 1);
3889 	sk_drops_reset(sk);
3890 }
3891 EXPORT_SYMBOL(sock_init_data_uid);
3892 
3893 void sock_init_data(struct socket *sock, struct sock *sk)
3894 {
3895 	kuid_t uid = sock ?
3896 		SOCK_INODE(sock)->i_uid :
3897 		make_kuid(sock_net(sk)->user_ns, 0);
3898 
3899 	sock_init_data_uid(sock, sk, uid);
3900 }
3901 EXPORT_SYMBOL(sock_init_data);
3902 
3903 void noinline lock_sock_nested(struct sock *sk, int subclass)
3904 {
3905 	/* The sk_lock has mutex_lock() semantics here. */
3906 	mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_);
3907 
3908 	might_sleep();
3909 #ifdef CONFIG_64BIT
3910 	if (sizeof(struct slock_owned) == sizeof(long)) {
3911 		socket_lock_t tmp = {
3912 			.slock = __SPIN_LOCK_UNLOCKED(tmp.slock),
3913 			.owned = 1,
3914 		};
3915 		socket_lock_t old = {
3916 			.slock = __SPIN_LOCK_UNLOCKED(old.slock),
3917 			.owned = 0,
3918 		};
3919 
3920 		if (likely(try_cmpxchg(&sk->sk_lock.combined,
3921 				       &old.combined, tmp.combined)))
3922 			return;
3923 	}
3924 #endif
3925 	spin_lock_bh(&sk->sk_lock.slock);
3926 	if (unlikely(sock_owned_by_user_nocheck(sk)))
3927 		__lock_sock(sk);
3928 	sk->sk_lock.owned = 1;
3929 	spin_unlock_bh(&sk->sk_lock.slock);
3930 }
3931 EXPORT_SYMBOL(lock_sock_nested);
3932 
3933 void release_sock(struct sock *sk)
3934 {
3935 	spin_lock_bh(&sk->sk_lock.slock);
3936 
3937 	if (unlikely(sk->sk_backlog.tail))
3938 		__release_sock(sk);
3939 
3940 	if (sk->sk_prot->release_cb) {
3941 		if (!tcp_release_cb_cond(sk))
3942 			sk->sk_prot->release_cb(sk);
3943 	}
3944 	sock_release_ownership(sk);
3945 	if (unlikely(waitqueue_active(&sk->sk_lock.wq)))
3946 		wake_up(&sk->sk_lock.wq);
3947 
3948 	spin_unlock_bh(&sk->sk_lock.slock);
3949 }
3950 EXPORT_SYMBOL(release_sock);
3951 
3952 bool __lock_sock_fast(struct sock *sk) __acquires(&sk->sk_lock.slock)
3953 {
3954 	might_sleep();
3955 	spin_lock_bh(&sk->sk_lock.slock);
3956 
3957 	if (likely(!sock_owned_by_user_nocheck(sk))) {
3958 		/*
3959 		 * Fast path return with bottom halves disabled and
3960 		 * sock::sk_lock.slock held.
3961 		 *
3962 		 * The 'mutex' is not contended and holding
3963 		 * sock::sk_lock.slock prevents all other lockers to
3964 		 * proceed so the corresponding unlock_sock_fast() can
3965 		 * avoid the slow path of release_sock() completely and
3966 		 * just release slock.
3967 		 *
3968 		 * From a semantical POV this is equivalent to 'acquiring'
3969 		 * the 'mutex', hence the corresponding lockdep
3970 		 * mutex_release() has to happen in the fast path of
3971 		 * unlock_sock_fast().
3972 		 */
3973 		return false;
3974 	}
3975 
3976 	__lock_sock(sk);
3977 	sk->sk_lock.owned = 1;
3978 	__acquire(&sk->sk_lock.slock);
3979 	spin_unlock_bh(&sk->sk_lock.slock);
3980 	return true;
3981 }
3982 EXPORT_SYMBOL(__lock_sock_fast);
3983 
3984 int sock_gettstamp(struct socket *sock, void __user *userstamp,
3985 		   bool timeval, bool time32)
3986 {
3987 	struct sock *sk = sock->sk;
3988 	struct timespec64 ts;
3989 
3990 	/* sk->sk_flags must only be changed under the socket lock,
3991 	 * because sock_set_flag() uses non atomic operations.
3992 	 */
3993 	if (!sock_flag(sk, SOCK_TIMESTAMP)) {
3994 		lock_sock(sk);
3995 		sock_enable_timestamp(sk, SOCK_TIMESTAMP);
3996 		release_sock(sk);
3997 	}
3998 	ts = ktime_to_timespec64(sock_read_timestamp(sk));
3999 	if (ts.tv_sec == -1)
4000 		return -ENOENT;
4001 	if (ts.tv_sec == 0) {
4002 		ktime_t kt = ktime_get_real();
4003 		sock_write_timestamp(sk, kt);
4004 		ts = ktime_to_timespec64(kt);
4005 	}
4006 
4007 	if (timeval)
4008 		ts.tv_nsec /= 1000;
4009 
4010 #ifdef CONFIG_COMPAT_32BIT_TIME
4011 	if (time32)
4012 		return put_old_timespec32(&ts, userstamp);
4013 #endif
4014 #ifdef CONFIG_SPARC64
4015 	/* beware of padding in sparc64 timeval */
4016 	if (timeval && !in_compat_syscall()) {
4017 		struct __kernel_old_timeval __user tv = {
4018 			.tv_sec = ts.tv_sec,
4019 			.tv_usec = ts.tv_nsec,
4020 		};
4021 		if (copy_to_user(userstamp, &tv, sizeof(tv)))
4022 			return -EFAULT;
4023 		return 0;
4024 	}
4025 #endif
4026 	return put_timespec64(&ts, userstamp);
4027 }
4028 EXPORT_SYMBOL(sock_gettstamp);
4029 
4030 void sock_enable_timestamp(struct sock *sk, enum sock_flags flag)
4031 {
4032 	if (!sock_flag(sk, flag)) {
4033 		unsigned long previous_flags = sk->sk_flags;
4034 
4035 		sock_set_flag(sk, flag);
4036 		/*
4037 		 * we just set one of the two flags which require net
4038 		 * time stamping, but time stamping might have been on
4039 		 * already because of the other one
4040 		 */
4041 		if (sock_needs_netstamp(sk) &&
4042 		    !(previous_flags & SK_FLAGS_TIMESTAMP))
4043 			net_enable_timestamp();
4044 	}
4045 }
4046 
4047 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len,
4048 		       int level, int type)
4049 {
4050 	struct sock_extended_err ee;
4051 	struct sk_buff *skb;
4052 	int copied, err;
4053 
4054 	err = -EAGAIN;
4055 	skb = sock_dequeue_err_skb(sk);
4056 	if (skb == NULL)
4057 		goto out;
4058 
4059 	copied = skb->len;
4060 	if (copied > len) {
4061 		msg->msg_flags |= MSG_TRUNC;
4062 		copied = len;
4063 	}
4064 	err = skb_copy_datagram_msg(skb, 0, msg, copied);
4065 	if (err)
4066 		goto out_free_skb;
4067 
4068 	sock_recv_timestamp(msg, sk, skb);
4069 
4070 	/* We must use a bounce buffer for CONFIG_HARDENED_USERCOPY=y */
4071 	ee = SKB_EXT_ERR(skb)->ee;
4072 	put_cmsg(msg, level, type, sizeof(ee), &ee);
4073 
4074 	msg->msg_flags |= MSG_ERRQUEUE;
4075 	err = copied;
4076 
4077 out_free_skb:
4078 	kfree_skb(skb);
4079 out:
4080 	return err;
4081 }
4082 EXPORT_SYMBOL(sock_recv_errqueue);
4083 
4084 /*
4085  *	Get a socket option on an socket.
4086  *
4087  *	FIX: POSIX 1003.1g is very ambiguous here. It states that
4088  *	asynchronous errors should be reported by getsockopt. We assume
4089  *	this means if you specify SO_ERROR (otherwise what is the point of it).
4090  */
4091 int sock_common_getsockopt(struct socket *sock, int level, int optname,
4092 			   char __user *optval, int __user *optlen)
4093 {
4094 	struct sock *sk = sock->sk;
4095 
4096 	return READ_ONCE(sk->sk_prot)->getsockopt(sk, level, optname, optval, optlen);
4097 }
4098 EXPORT_SYMBOL(sock_common_getsockopt);
4099 
4100 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
4101 			int flags)
4102 {
4103 	struct sock *sk = sock->sk;
4104 
4105 	return sk->sk_prot->recvmsg(sk, msg, size, flags);
4106 }
4107 EXPORT_SYMBOL(sock_common_recvmsg);
4108 
4109 /*
4110  *	Set socket options on an inet socket.
4111  */
4112 int sock_common_setsockopt(struct socket *sock, int level, int optname,
4113 			   sockptr_t optval, unsigned int optlen)
4114 {
4115 	struct sock *sk = sock->sk;
4116 
4117 	return READ_ONCE(sk->sk_prot)->setsockopt(sk, level, optname, optval, optlen);
4118 }
4119 EXPORT_SYMBOL(sock_common_setsockopt);
4120 
4121 void sk_common_release(struct sock *sk)
4122 {
4123 	if (sk->sk_prot->destroy)
4124 		sk->sk_prot->destroy(sk);
4125 
4126 	/*
4127 	 * Observation: when sk_common_release is called, processes have
4128 	 * no access to socket. But net still has.
4129 	 * Step one, detach it from networking:
4130 	 *
4131 	 * A. Remove from hash tables.
4132 	 */
4133 
4134 	sk->sk_prot->unhash(sk);
4135 
4136 	/*
4137 	 * In this point socket cannot receive new packets, but it is possible
4138 	 * that some packets are in flight because some CPU runs receiver and
4139 	 * did hash table lookup before we unhashed socket. They will achieve
4140 	 * receive queue and will be purged by socket destructor.
4141 	 *
4142 	 * Also we still have packets pending on receive queue and probably,
4143 	 * our own packets waiting in device queues. sock_destroy will drain
4144 	 * receive queue, but transmitted packets will delay socket destruction
4145 	 * until the last reference will be released.
4146 	 */
4147 
4148 	sock_orphan(sk);
4149 
4150 	xfrm_sk_free_policy(sk);
4151 
4152 	sock_put(sk);
4153 }
4154 EXPORT_SYMBOL(sk_common_release);
4155 
4156 void sk_get_meminfo(const struct sock *sk, u32 *mem)
4157 {
4158 	memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS);
4159 
4160 	mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk);
4161 	mem[SK_MEMINFO_RCVBUF] = READ_ONCE(sk->sk_rcvbuf);
4162 	mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk);
4163 	mem[SK_MEMINFO_SNDBUF] = READ_ONCE(sk->sk_sndbuf);
4164 	mem[SK_MEMINFO_FWD_ALLOC] = READ_ONCE(sk->sk_forward_alloc);
4165 	mem[SK_MEMINFO_WMEM_QUEUED] = READ_ONCE(sk->sk_wmem_queued);
4166 	mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc);
4167 	mem[SK_MEMINFO_BACKLOG] = READ_ONCE(sk->sk_backlog.len);
4168 	mem[SK_MEMINFO_DROPS] = sk_drops_read(sk);
4169 }
4170 
4171 #ifdef CONFIG_PROC_FS
4172 static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR);
4173 
4174 int sock_prot_inuse_get(struct net *net, struct proto *prot)
4175 {
4176 	int cpu, idx = prot->inuse_idx;
4177 	int res = 0;
4178 
4179 	for_each_possible_cpu(cpu)
4180 		res += per_cpu_ptr(net->core.prot_inuse, cpu)->val[idx];
4181 
4182 	return res >= 0 ? res : 0;
4183 }
4184 EXPORT_SYMBOL_GPL(sock_prot_inuse_get);
4185 
4186 int sock_inuse_get(struct net *net)
4187 {
4188 	int cpu, res = 0;
4189 
4190 	for_each_possible_cpu(cpu)
4191 		res += per_cpu_ptr(net->core.prot_inuse, cpu)->all;
4192 
4193 	return res;
4194 }
4195 
4196 EXPORT_SYMBOL_GPL(sock_inuse_get);
4197 
4198 static int __net_init sock_inuse_init_net(struct net *net)
4199 {
4200 	net->core.prot_inuse = alloc_percpu(struct prot_inuse);
4201 	if (net->core.prot_inuse == NULL)
4202 		return -ENOMEM;
4203 	return 0;
4204 }
4205 
4206 static void __net_exit sock_inuse_exit_net(struct net *net)
4207 {
4208 	free_percpu(net->core.prot_inuse);
4209 }
4210 
4211 static struct pernet_operations net_inuse_ops = {
4212 	.init = sock_inuse_init_net,
4213 	.exit = sock_inuse_exit_net,
4214 };
4215 
4216 static __init int net_inuse_init(void)
4217 {
4218 	if (register_pernet_subsys(&net_inuse_ops))
4219 		panic("Cannot initialize net inuse counters");
4220 
4221 	return 0;
4222 }
4223 
4224 core_initcall(net_inuse_init);
4225 
4226 static int assign_proto_idx(struct proto *prot)
4227 {
4228 	prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR);
4229 
4230 	if (unlikely(prot->inuse_idx == PROTO_INUSE_NR)) {
4231 		pr_err("PROTO_INUSE_NR exhausted\n");
4232 		return -ENOSPC;
4233 	}
4234 
4235 	set_bit(prot->inuse_idx, proto_inuse_idx);
4236 	return 0;
4237 }
4238 
4239 static void release_proto_idx(struct proto *prot)
4240 {
4241 	if (prot->inuse_idx != PROTO_INUSE_NR)
4242 		clear_bit(prot->inuse_idx, proto_inuse_idx);
4243 }
4244 #else
4245 static inline int assign_proto_idx(struct proto *prot)
4246 {
4247 	return 0;
4248 }
4249 
4250 static inline void release_proto_idx(struct proto *prot)
4251 {
4252 }
4253 
4254 #endif
4255 
4256 static void tw_prot_cleanup(struct timewait_sock_ops *twsk_prot)
4257 {
4258 	if (!twsk_prot)
4259 		return;
4260 	kfree(twsk_prot->twsk_slab_name);
4261 	twsk_prot->twsk_slab_name = NULL;
4262 	kmem_cache_destroy(twsk_prot->twsk_slab);
4263 	twsk_prot->twsk_slab = NULL;
4264 }
4265 
4266 static int tw_prot_init(const struct proto *prot)
4267 {
4268 	struct timewait_sock_ops *twsk_prot = prot->twsk_prot;
4269 
4270 	if (!twsk_prot)
4271 		return 0;
4272 
4273 	twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s",
4274 					      prot->name);
4275 	if (!twsk_prot->twsk_slab_name)
4276 		return -ENOMEM;
4277 
4278 	twsk_prot->twsk_slab =
4279 		kmem_cache_create(twsk_prot->twsk_slab_name,
4280 				  twsk_prot->twsk_obj_size, 0,
4281 				  SLAB_ACCOUNT | prot->slab_flags,
4282 				  NULL);
4283 	if (!twsk_prot->twsk_slab) {
4284 		pr_crit("%s: Can't create timewait sock SLAB cache!\n",
4285 			prot->name);
4286 		return -ENOMEM;
4287 	}
4288 
4289 	return 0;
4290 }
4291 
4292 static void req_prot_cleanup(struct request_sock_ops *rsk_prot)
4293 {
4294 	if (!rsk_prot)
4295 		return;
4296 	kfree(rsk_prot->slab_name);
4297 	rsk_prot->slab_name = NULL;
4298 	kmem_cache_destroy(rsk_prot->slab);
4299 	rsk_prot->slab = NULL;
4300 }
4301 
4302 static int req_prot_init(const struct proto *prot)
4303 {
4304 	struct request_sock_ops *rsk_prot = prot->rsk_prot;
4305 
4306 	if (!rsk_prot)
4307 		return 0;
4308 
4309 	rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s",
4310 					prot->name);
4311 	if (!rsk_prot->slab_name)
4312 		return -ENOMEM;
4313 
4314 	rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name,
4315 					   rsk_prot->obj_size, 0,
4316 					   SLAB_ACCOUNT | prot->slab_flags,
4317 					   NULL);
4318 
4319 	if (!rsk_prot->slab) {
4320 		pr_crit("%s: Can't create request sock SLAB cache!\n",
4321 			prot->name);
4322 		return -ENOMEM;
4323 	}
4324 	return 0;
4325 }
4326 
4327 int proto_register(struct proto *prot, int alloc_slab)
4328 {
4329 	int ret = -ENOBUFS;
4330 
4331 	if (prot->memory_allocated && !prot->sysctl_mem) {
4332 		pr_err("%s: missing sysctl_mem\n", prot->name);
4333 		return -EINVAL;
4334 	}
4335 	if (prot->memory_allocated && !prot->per_cpu_fw_alloc) {
4336 		pr_err("%s: missing per_cpu_fw_alloc\n", prot->name);
4337 		return -EINVAL;
4338 	}
4339 	if (alloc_slab) {
4340 		struct kmem_cache_args args = {
4341 			.useroffset	= prot->useroffset,
4342 			.usersize	= prot->usersize,
4343 			.freeptr_offset = prot->freeptr_offset,
4344 			.use_freeptr_offset = !!prot->freeptr_offset,
4345 		};
4346 
4347 		prot->slab = kmem_cache_create(prot->name, prot->obj_size,
4348 					&args,
4349 					SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT |
4350 					prot->slab_flags);
4351 		if (prot->slab == NULL) {
4352 			pr_crit("%s: Can't create sock SLAB cache!\n",
4353 				prot->name);
4354 			goto out;
4355 		}
4356 
4357 		if (req_prot_init(prot))
4358 			goto out_free_request_sock_slab;
4359 
4360 		if (tw_prot_init(prot))
4361 			goto out_free_timewait_sock_slab;
4362 	}
4363 
4364 	mutex_lock(&proto_list_mutex);
4365 	ret = assign_proto_idx(prot);
4366 	if (ret) {
4367 		mutex_unlock(&proto_list_mutex);
4368 		goto out_free_timewait_sock_slab;
4369 	}
4370 	list_add(&prot->node, &proto_list);
4371 	mutex_unlock(&proto_list_mutex);
4372 	return ret;
4373 
4374 out_free_timewait_sock_slab:
4375 	if (alloc_slab)
4376 		tw_prot_cleanup(prot->twsk_prot);
4377 out_free_request_sock_slab:
4378 	if (alloc_slab) {
4379 		req_prot_cleanup(prot->rsk_prot);
4380 
4381 		kmem_cache_destroy(prot->slab);
4382 		prot->slab = NULL;
4383 	}
4384 out:
4385 	return ret;
4386 }
4387 EXPORT_SYMBOL(proto_register);
4388 
4389 void proto_unregister(struct proto *prot)
4390 {
4391 	mutex_lock(&proto_list_mutex);
4392 	release_proto_idx(prot);
4393 	list_del(&prot->node);
4394 	mutex_unlock(&proto_list_mutex);
4395 
4396 	kmem_cache_destroy(prot->slab);
4397 	prot->slab = NULL;
4398 
4399 	req_prot_cleanup(prot->rsk_prot);
4400 	tw_prot_cleanup(prot->twsk_prot);
4401 }
4402 EXPORT_SYMBOL(proto_unregister);
4403 
4404 int sock_load_diag_module(int family, int protocol)
4405 {
4406 	if (!protocol) {
4407 		if (!sock_is_registered(family))
4408 			return -ENOENT;
4409 
4410 		return request_module("net-pf-%d-proto-%d-type-%d", PF_NETLINK,
4411 				      NETLINK_SOCK_DIAG, family);
4412 	}
4413 
4414 #ifdef CONFIG_INET
4415 	if (family == AF_INET &&
4416 	    protocol != IPPROTO_RAW &&
4417 	    protocol < MAX_INET_PROTOS &&
4418 	    !rcu_access_pointer(inet_protos[protocol]))
4419 		return -ENOENT;
4420 #endif
4421 
4422 	return request_module("net-pf-%d-proto-%d-type-%d-%d", PF_NETLINK,
4423 			      NETLINK_SOCK_DIAG, family, protocol);
4424 }
4425 EXPORT_SYMBOL(sock_load_diag_module);
4426 
4427 #ifdef CONFIG_PROC_FS
4428 static void *proto_seq_start(struct seq_file *seq, loff_t *pos)
4429 	__acquires(proto_list_mutex)
4430 {
4431 	mutex_lock(&proto_list_mutex);
4432 	return seq_list_start_head(&proto_list, *pos);
4433 }
4434 
4435 static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4436 {
4437 	return seq_list_next(v, &proto_list, pos);
4438 }
4439 
4440 static void proto_seq_stop(struct seq_file *seq, void *v)
4441 	__releases(proto_list_mutex)
4442 {
4443 	mutex_unlock(&proto_list_mutex);
4444 }
4445 
4446 static char proto_method_implemented(const void *method)
4447 {
4448 	return method == NULL ? 'n' : 'y';
4449 }
4450 static long sock_prot_memory_allocated(struct proto *proto)
4451 {
4452 	return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L;
4453 }
4454 
4455 static const char *sock_prot_memory_pressure(struct proto *proto)
4456 {
4457 	return proto->memory_pressure != NULL ?
4458 	proto_memory_pressure(proto) ? "yes" : "no" : "NI";
4459 }
4460 
4461 static void proto_seq_printf(struct seq_file *seq, struct proto *proto)
4462 {
4463 
4464 	seq_printf(seq, "%-9s %4u %6d  %6ld   %-3s %6u   %-3s  %-10s "
4465 			"%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n",
4466 		   proto->name,
4467 		   proto->obj_size,
4468 		   sock_prot_inuse_get(seq_file_net(seq), proto),
4469 		   sock_prot_memory_allocated(proto),
4470 		   sock_prot_memory_pressure(proto),
4471 		   proto->max_header,
4472 		   proto->slab == NULL ? "no" : "yes",
4473 		   module_name(proto->owner),
4474 		   proto_method_implemented(proto->close),
4475 		   proto_method_implemented(proto->connect),
4476 		   proto_method_implemented(proto->disconnect),
4477 		   proto_method_implemented(proto->accept),
4478 		   proto_method_implemented(proto->ioctl),
4479 		   proto_method_implemented(proto->init),
4480 		   proto_method_implemented(proto->destroy),
4481 		   proto_method_implemented(proto->shutdown),
4482 		   proto_method_implemented(proto->setsockopt),
4483 		   proto_method_implemented(proto->getsockopt),
4484 		   proto_method_implemented(proto->sendmsg),
4485 		   proto_method_implemented(proto->recvmsg),
4486 		   proto_method_implemented(proto->bind),
4487 		   proto_method_implemented(proto->backlog_rcv),
4488 		   proto_method_implemented(proto->hash),
4489 		   proto_method_implemented(proto->unhash),
4490 		   proto_method_implemented(proto->get_port),
4491 		   proto_method_implemented(proto->enter_memory_pressure));
4492 }
4493 
4494 static int proto_seq_show(struct seq_file *seq, void *v)
4495 {
4496 	if (v == &proto_list)
4497 		seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s",
4498 			   "protocol",
4499 			   "size",
4500 			   "sockets",
4501 			   "memory",
4502 			   "press",
4503 			   "maxhdr",
4504 			   "slab",
4505 			   "module",
4506 			   "cl co di ac io in de sh ss gs se re bi br ha uh gp em\n");
4507 	else
4508 		proto_seq_printf(seq, list_entry(v, struct proto, node));
4509 	return 0;
4510 }
4511 
4512 static const struct seq_operations proto_seq_ops = {
4513 	.start  = proto_seq_start,
4514 	.next   = proto_seq_next,
4515 	.stop   = proto_seq_stop,
4516 	.show   = proto_seq_show,
4517 };
4518 
4519 static __net_init int proto_init_net(struct net *net)
4520 {
4521 	if (!proc_create_net("protocols", 0444, net->proc_net, &proto_seq_ops,
4522 			sizeof(struct seq_net_private)))
4523 		return -ENOMEM;
4524 
4525 	return 0;
4526 }
4527 
4528 static __net_exit void proto_exit_net(struct net *net)
4529 {
4530 	remove_proc_entry("protocols", net->proc_net);
4531 }
4532 
4533 
4534 static __net_initdata struct pernet_operations proto_net_ops = {
4535 	.init = proto_init_net,
4536 	.exit = proto_exit_net,
4537 };
4538 
4539 static int __init proto_init(void)
4540 {
4541 	return register_pernet_subsys(&proto_net_ops);
4542 }
4543 
4544 subsys_initcall(proto_init);
4545 
4546 #endif /* PROC_FS */
4547 
4548 #ifdef CONFIG_NET_RX_BUSY_POLL
4549 bool sk_busy_loop_end(void *p, unsigned long start_time)
4550 {
4551 	struct sock *sk = p;
4552 
4553 	if (!skb_queue_empty_lockless(&sk->sk_receive_queue))
4554 		return true;
4555 
4556 	if (sk_is_udp(sk) &&
4557 	    !skb_queue_empty_lockless(&udp_sk(sk)->reader_queue))
4558 		return true;
4559 
4560 	return sk_busy_loop_timeout(sk, start_time);
4561 }
4562 EXPORT_SYMBOL(sk_busy_loop_end);
4563 #endif /* CONFIG_NET_RX_BUSY_POLL */
4564 
4565 int sock_bind_add(struct sock *sk, struct sockaddr_unsized *addr, int addr_len)
4566 {
4567 	if (!sk->sk_prot->bind_add)
4568 		return -EOPNOTSUPP;
4569 	return sk->sk_prot->bind_add(sk, addr, addr_len);
4570 }
4571 EXPORT_SYMBOL(sock_bind_add);
4572 
4573 /* Copy 'size' bytes from userspace and return `size` back to userspace */
4574 int sock_ioctl_inout(struct sock *sk, unsigned int cmd,
4575 		     void __user *arg, void *karg, size_t size)
4576 {
4577 	int ret;
4578 
4579 	if (copy_from_user(karg, arg, size))
4580 		return -EFAULT;
4581 
4582 	ret = READ_ONCE(sk->sk_prot)->ioctl(sk, cmd, karg);
4583 	if (ret)
4584 		return ret;
4585 
4586 	if (copy_to_user(arg, karg, size))
4587 		return -EFAULT;
4588 
4589 	return 0;
4590 }
4591 EXPORT_SYMBOL(sock_ioctl_inout);
4592 
4593 /* This is the most common ioctl prep function, where the result (4 bytes) is
4594  * copied back to userspace if the ioctl() returns successfully. No input is
4595  * copied from userspace as input argument.
4596  */
4597 static int sock_ioctl_out(struct sock *sk, unsigned int cmd, void __user *arg)
4598 {
4599 	int ret, karg = 0;
4600 
4601 	ret = READ_ONCE(sk->sk_prot)->ioctl(sk, cmd, &karg);
4602 	if (ret)
4603 		return ret;
4604 
4605 	return put_user(karg, (int __user *)arg);
4606 }
4607 
4608 /* A wrapper around sock ioctls, which copies the data from userspace
4609  * (depending on the protocol/ioctl), and copies back the result to userspace.
4610  * The main motivation for this function is to pass kernel memory to the
4611  * protocol ioctl callbacks, instead of userspace memory.
4612  */
4613 int sk_ioctl(struct sock *sk, unsigned int cmd, void __user *arg)
4614 {
4615 	int rc = 1;
4616 
4617 	if (sk->sk_type == SOCK_RAW && sk->sk_family == AF_INET)
4618 		rc = ipmr_sk_ioctl(sk, cmd, arg);
4619 	else if (sk->sk_type == SOCK_RAW && sk->sk_family == AF_INET6)
4620 		rc = ip6mr_sk_ioctl(sk, cmd, arg);
4621 	else if (sk_is_phonet(sk))
4622 		rc = phonet_sk_ioctl(sk, cmd, arg);
4623 
4624 	/* If ioctl was processed, returns its value */
4625 	if (rc <= 0)
4626 		return rc;
4627 
4628 	/* Otherwise call the default handler */
4629 	return sock_ioctl_out(sk, cmd, arg);
4630 }
4631 EXPORT_SYMBOL(sk_ioctl);
4632 
4633 static int __init sock_struct_check(void)
4634 {
4635 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_drops);
4636 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_peek_off);
4637 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_error_queue);
4638 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_receive_queue);
4639 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_backlog);
4640 
4641 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rx_dst);
4642 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rx_dst_ifindex);
4643 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rx_dst_cookie);
4644 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rcvbuf);
4645 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_filter);
4646 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_wq);
4647 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_data_ready);
4648 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rcvtimeo);
4649 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rcvlowat);
4650 
4651 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rxtx, sk_err);
4652 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rxtx, sk_socket);
4653 #ifdef CONFIG_MEMCG
4654 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rxtx, sk_memcg);
4655 #endif
4656 
4657 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_lock);
4658 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_reserved_mem);
4659 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_forward_alloc);
4660 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_tsflags);
4661 
4662 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_omem_alloc);
4663 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_omem_alloc);
4664 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_err_soft);
4665 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_wmem_queued);
4666 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_wmem_alloc);
4667 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_tsq_flags);
4668 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_send_head);
4669 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_write_queue);
4670 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_write_pending);
4671 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_frag);
4672 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_timer);
4673 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_pacing_rate);
4674 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_zckey);
4675 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_tskey);
4676 
4677 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_dst_pending_confirm);
4678 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_pacing_status);
4679 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_max_pacing_rate);
4680 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_sndtimeo);
4681 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_priority);
4682 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_mark);
4683 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_uid);
4684 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_protocol);
4685 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_dst_cache);
4686 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_route_caps);
4687 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_gso_type);
4688 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_gso_max_size);
4689 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_allocation);
4690 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_txhash);
4691 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_sndbuf);
4692 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_gso_max_segs);
4693 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_pacing_shift);
4694 	CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_use_task_frag);
4695 	return 0;
4696 }
4697 
4698 core_initcall(sock_struct_check);
4699