1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * Generic socket support routines. Memory allocators, socket lock/release 8 * handler for protocols to use and generic option handler. 9 * 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Florian La Roche, <flla@stud.uni-sb.de> 13 * Alan Cox, <A.Cox@swansea.ac.uk> 14 * 15 * Fixes: 16 * Alan Cox : Numerous verify_area() problems 17 * Alan Cox : Connecting on a connecting socket 18 * now returns an error for tcp. 19 * Alan Cox : sock->protocol is set correctly. 20 * and is not sometimes left as 0. 21 * Alan Cox : connect handles icmp errors on a 22 * connect properly. Unfortunately there 23 * is a restart syscall nasty there. I 24 * can't match BSD without hacking the C 25 * library. Ideas urgently sought! 26 * Alan Cox : Disallow bind() to addresses that are 27 * not ours - especially broadcast ones!! 28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost) 29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets, 30 * instead they leave that for the DESTROY timer. 31 * Alan Cox : Clean up error flag in accept 32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer 33 * was buggy. Put a remove_sock() in the handler 34 * for memory when we hit 0. Also altered the timer 35 * code. The ACK stuff can wait and needs major 36 * TCP layer surgery. 37 * Alan Cox : Fixed TCP ack bug, removed remove sock 38 * and fixed timer/inet_bh race. 39 * Alan Cox : Added zapped flag for TCP 40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code 41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb 42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources 43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing. 44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so... 45 * Rick Sladkey : Relaxed UDP rules for matching packets. 46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support 47 * Pauline Middelink : identd support 48 * Alan Cox : Fixed connect() taking signals I think. 49 * Alan Cox : SO_LINGER supported 50 * Alan Cox : Error reporting fixes 51 * Anonymous : inet_create tidied up (sk->reuse setting) 52 * Alan Cox : inet sockets don't set sk->type! 53 * Alan Cox : Split socket option code 54 * Alan Cox : Callbacks 55 * Alan Cox : Nagle flag for Charles & Johannes stuff 56 * Alex : Removed restriction on inet fioctl 57 * Alan Cox : Splitting INET from NET core 58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt() 59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code 60 * Alan Cox : Split IP from generic code 61 * Alan Cox : New kfree_skbmem() 62 * Alan Cox : Make SO_DEBUG superuser only. 63 * Alan Cox : Allow anyone to clear SO_DEBUG 64 * (compatibility fix) 65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput. 66 * Alan Cox : Allocator for a socket is settable. 67 * Alan Cox : SO_ERROR includes soft errors. 68 * Alan Cox : Allow NULL arguments on some SO_ opts 69 * Alan Cox : Generic socket allocation to make hooks 70 * easier (suggested by Craig Metz). 71 * Michael Pall : SO_ERROR returns positive errno again 72 * Steve Whitehouse: Added default destructor to free 73 * protocol private data. 74 * Steve Whitehouse: Added various other default routines 75 * common to several socket families. 76 * Chris Evans : Call suser() check last on F_SETOWN 77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER. 78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s() 79 * Andi Kleen : Fix write_space callback 80 * Chris Evans : Security fixes - signedness again 81 * Arnaldo C. Melo : cleanups, use skb_queue_purge 82 * 83 * To Fix: 84 */ 85 86 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 87 88 #include <linux/unaligned.h> 89 #include <linux/capability.h> 90 #include <linux/errno.h> 91 #include <linux/errqueue.h> 92 #include <linux/types.h> 93 #include <linux/socket.h> 94 #include <linux/in.h> 95 #include <linux/kernel.h> 96 #include <linux/module.h> 97 #include <linux/proc_fs.h> 98 #include <linux/seq_file.h> 99 #include <linux/sched.h> 100 #include <linux/sched/mm.h> 101 #include <linux/timer.h> 102 #include <linux/string.h> 103 #include <linux/sockios.h> 104 #include <linux/net.h> 105 #include <linux/mm.h> 106 #include <linux/slab.h> 107 #include <linux/interrupt.h> 108 #include <linux/poll.h> 109 #include <linux/tcp.h> 110 #include <linux/udp.h> 111 #include <linux/init.h> 112 #include <linux/highmem.h> 113 #include <linux/user_namespace.h> 114 #include <linux/static_key.h> 115 #include <linux/memcontrol.h> 116 #include <linux/prefetch.h> 117 #include <linux/compat.h> 118 #include <linux/mroute.h> 119 #include <linux/mroute6.h> 120 #include <linux/icmpv6.h> 121 122 #include <linux/uaccess.h> 123 124 #include <linux/netdevice.h> 125 #include <net/protocol.h> 126 #include <linux/skbuff.h> 127 #include <linux/skbuff_ref.h> 128 #include <net/net_namespace.h> 129 #include <net/request_sock.h> 130 #include <net/sock.h> 131 #include <net/proto_memory.h> 132 #include <linux/net_tstamp.h> 133 #include <net/xfrm.h> 134 #include <linux/ipsec.h> 135 #include <net/cls_cgroup.h> 136 #include <net/netprio_cgroup.h> 137 #include <linux/sock_diag.h> 138 139 #include <linux/filter.h> 140 #include <net/sock_reuseport.h> 141 #include <net/bpf_sk_storage.h> 142 143 #include <trace/events/sock.h> 144 145 #include <net/tcp.h> 146 #include <net/busy_poll.h> 147 #include <net/phonet/phonet.h> 148 149 #include <linux/ethtool.h> 150 151 #include <uapi/linux/pidfd.h> 152 153 #include "dev.h" 154 155 static DEFINE_MUTEX(proto_list_mutex); 156 static LIST_HEAD(proto_list); 157 158 static void sock_def_write_space_wfree(struct sock *sk, int wmem_alloc); 159 static void sock_def_write_space(struct sock *sk); 160 161 /** 162 * sk_ns_capable - General socket capability test 163 * @sk: Socket to use a capability on or through 164 * @user_ns: The user namespace of the capability to use 165 * @cap: The capability to use 166 * 167 * Test to see if the opener of the socket had when the socket was 168 * created and the current process has the capability @cap in the user 169 * namespace @user_ns. 170 */ 171 bool sk_ns_capable(const struct sock *sk, 172 struct user_namespace *user_ns, int cap) 173 { 174 return file_ns_capable(sk->sk_socket->file, user_ns, cap) && 175 ns_capable(user_ns, cap); 176 } 177 EXPORT_SYMBOL(sk_ns_capable); 178 179 /** 180 * sk_capable - Socket global capability test 181 * @sk: Socket to use a capability on or through 182 * @cap: The global capability to use 183 * 184 * Test to see if the opener of the socket had when the socket was 185 * created and the current process has the capability @cap in all user 186 * namespaces. 187 */ 188 bool sk_capable(const struct sock *sk, int cap) 189 { 190 return sk_ns_capable(sk, &init_user_ns, cap); 191 } 192 EXPORT_SYMBOL(sk_capable); 193 194 /** 195 * sk_net_capable - Network namespace socket capability test 196 * @sk: Socket to use a capability on or through 197 * @cap: The capability to use 198 * 199 * Test to see if the opener of the socket had when the socket was created 200 * and the current process has the capability @cap over the network namespace 201 * the socket is a member of. 202 */ 203 bool sk_net_capable(const struct sock *sk, int cap) 204 { 205 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap); 206 } 207 EXPORT_SYMBOL(sk_net_capable); 208 209 /* 210 * Each address family might have different locking rules, so we have 211 * one slock key per address family and separate keys for internal and 212 * userspace sockets. 213 */ 214 static struct lock_class_key af_family_keys[AF_MAX]; 215 static struct lock_class_key af_family_kern_keys[AF_MAX]; 216 static struct lock_class_key af_family_slock_keys[AF_MAX]; 217 static struct lock_class_key af_family_kern_slock_keys[AF_MAX]; 218 219 /* 220 * Make lock validator output more readable. (we pre-construct these 221 * strings build-time, so that runtime initialization of socket 222 * locks is fast): 223 */ 224 225 #define _sock_locks(x) \ 226 x "AF_UNSPEC", x "AF_UNIX" , x "AF_INET" , \ 227 x "AF_AX25" , x "AF_IPX" , x "AF_APPLETALK", \ 228 x "AF_NETROM", x "AF_BRIDGE" , x "AF_ATMPVC" , \ 229 x "AF_X25" , x "AF_INET6" , x "AF_ROSE" , \ 230 x "AF_DECnet", x "AF_NETBEUI" , x "AF_SECURITY" , \ 231 x "AF_KEY" , x "AF_NETLINK" , x "AF_PACKET" , \ 232 x "AF_ASH" , x "AF_ECONET" , x "AF_ATMSVC" , \ 233 x "AF_RDS" , x "AF_SNA" , x "AF_IRDA" , \ 234 x "AF_PPPOX" , x "AF_WANPIPE" , x "AF_LLC" , \ 235 x "27" , x "28" , x "AF_CAN" , \ 236 x "AF_TIPC" , x "AF_BLUETOOTH", x "IUCV" , \ 237 x "AF_RXRPC" , x "AF_ISDN" , x "AF_PHONET" , \ 238 x "AF_IEEE802154", x "AF_CAIF" , x "AF_ALG" , \ 239 x "AF_NFC" , x "AF_VSOCK" , x "AF_KCM" , \ 240 x "AF_QIPCRTR", x "AF_SMC" , x "AF_XDP" , \ 241 x "AF_MCTP" , \ 242 x "AF_MAX" 243 244 static const char *const af_family_key_strings[AF_MAX+1] = { 245 _sock_locks("sk_lock-") 246 }; 247 static const char *const af_family_slock_key_strings[AF_MAX+1] = { 248 _sock_locks("slock-") 249 }; 250 static const char *const af_family_clock_key_strings[AF_MAX+1] = { 251 _sock_locks("clock-") 252 }; 253 254 static const char *const af_family_kern_key_strings[AF_MAX+1] = { 255 _sock_locks("k-sk_lock-") 256 }; 257 static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = { 258 _sock_locks("k-slock-") 259 }; 260 static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = { 261 _sock_locks("k-clock-") 262 }; 263 static const char *const af_family_rlock_key_strings[AF_MAX+1] = { 264 _sock_locks("rlock-") 265 }; 266 static const char *const af_family_wlock_key_strings[AF_MAX+1] = { 267 _sock_locks("wlock-") 268 }; 269 static const char *const af_family_elock_key_strings[AF_MAX+1] = { 270 _sock_locks("elock-") 271 }; 272 273 /* 274 * sk_callback_lock and sk queues locking rules are per-address-family, 275 * so split the lock classes by using a per-AF key: 276 */ 277 static struct lock_class_key af_callback_keys[AF_MAX]; 278 static struct lock_class_key af_rlock_keys[AF_MAX]; 279 static struct lock_class_key af_wlock_keys[AF_MAX]; 280 static struct lock_class_key af_elock_keys[AF_MAX]; 281 static struct lock_class_key af_kern_callback_keys[AF_MAX]; 282 283 /* Run time adjustable parameters. */ 284 __u32 sysctl_wmem_max __read_mostly = 4 << 20; 285 EXPORT_SYMBOL(sysctl_wmem_max); 286 __u32 sysctl_rmem_max __read_mostly = 4 << 20; 287 EXPORT_SYMBOL(sysctl_rmem_max); 288 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_DEFAULT; 289 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_DEFAULT; 290 291 DEFINE_STATIC_KEY_FALSE(memalloc_socks_key); 292 EXPORT_SYMBOL_GPL(memalloc_socks_key); 293 294 /** 295 * sk_set_memalloc - sets %SOCK_MEMALLOC 296 * @sk: socket to set it on 297 * 298 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves. 299 * It's the responsibility of the admin to adjust min_free_kbytes 300 * to meet the requirements 301 */ 302 void sk_set_memalloc(struct sock *sk) 303 { 304 sock_set_flag(sk, SOCK_MEMALLOC); 305 sk->sk_allocation |= __GFP_MEMALLOC; 306 static_branch_inc(&memalloc_socks_key); 307 } 308 EXPORT_SYMBOL_GPL(sk_set_memalloc); 309 310 void sk_clear_memalloc(struct sock *sk) 311 { 312 sock_reset_flag(sk, SOCK_MEMALLOC); 313 sk->sk_allocation &= ~__GFP_MEMALLOC; 314 static_branch_dec(&memalloc_socks_key); 315 316 /* 317 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward 318 * progress of swapping. SOCK_MEMALLOC may be cleared while 319 * it has rmem allocations due to the last swapfile being deactivated 320 * but there is a risk that the socket is unusable due to exceeding 321 * the rmem limits. Reclaim the reserves and obey rmem limits again. 322 */ 323 sk_mem_reclaim(sk); 324 } 325 EXPORT_SYMBOL_GPL(sk_clear_memalloc); 326 327 int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb) 328 { 329 int ret; 330 unsigned int noreclaim_flag; 331 332 /* these should have been dropped before queueing */ 333 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC)); 334 335 noreclaim_flag = memalloc_noreclaim_save(); 336 ret = INDIRECT_CALL_INET(sk->sk_backlog_rcv, 337 tcp_v6_do_rcv, 338 tcp_v4_do_rcv, 339 sk, skb); 340 memalloc_noreclaim_restore(noreclaim_flag); 341 342 return ret; 343 } 344 EXPORT_SYMBOL(__sk_backlog_rcv); 345 346 void sk_error_report(struct sock *sk) 347 { 348 sk->sk_error_report(sk); 349 350 switch (sk->sk_family) { 351 case AF_INET: 352 fallthrough; 353 case AF_INET6: 354 trace_inet_sk_error_report(sk); 355 break; 356 default: 357 break; 358 } 359 } 360 EXPORT_SYMBOL(sk_error_report); 361 362 int sock_get_timeout(long timeo, void *optval, bool old_timeval) 363 { 364 struct __kernel_sock_timeval tv; 365 366 if (timeo == MAX_SCHEDULE_TIMEOUT) { 367 tv.tv_sec = 0; 368 tv.tv_usec = 0; 369 } else { 370 tv.tv_sec = timeo / HZ; 371 tv.tv_usec = ((timeo % HZ) * USEC_PER_SEC) / HZ; 372 } 373 374 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) { 375 struct old_timeval32 tv32 = { tv.tv_sec, tv.tv_usec }; 376 *(struct old_timeval32 *)optval = tv32; 377 return sizeof(tv32); 378 } 379 380 if (old_timeval) { 381 struct __kernel_old_timeval old_tv; 382 old_tv.tv_sec = tv.tv_sec; 383 old_tv.tv_usec = tv.tv_usec; 384 *(struct __kernel_old_timeval *)optval = old_tv; 385 return sizeof(old_tv); 386 } 387 388 *(struct __kernel_sock_timeval *)optval = tv; 389 return sizeof(tv); 390 } 391 EXPORT_SYMBOL(sock_get_timeout); 392 393 int sock_copy_user_timeval(struct __kernel_sock_timeval *tv, 394 sockptr_t optval, int optlen, bool old_timeval) 395 { 396 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) { 397 struct old_timeval32 tv32; 398 399 if (optlen < sizeof(tv32)) 400 return -EINVAL; 401 402 if (copy_from_sockptr(&tv32, optval, sizeof(tv32))) 403 return -EFAULT; 404 tv->tv_sec = tv32.tv_sec; 405 tv->tv_usec = tv32.tv_usec; 406 } else if (old_timeval) { 407 struct __kernel_old_timeval old_tv; 408 409 if (optlen < sizeof(old_tv)) 410 return -EINVAL; 411 if (copy_from_sockptr(&old_tv, optval, sizeof(old_tv))) 412 return -EFAULT; 413 tv->tv_sec = old_tv.tv_sec; 414 tv->tv_usec = old_tv.tv_usec; 415 } else { 416 if (optlen < sizeof(*tv)) 417 return -EINVAL; 418 if (copy_from_sockptr(tv, optval, sizeof(*tv))) 419 return -EFAULT; 420 } 421 422 return 0; 423 } 424 EXPORT_SYMBOL(sock_copy_user_timeval); 425 426 static int sock_set_timeout(long *timeo_p, sockptr_t optval, int optlen, 427 bool old_timeval) 428 { 429 struct __kernel_sock_timeval tv; 430 int err = sock_copy_user_timeval(&tv, optval, optlen, old_timeval); 431 long val; 432 433 if (err) 434 return err; 435 436 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC) 437 return -EDOM; 438 439 if (tv.tv_sec < 0) { 440 static int warned __read_mostly; 441 442 WRITE_ONCE(*timeo_p, 0); 443 if (warned < 10 && net_ratelimit()) { 444 warned++; 445 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n", 446 __func__, current->comm, task_pid_nr(current)); 447 } 448 return 0; 449 } 450 val = MAX_SCHEDULE_TIMEOUT; 451 if ((tv.tv_sec || tv.tv_usec) && 452 (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT / HZ - 1))) 453 val = tv.tv_sec * HZ + DIV_ROUND_UP((unsigned long)tv.tv_usec, 454 USEC_PER_SEC / HZ); 455 WRITE_ONCE(*timeo_p, val); 456 return 0; 457 } 458 459 static bool sk_set_prio_allowed(const struct sock *sk, int val) 460 { 461 return ((val >= TC_PRIO_BESTEFFORT && val <= TC_PRIO_INTERACTIVE) || 462 sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) || 463 sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)); 464 } 465 466 static bool sock_needs_netstamp(const struct sock *sk) 467 { 468 switch (sk->sk_family) { 469 case AF_UNSPEC: 470 case AF_UNIX: 471 return false; 472 default: 473 return true; 474 } 475 } 476 477 static void sock_disable_timestamp(struct sock *sk, unsigned long flags) 478 { 479 if (sk->sk_flags & flags) { 480 sk->sk_flags &= ~flags; 481 if (sock_needs_netstamp(sk) && 482 !(sk->sk_flags & SK_FLAGS_TIMESTAMP)) 483 net_disable_timestamp(); 484 } 485 } 486 487 488 int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 489 { 490 unsigned long flags; 491 struct sk_buff_head *list = &sk->sk_receive_queue; 492 493 if (atomic_read(&sk->sk_rmem_alloc) >= READ_ONCE(sk->sk_rcvbuf)) { 494 sk_drops_inc(sk); 495 trace_sock_rcvqueue_full(sk, skb); 496 return -ENOMEM; 497 } 498 499 if (!sk_rmem_schedule(sk, skb, skb->truesize)) { 500 sk_drops_inc(sk); 501 return -ENOBUFS; 502 } 503 504 skb->dev = NULL; 505 skb_set_owner_r(skb, sk); 506 507 /* we escape from rcu protected region, make sure we dont leak 508 * a norefcounted dst 509 */ 510 skb_dst_force(skb); 511 512 spin_lock_irqsave(&list->lock, flags); 513 sock_skb_set_dropcount(sk, skb); 514 __skb_queue_tail(list, skb); 515 spin_unlock_irqrestore(&list->lock, flags); 516 517 if (!sock_flag(sk, SOCK_DEAD)) 518 sk->sk_data_ready(sk); 519 return 0; 520 } 521 EXPORT_SYMBOL(__sock_queue_rcv_skb); 522 523 enum skb_drop_reason 524 sock_queue_rcv_skb_reason(struct sock *sk, struct sk_buff *skb) 525 { 526 enum skb_drop_reason drop_reason; 527 int err; 528 529 drop_reason = sk_filter_reason(sk, skb); 530 if (drop_reason) 531 return drop_reason; 532 533 err = __sock_queue_rcv_skb(sk, skb); 534 switch (err) { 535 case -ENOMEM: 536 return SKB_DROP_REASON_SOCKET_RCVBUFF; 537 case -ENOBUFS: 538 return SKB_DROP_REASON_PROTO_MEM; 539 } 540 return SKB_NOT_DROPPED_YET; 541 } 542 EXPORT_SYMBOL(sock_queue_rcv_skb_reason); 543 544 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, 545 const int nested, unsigned int trim_cap, bool refcounted) 546 { 547 enum skb_drop_reason reason; 548 int rc = NET_RX_SUCCESS; 549 int err; 550 551 reason = sk_filter_trim_cap(sk, skb, trim_cap); 552 if (reason) 553 goto discard_and_relse; 554 555 skb->dev = NULL; 556 557 if (sk_rcvqueues_full(sk, READ_ONCE(sk->sk_rcvbuf))) { 558 sk_drops_inc(sk); 559 reason = SKB_DROP_REASON_SOCKET_RCVBUFF; 560 goto discard_and_relse; 561 } 562 if (nested) 563 bh_lock_sock_nested(sk); 564 else 565 bh_lock_sock(sk); 566 if (!sock_owned_by_user(sk)) { 567 /* 568 * trylock + unlock semantics: 569 */ 570 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_); 571 572 rc = sk_backlog_rcv(sk, skb); 573 574 mutex_release(&sk->sk_lock.dep_map, _RET_IP_); 575 } else if ((err = sk_add_backlog(sk, skb, READ_ONCE(sk->sk_rcvbuf)))) { 576 bh_unlock_sock(sk); 577 if (err == -ENOMEM) 578 reason = SKB_DROP_REASON_PFMEMALLOC; 579 if (err == -ENOBUFS) 580 reason = SKB_DROP_REASON_SOCKET_BACKLOG; 581 sk_drops_inc(sk); 582 goto discard_and_relse; 583 } 584 585 bh_unlock_sock(sk); 586 out: 587 if (refcounted) 588 sock_put(sk); 589 return rc; 590 discard_and_relse: 591 sk_skb_reason_drop(sk, skb, reason); 592 goto out; 593 } 594 EXPORT_SYMBOL(__sk_receive_skb); 595 596 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ip6_dst_check(struct dst_entry *, 597 u32)); 598 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *, 599 u32)); 600 struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie) 601 { 602 struct dst_entry *dst = __sk_dst_get(sk); 603 604 if (dst && READ_ONCE(dst->obsolete) && 605 INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check, 606 dst, cookie) == NULL) { 607 sk_tx_queue_clear(sk); 608 WRITE_ONCE(sk->sk_dst_pending_confirm, 0); 609 RCU_INIT_POINTER(sk->sk_dst_cache, NULL); 610 dst_release(dst); 611 return NULL; 612 } 613 614 return dst; 615 } 616 EXPORT_SYMBOL(__sk_dst_check); 617 618 struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie) 619 { 620 struct dst_entry *dst = sk_dst_get(sk); 621 622 if (dst && READ_ONCE(dst->obsolete) && 623 INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check, 624 dst, cookie) == NULL) { 625 sk_dst_reset(sk); 626 dst_release(dst); 627 return NULL; 628 } 629 630 return dst; 631 } 632 EXPORT_SYMBOL(sk_dst_check); 633 634 static int sock_bindtoindex_locked(struct sock *sk, int ifindex) 635 { 636 int ret = -ENOPROTOOPT; 637 #ifdef CONFIG_NETDEVICES 638 struct net *net = sock_net(sk); 639 640 /* Sorry... */ 641 ret = -EPERM; 642 if (sk->sk_bound_dev_if && !ns_capable(net->user_ns, CAP_NET_RAW)) 643 goto out; 644 645 ret = -EINVAL; 646 if (ifindex < 0) 647 goto out; 648 649 /* Paired with all READ_ONCE() done locklessly. */ 650 WRITE_ONCE(sk->sk_bound_dev_if, ifindex); 651 652 if (sk->sk_prot->rehash) 653 sk->sk_prot->rehash(sk); 654 sk_dst_reset(sk); 655 656 ret = 0; 657 658 out: 659 #endif 660 661 return ret; 662 } 663 664 int sock_bindtoindex(struct sock *sk, int ifindex, bool lock_sk) 665 { 666 int ret; 667 668 if (lock_sk) 669 lock_sock(sk); 670 ret = sock_bindtoindex_locked(sk, ifindex); 671 if (lock_sk) 672 release_sock(sk); 673 674 return ret; 675 } 676 EXPORT_SYMBOL(sock_bindtoindex); 677 678 static int sock_setbindtodevice(struct sock *sk, sockptr_t optval, int optlen) 679 { 680 int ret = -ENOPROTOOPT; 681 #ifdef CONFIG_NETDEVICES 682 struct net *net = sock_net(sk); 683 char devname[IFNAMSIZ]; 684 int index; 685 686 ret = -EINVAL; 687 if (optlen < 0) 688 goto out; 689 690 /* Bind this socket to a particular device like "eth0", 691 * as specified in the passed interface name. If the 692 * name is "" or the option length is zero the socket 693 * is not bound. 694 */ 695 if (optlen > IFNAMSIZ - 1) 696 optlen = IFNAMSIZ - 1; 697 memset(devname, 0, sizeof(devname)); 698 699 ret = -EFAULT; 700 if (copy_from_sockptr(devname, optval, optlen)) 701 goto out; 702 703 index = 0; 704 if (devname[0] != '\0') { 705 struct net_device *dev; 706 707 rcu_read_lock(); 708 dev = dev_get_by_name_rcu(net, devname); 709 if (dev) 710 index = dev->ifindex; 711 rcu_read_unlock(); 712 ret = -ENODEV; 713 if (!dev) 714 goto out; 715 } 716 717 sockopt_lock_sock(sk); 718 ret = sock_bindtoindex_locked(sk, index); 719 sockopt_release_sock(sk); 720 out: 721 #endif 722 723 return ret; 724 } 725 726 static int sock_getbindtodevice(struct sock *sk, sockptr_t optval, 727 sockptr_t optlen, int len) 728 { 729 int ret = -ENOPROTOOPT; 730 #ifdef CONFIG_NETDEVICES 731 int bound_dev_if = READ_ONCE(sk->sk_bound_dev_if); 732 struct net *net = sock_net(sk); 733 char devname[IFNAMSIZ]; 734 735 if (bound_dev_if == 0) { 736 len = 0; 737 goto zero; 738 } 739 740 ret = -EINVAL; 741 if (len < IFNAMSIZ) 742 goto out; 743 744 ret = netdev_get_name(net, devname, bound_dev_if); 745 if (ret) 746 goto out; 747 748 len = strlen(devname) + 1; 749 750 ret = -EFAULT; 751 if (copy_to_sockptr(optval, devname, len)) 752 goto out; 753 754 zero: 755 ret = -EFAULT; 756 if (copy_to_sockptr(optlen, &len, sizeof(int))) 757 goto out; 758 759 ret = 0; 760 761 out: 762 #endif 763 764 return ret; 765 } 766 767 bool sk_mc_loop(const struct sock *sk) 768 { 769 if (dev_recursion_level()) 770 return false; 771 if (!sk) 772 return true; 773 /* IPV6_ADDRFORM can change sk->sk_family under us. */ 774 switch (READ_ONCE(sk->sk_family)) { 775 case AF_INET: 776 return inet_test_bit(MC_LOOP, sk); 777 #if IS_ENABLED(CONFIG_IPV6) 778 case AF_INET6: 779 return inet6_test_bit(MC6_LOOP, sk); 780 #endif 781 } 782 return true; 783 } 784 EXPORT_SYMBOL(sk_mc_loop); 785 786 void sock_set_reuseaddr(struct sock *sk) 787 { 788 lock_sock(sk); 789 sk->sk_reuse = SK_CAN_REUSE; 790 release_sock(sk); 791 } 792 EXPORT_SYMBOL(sock_set_reuseaddr); 793 794 void sock_set_reuseport(struct sock *sk) 795 { 796 lock_sock(sk); 797 sk->sk_reuseport = true; 798 release_sock(sk); 799 } 800 EXPORT_SYMBOL(sock_set_reuseport); 801 802 void sock_no_linger(struct sock *sk) 803 { 804 lock_sock(sk); 805 WRITE_ONCE(sk->sk_lingertime, 0); 806 sock_set_flag(sk, SOCK_LINGER); 807 release_sock(sk); 808 } 809 EXPORT_SYMBOL(sock_no_linger); 810 811 void sock_set_priority(struct sock *sk, u32 priority) 812 { 813 WRITE_ONCE(sk->sk_priority, priority); 814 } 815 EXPORT_SYMBOL(sock_set_priority); 816 817 void sock_set_sndtimeo(struct sock *sk, s64 secs) 818 { 819 if (secs && secs < MAX_SCHEDULE_TIMEOUT / HZ - 1) 820 WRITE_ONCE(sk->sk_sndtimeo, secs * HZ); 821 else 822 WRITE_ONCE(sk->sk_sndtimeo, MAX_SCHEDULE_TIMEOUT); 823 } 824 EXPORT_SYMBOL(sock_set_sndtimeo); 825 826 static void __sock_set_timestamps(struct sock *sk, bool val, bool new, bool ns) 827 { 828 sock_valbool_flag(sk, SOCK_RCVTSTAMP, val); 829 sock_valbool_flag(sk, SOCK_RCVTSTAMPNS, val && ns); 830 if (val) { 831 sock_valbool_flag(sk, SOCK_TSTAMP_NEW, new); 832 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 833 } 834 } 835 836 void sock_set_timestamp(struct sock *sk, int optname, bool valbool) 837 { 838 switch (optname) { 839 case SO_TIMESTAMP_OLD: 840 __sock_set_timestamps(sk, valbool, false, false); 841 break; 842 case SO_TIMESTAMP_NEW: 843 __sock_set_timestamps(sk, valbool, true, false); 844 break; 845 case SO_TIMESTAMPNS_OLD: 846 __sock_set_timestamps(sk, valbool, false, true); 847 break; 848 case SO_TIMESTAMPNS_NEW: 849 __sock_set_timestamps(sk, valbool, true, true); 850 break; 851 } 852 } 853 854 static int sock_timestamping_bind_phc(struct sock *sk, int phc_index) 855 { 856 struct net *net = sock_net(sk); 857 struct net_device *dev = NULL; 858 bool match = false; 859 int *vclock_index; 860 int i, num; 861 862 if (sk->sk_bound_dev_if) 863 dev = dev_get_by_index(net, sk->sk_bound_dev_if); 864 865 if (!dev) { 866 pr_err("%s: sock not bind to device\n", __func__); 867 return -EOPNOTSUPP; 868 } 869 870 num = ethtool_get_phc_vclocks(dev, &vclock_index); 871 dev_put(dev); 872 873 for (i = 0; i < num; i++) { 874 if (*(vclock_index + i) == phc_index) { 875 match = true; 876 break; 877 } 878 } 879 880 if (num > 0) 881 kfree(vclock_index); 882 883 if (!match) 884 return -EINVAL; 885 886 WRITE_ONCE(sk->sk_bind_phc, phc_index); 887 888 return 0; 889 } 890 891 int sock_set_timestamping(struct sock *sk, int optname, 892 struct so_timestamping timestamping) 893 { 894 int val = timestamping.flags; 895 int ret; 896 897 if (val & ~SOF_TIMESTAMPING_MASK) 898 return -EINVAL; 899 900 if (val & SOF_TIMESTAMPING_OPT_ID_TCP && 901 !(val & SOF_TIMESTAMPING_OPT_ID)) 902 return -EINVAL; 903 904 if (val & SOF_TIMESTAMPING_OPT_ID && 905 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) { 906 if (sk_is_tcp(sk)) { 907 if ((1 << sk->sk_state) & 908 (TCPF_CLOSE | TCPF_LISTEN)) 909 return -EINVAL; 910 if (val & SOF_TIMESTAMPING_OPT_ID_TCP) 911 atomic_set(&sk->sk_tskey, tcp_sk(sk)->write_seq); 912 else 913 atomic_set(&sk->sk_tskey, tcp_sk(sk)->snd_una); 914 } else { 915 atomic_set(&sk->sk_tskey, 0); 916 } 917 } 918 919 if (val & SOF_TIMESTAMPING_OPT_STATS && 920 !(val & SOF_TIMESTAMPING_OPT_TSONLY)) 921 return -EINVAL; 922 923 if (val & SOF_TIMESTAMPING_BIND_PHC) { 924 ret = sock_timestamping_bind_phc(sk, timestamping.bind_phc); 925 if (ret) 926 return ret; 927 } 928 929 WRITE_ONCE(sk->sk_tsflags, val); 930 sock_valbool_flag(sk, SOCK_TSTAMP_NEW, optname == SO_TIMESTAMPING_NEW); 931 sock_valbool_flag(sk, SOCK_TIMESTAMPING_ANY, !!(val & TSFLAGS_ANY)); 932 933 if (val & SOF_TIMESTAMPING_RX_SOFTWARE) 934 sock_enable_timestamp(sk, 935 SOCK_TIMESTAMPING_RX_SOFTWARE); 936 else 937 sock_disable_timestamp(sk, 938 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE)); 939 return 0; 940 } 941 942 #if defined(CONFIG_CGROUP_BPF) 943 void bpf_skops_tx_timestamping(struct sock *sk, struct sk_buff *skb, int op) 944 { 945 struct bpf_sock_ops_kern sock_ops; 946 947 memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); 948 sock_ops.op = op; 949 sock_ops.is_fullsock = 1; 950 sock_ops.sk = sk; 951 bpf_skops_init_skb(&sock_ops, skb, 0); 952 __cgroup_bpf_run_filter_sock_ops(sk, &sock_ops, CGROUP_SOCK_OPS); 953 } 954 #endif 955 956 void sock_set_keepalive(struct sock *sk) 957 { 958 lock_sock(sk); 959 if (sk->sk_prot->keepalive) 960 sk->sk_prot->keepalive(sk, true); 961 sock_valbool_flag(sk, SOCK_KEEPOPEN, true); 962 release_sock(sk); 963 } 964 EXPORT_SYMBOL(sock_set_keepalive); 965 966 static void __sock_set_rcvbuf(struct sock *sk, int val) 967 { 968 struct socket *sock = sk->sk_socket; 969 970 /* Ensure val * 2 fits into an int, to prevent max_t() from treating it 971 * as a negative value. 972 */ 973 val = min_t(int, val, INT_MAX / 2); 974 sk->sk_userlocks |= SOCK_RCVBUF_LOCK; 975 976 /* We double it on the way in to account for "struct sk_buff" etc. 977 * overhead. Applications assume that the SO_RCVBUF setting they make 978 * will allow that much actual data to be received on that socket. 979 * 980 * Applications are unaware that "struct sk_buff" and other overheads 981 * allocate from the receive buffer during socket buffer allocation. 982 * 983 * And after considering the possible alternatives, returning the value 984 * we actually used in getsockopt is the most desirable behavior. 985 */ 986 WRITE_ONCE(sk->sk_rcvbuf, max_t(int, val * 2, SOCK_MIN_RCVBUF)); 987 988 if (sock) { 989 const struct proto_ops *ops = READ_ONCE(sock->ops); 990 991 if (ops->set_rcvbuf) 992 ops->set_rcvbuf(sk, sk->sk_rcvbuf); 993 } 994 } 995 996 void sock_set_rcvbuf(struct sock *sk, int val) 997 { 998 lock_sock(sk); 999 __sock_set_rcvbuf(sk, val); 1000 release_sock(sk); 1001 } 1002 EXPORT_SYMBOL(sock_set_rcvbuf); 1003 1004 static void __sock_set_mark(struct sock *sk, u32 val) 1005 { 1006 if (val != sk->sk_mark) { 1007 WRITE_ONCE(sk->sk_mark, val); 1008 sk_dst_reset(sk); 1009 } 1010 } 1011 1012 void sock_set_mark(struct sock *sk, u32 val) 1013 { 1014 lock_sock(sk); 1015 __sock_set_mark(sk, val); 1016 release_sock(sk); 1017 } 1018 EXPORT_SYMBOL(sock_set_mark); 1019 1020 static void sock_release_reserved_memory(struct sock *sk, int bytes) 1021 { 1022 /* Round down bytes to multiple of pages */ 1023 bytes = round_down(bytes, PAGE_SIZE); 1024 1025 WARN_ON(bytes > sk->sk_reserved_mem); 1026 WRITE_ONCE(sk->sk_reserved_mem, sk->sk_reserved_mem - bytes); 1027 sk_mem_reclaim(sk); 1028 } 1029 1030 static int sock_reserve_memory(struct sock *sk, int bytes) 1031 { 1032 long allocated; 1033 bool charged; 1034 int pages; 1035 1036 if (!mem_cgroup_sk_enabled(sk) || !sk_has_account(sk)) 1037 return -EOPNOTSUPP; 1038 1039 if (!bytes) 1040 return 0; 1041 1042 pages = sk_mem_pages(bytes); 1043 1044 /* pre-charge to memcg */ 1045 charged = mem_cgroup_sk_charge(sk, pages, 1046 GFP_KERNEL | __GFP_RETRY_MAYFAIL); 1047 if (!charged) 1048 return -ENOMEM; 1049 1050 if (sk->sk_bypass_prot_mem) 1051 goto success; 1052 1053 /* pre-charge to forward_alloc */ 1054 sk_memory_allocated_add(sk, pages); 1055 allocated = sk_memory_allocated(sk); 1056 1057 /* If the system goes into memory pressure with this 1058 * precharge, give up and return error. 1059 */ 1060 if (allocated > sk_prot_mem_limits(sk, 1)) { 1061 sk_memory_allocated_sub(sk, pages); 1062 mem_cgroup_sk_uncharge(sk, pages); 1063 return -ENOMEM; 1064 } 1065 1066 success: 1067 sk_forward_alloc_add(sk, pages << PAGE_SHIFT); 1068 1069 WRITE_ONCE(sk->sk_reserved_mem, 1070 sk->sk_reserved_mem + (pages << PAGE_SHIFT)); 1071 1072 return 0; 1073 } 1074 1075 #ifdef CONFIG_PAGE_POOL 1076 1077 /* This is the number of tokens and frags that the user can SO_DEVMEM_DONTNEED 1078 * in 1 syscall. The limit exists to limit the amount of memory the kernel 1079 * allocates to copy these tokens, and to prevent looping over the frags for 1080 * too long. 1081 */ 1082 #define MAX_DONTNEED_TOKENS 128 1083 #define MAX_DONTNEED_FRAGS 1024 1084 1085 static noinline_for_stack int 1086 sock_devmem_dontneed(struct sock *sk, sockptr_t optval, unsigned int optlen) 1087 { 1088 unsigned int num_tokens, i, j, k, netmem_num = 0; 1089 struct dmabuf_token *tokens; 1090 int ret = 0, num_frags = 0; 1091 netmem_ref netmems[16]; 1092 1093 if (!sk_is_tcp(sk)) 1094 return -EBADF; 1095 1096 if (optlen % sizeof(*tokens) || 1097 optlen > sizeof(*tokens) * MAX_DONTNEED_TOKENS) 1098 return -EINVAL; 1099 1100 num_tokens = optlen / sizeof(*tokens); 1101 tokens = kvmalloc_objs(*tokens, num_tokens); 1102 if (!tokens) 1103 return -ENOMEM; 1104 1105 if (copy_from_sockptr(tokens, optval, optlen)) { 1106 kvfree(tokens); 1107 return -EFAULT; 1108 } 1109 1110 xa_lock_bh(&sk->sk_user_frags); 1111 for (i = 0; i < num_tokens; i++) { 1112 for (j = 0; j < tokens[i].token_count; j++) { 1113 if (++num_frags > MAX_DONTNEED_FRAGS) 1114 goto frag_limit_reached; 1115 1116 netmem_ref netmem = (__force netmem_ref)__xa_erase( 1117 &sk->sk_user_frags, tokens[i].token_start + j); 1118 1119 if (!netmem || WARN_ON_ONCE(!netmem_is_net_iov(netmem))) 1120 continue; 1121 1122 netmems[netmem_num++] = netmem; 1123 if (netmem_num == ARRAY_SIZE(netmems)) { 1124 xa_unlock_bh(&sk->sk_user_frags); 1125 for (k = 0; k < netmem_num; k++) 1126 WARN_ON_ONCE(!napi_pp_put_page(netmems[k])); 1127 netmem_num = 0; 1128 xa_lock_bh(&sk->sk_user_frags); 1129 } 1130 ret++; 1131 } 1132 } 1133 1134 frag_limit_reached: 1135 xa_unlock_bh(&sk->sk_user_frags); 1136 for (k = 0; k < netmem_num; k++) 1137 WARN_ON_ONCE(!napi_pp_put_page(netmems[k])); 1138 1139 kvfree(tokens); 1140 return ret; 1141 } 1142 #endif 1143 1144 void sockopt_lock_sock(struct sock *sk) 1145 { 1146 /* When current->bpf_ctx is set, the setsockopt is called from 1147 * a bpf prog. bpf has ensured the sk lock has been 1148 * acquired before calling setsockopt(). 1149 */ 1150 if (has_current_bpf_ctx()) 1151 return; 1152 1153 lock_sock(sk); 1154 } 1155 EXPORT_SYMBOL(sockopt_lock_sock); 1156 1157 void sockopt_release_sock(struct sock *sk) 1158 { 1159 if (has_current_bpf_ctx()) 1160 return; 1161 1162 release_sock(sk); 1163 } 1164 EXPORT_SYMBOL(sockopt_release_sock); 1165 1166 bool sockopt_ns_capable(struct user_namespace *ns, int cap) 1167 { 1168 return has_current_bpf_ctx() || ns_capable(ns, cap); 1169 } 1170 EXPORT_SYMBOL(sockopt_ns_capable); 1171 1172 bool sockopt_capable(int cap) 1173 { 1174 return has_current_bpf_ctx() || capable(cap); 1175 } 1176 EXPORT_SYMBOL(sockopt_capable); 1177 1178 static int sockopt_validate_clockid(__kernel_clockid_t value) 1179 { 1180 switch (value) { 1181 case CLOCK_REALTIME: 1182 case CLOCK_MONOTONIC: 1183 case CLOCK_TAI: 1184 return 0; 1185 } 1186 return -EINVAL; 1187 } 1188 1189 /* 1190 * This is meant for all protocols to use and covers goings on 1191 * at the socket level. Everything here is generic. 1192 */ 1193 1194 int sk_setsockopt(struct sock *sk, int level, int optname, 1195 sockptr_t optval, unsigned int optlen) 1196 { 1197 struct so_timestamping timestamping; 1198 struct socket *sock = sk->sk_socket; 1199 struct sock_txtime sk_txtime; 1200 int val; 1201 int valbool; 1202 struct linger ling; 1203 int ret = 0; 1204 1205 /* 1206 * Options without arguments 1207 */ 1208 1209 if (optname == SO_BINDTODEVICE) 1210 return sock_setbindtodevice(sk, optval, optlen); 1211 1212 if (optlen < sizeof(int)) 1213 return -EINVAL; 1214 1215 if (copy_from_sockptr(&val, optval, sizeof(val))) 1216 return -EFAULT; 1217 1218 valbool = val ? 1 : 0; 1219 1220 /* handle options which do not require locking the socket. */ 1221 switch (optname) { 1222 case SO_PRIORITY: 1223 if (sk_set_prio_allowed(sk, val)) { 1224 sock_set_priority(sk, val); 1225 return 0; 1226 } 1227 return -EPERM; 1228 case SO_TYPE: 1229 case SO_PROTOCOL: 1230 case SO_DOMAIN: 1231 case SO_ERROR: 1232 return -ENOPROTOOPT; 1233 #ifdef CONFIG_NET_RX_BUSY_POLL 1234 case SO_BUSY_POLL: 1235 if (val < 0) 1236 return -EINVAL; 1237 WRITE_ONCE(sk->sk_ll_usec, val); 1238 return 0; 1239 case SO_PREFER_BUSY_POLL: 1240 if (valbool && !sockopt_capable(CAP_NET_ADMIN)) 1241 return -EPERM; 1242 WRITE_ONCE(sk->sk_prefer_busy_poll, valbool); 1243 return 0; 1244 case SO_BUSY_POLL_BUDGET: 1245 if (val > READ_ONCE(sk->sk_busy_poll_budget) && 1246 !sockopt_capable(CAP_NET_ADMIN)) 1247 return -EPERM; 1248 if (val < 0 || val > U16_MAX) 1249 return -EINVAL; 1250 WRITE_ONCE(sk->sk_busy_poll_budget, val); 1251 return 0; 1252 #endif 1253 case SO_MAX_PACING_RATE: 1254 { 1255 unsigned long ulval = (val == ~0U) ? ~0UL : (unsigned int)val; 1256 unsigned long pacing_rate; 1257 1258 if (sizeof(ulval) != sizeof(val) && 1259 optlen >= sizeof(ulval) && 1260 copy_from_sockptr(&ulval, optval, sizeof(ulval))) { 1261 return -EFAULT; 1262 } 1263 if (ulval != ~0UL) 1264 cmpxchg(&sk->sk_pacing_status, 1265 SK_PACING_NONE, 1266 SK_PACING_NEEDED); 1267 /* Pairs with READ_ONCE() from sk_getsockopt() */ 1268 WRITE_ONCE(sk->sk_max_pacing_rate, ulval); 1269 pacing_rate = READ_ONCE(sk->sk_pacing_rate); 1270 if (ulval < pacing_rate) 1271 WRITE_ONCE(sk->sk_pacing_rate, ulval); 1272 return 0; 1273 } 1274 case SO_TXREHASH: 1275 if (!sk_is_tcp(sk)) 1276 return -EOPNOTSUPP; 1277 if (val < -1 || val > 1) 1278 return -EINVAL; 1279 if ((u8)val == SOCK_TXREHASH_DEFAULT) 1280 val = READ_ONCE(sock_net(sk)->core.sysctl_txrehash); 1281 /* Paired with READ_ONCE() in tcp_rtx_synack() 1282 * and sk_getsockopt(). 1283 */ 1284 WRITE_ONCE(sk->sk_txrehash, (u8)val); 1285 return 0; 1286 case SO_PEEK_OFF: 1287 { 1288 int (*set_peek_off)(struct sock *sk, int val); 1289 1290 set_peek_off = READ_ONCE(sock->ops)->set_peek_off; 1291 if (set_peek_off) 1292 ret = set_peek_off(sk, val); 1293 else 1294 ret = -EOPNOTSUPP; 1295 return ret; 1296 } 1297 #ifdef CONFIG_PAGE_POOL 1298 case SO_DEVMEM_DONTNEED: 1299 return sock_devmem_dontneed(sk, optval, optlen); 1300 #endif 1301 case SO_SNDTIMEO_OLD: 1302 case SO_SNDTIMEO_NEW: 1303 return sock_set_timeout(&sk->sk_sndtimeo, optval, 1304 optlen, optname == SO_SNDTIMEO_OLD); 1305 case SO_RCVTIMEO_OLD: 1306 case SO_RCVTIMEO_NEW: 1307 return sock_set_timeout(&sk->sk_rcvtimeo, optval, 1308 optlen, optname == SO_RCVTIMEO_OLD); 1309 } 1310 1311 sockopt_lock_sock(sk); 1312 1313 switch (optname) { 1314 case SO_DEBUG: 1315 if (val && !sockopt_capable(CAP_NET_ADMIN)) 1316 ret = -EACCES; 1317 else 1318 sock_valbool_flag(sk, SOCK_DBG, valbool); 1319 break; 1320 case SO_REUSEADDR: 1321 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE); 1322 break; 1323 case SO_REUSEPORT: 1324 if (valbool && !sk_is_inet(sk)) 1325 ret = -EOPNOTSUPP; 1326 else 1327 sk->sk_reuseport = valbool; 1328 break; 1329 case SO_DONTROUTE: 1330 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool); 1331 sk_dst_reset(sk); 1332 break; 1333 case SO_BROADCAST: 1334 sock_valbool_flag(sk, SOCK_BROADCAST, valbool); 1335 break; 1336 case SO_SNDBUF: 1337 /* Don't error on this BSD doesn't and if you think 1338 * about it this is right. Otherwise apps have to 1339 * play 'guess the biggest size' games. RCVBUF/SNDBUF 1340 * are treated in BSD as hints 1341 */ 1342 val = min_t(u32, val, READ_ONCE(sysctl_wmem_max)); 1343 set_sndbuf: 1344 /* Ensure val * 2 fits into an int, to prevent max_t() 1345 * from treating it as a negative value. 1346 */ 1347 val = min_t(int, val, INT_MAX / 2); 1348 sk->sk_userlocks |= SOCK_SNDBUF_LOCK; 1349 WRITE_ONCE(sk->sk_sndbuf, 1350 max_t(int, val * 2, SOCK_MIN_SNDBUF)); 1351 /* Wake up sending tasks if we upped the value. */ 1352 sk->sk_write_space(sk); 1353 break; 1354 1355 case SO_SNDBUFFORCE: 1356 if (!sockopt_capable(CAP_NET_ADMIN)) { 1357 ret = -EPERM; 1358 break; 1359 } 1360 1361 /* No negative values (to prevent underflow, as val will be 1362 * multiplied by 2). 1363 */ 1364 if (val < 0) 1365 val = 0; 1366 goto set_sndbuf; 1367 1368 case SO_RCVBUF: 1369 /* Don't error on this BSD doesn't and if you think 1370 * about it this is right. Otherwise apps have to 1371 * play 'guess the biggest size' games. RCVBUF/SNDBUF 1372 * are treated in BSD as hints 1373 */ 1374 __sock_set_rcvbuf(sk, min_t(u32, val, READ_ONCE(sysctl_rmem_max))); 1375 break; 1376 1377 case SO_RCVBUFFORCE: 1378 if (!sockopt_capable(CAP_NET_ADMIN)) { 1379 ret = -EPERM; 1380 break; 1381 } 1382 1383 /* No negative values (to prevent underflow, as val will be 1384 * multiplied by 2). 1385 */ 1386 __sock_set_rcvbuf(sk, max(val, 0)); 1387 break; 1388 1389 case SO_KEEPALIVE: 1390 if (sk->sk_prot->keepalive) 1391 sk->sk_prot->keepalive(sk, valbool); 1392 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool); 1393 break; 1394 1395 case SO_OOBINLINE: 1396 sock_valbool_flag(sk, SOCK_URGINLINE, valbool); 1397 break; 1398 1399 case SO_NO_CHECK: 1400 sk->sk_no_check_tx = valbool; 1401 break; 1402 1403 case SO_LINGER: 1404 if (optlen < sizeof(ling)) { 1405 ret = -EINVAL; /* 1003.1g */ 1406 break; 1407 } 1408 if (copy_from_sockptr(&ling, optval, sizeof(ling))) { 1409 ret = -EFAULT; 1410 break; 1411 } 1412 if (!ling.l_onoff) { 1413 sock_reset_flag(sk, SOCK_LINGER); 1414 } else { 1415 unsigned long t_sec = ling.l_linger; 1416 1417 if (t_sec >= MAX_SCHEDULE_TIMEOUT / HZ) 1418 WRITE_ONCE(sk->sk_lingertime, MAX_SCHEDULE_TIMEOUT); 1419 else 1420 WRITE_ONCE(sk->sk_lingertime, t_sec * HZ); 1421 sock_set_flag(sk, SOCK_LINGER); 1422 } 1423 break; 1424 1425 case SO_BSDCOMPAT: 1426 break; 1427 1428 case SO_TIMESTAMP_OLD: 1429 case SO_TIMESTAMP_NEW: 1430 case SO_TIMESTAMPNS_OLD: 1431 case SO_TIMESTAMPNS_NEW: 1432 sock_set_timestamp(sk, optname, valbool); 1433 break; 1434 1435 case SO_TIMESTAMPING_NEW: 1436 case SO_TIMESTAMPING_OLD: 1437 if (optlen == sizeof(timestamping)) { 1438 if (copy_from_sockptr(×tamping, optval, 1439 sizeof(timestamping))) { 1440 ret = -EFAULT; 1441 break; 1442 } 1443 } else { 1444 memset(×tamping, 0, sizeof(timestamping)); 1445 timestamping.flags = val; 1446 } 1447 ret = sock_set_timestamping(sk, optname, timestamping); 1448 break; 1449 1450 case SO_RCVLOWAT: 1451 { 1452 int (*set_rcvlowat)(struct sock *sk, int val) = NULL; 1453 1454 if (val < 0) 1455 val = INT_MAX; 1456 if (sock) 1457 set_rcvlowat = READ_ONCE(sock->ops)->set_rcvlowat; 1458 if (set_rcvlowat) 1459 ret = set_rcvlowat(sk, val); 1460 else 1461 WRITE_ONCE(sk->sk_rcvlowat, val ? : 1); 1462 break; 1463 } 1464 case SO_ATTACH_FILTER: { 1465 struct sock_fprog fprog; 1466 1467 if (sk_is_tcp(sk) && 1468 !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1469 ret = -EPERM; 1470 break; 1471 } 1472 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen); 1473 if (!ret) 1474 ret = sk_attach_filter(&fprog, sk); 1475 break; 1476 } 1477 case SO_ATTACH_BPF: 1478 ret = -EINVAL; 1479 if (optlen == sizeof(u32)) { 1480 u32 ufd; 1481 1482 ret = -EFAULT; 1483 if (copy_from_sockptr(&ufd, optval, sizeof(ufd))) 1484 break; 1485 1486 ret = sk_attach_bpf(ufd, sk); 1487 } 1488 break; 1489 1490 case SO_ATTACH_REUSEPORT_CBPF: { 1491 struct sock_fprog fprog; 1492 1493 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen); 1494 if (!ret) 1495 ret = sk_reuseport_attach_filter(&fprog, sk); 1496 break; 1497 } 1498 case SO_ATTACH_REUSEPORT_EBPF: 1499 ret = -EINVAL; 1500 if (optlen == sizeof(u32)) { 1501 u32 ufd; 1502 1503 ret = -EFAULT; 1504 if (copy_from_sockptr(&ufd, optval, sizeof(ufd))) 1505 break; 1506 1507 ret = sk_reuseport_attach_bpf(ufd, sk); 1508 } 1509 break; 1510 1511 case SO_DETACH_REUSEPORT_BPF: 1512 ret = reuseport_detach_prog(sk); 1513 break; 1514 1515 case SO_DETACH_FILTER: 1516 ret = sk_detach_filter(sk); 1517 break; 1518 1519 case SO_LOCK_FILTER: 1520 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool) 1521 ret = -EPERM; 1522 else 1523 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool); 1524 break; 1525 1526 case SO_MARK: 1527 if (!sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && 1528 !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1529 ret = -EPERM; 1530 break; 1531 } 1532 1533 __sock_set_mark(sk, val); 1534 break; 1535 case SO_RCVMARK: 1536 sock_valbool_flag(sk, SOCK_RCVMARK, valbool); 1537 break; 1538 1539 case SO_RCVPRIORITY: 1540 sock_valbool_flag(sk, SOCK_RCVPRIORITY, valbool); 1541 break; 1542 1543 case SO_RXQ_OVFL: 1544 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool); 1545 break; 1546 1547 case SO_WIFI_STATUS: 1548 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool); 1549 break; 1550 1551 case SO_NOFCS: 1552 sock_valbool_flag(sk, SOCK_NOFCS, valbool); 1553 break; 1554 1555 case SO_SELECT_ERR_QUEUE: 1556 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool); 1557 break; 1558 1559 case SO_PASSCRED: 1560 if (sk_may_scm_recv(sk)) 1561 sk->sk_scm_credentials = valbool; 1562 else 1563 ret = -EOPNOTSUPP; 1564 break; 1565 1566 case SO_PASSSEC: 1567 if (IS_ENABLED(CONFIG_SECURITY_NETWORK) && sk_may_scm_recv(sk)) 1568 sk->sk_scm_security = valbool; 1569 else 1570 ret = -EOPNOTSUPP; 1571 break; 1572 1573 case SO_PASSPIDFD: 1574 if (sk_is_unix(sk)) 1575 sk->sk_scm_pidfd = valbool; 1576 else 1577 ret = -EOPNOTSUPP; 1578 break; 1579 1580 case SO_PASSRIGHTS: 1581 if (sk_is_unix(sk)) 1582 sk->sk_scm_rights = valbool; 1583 else 1584 ret = -EOPNOTSUPP; 1585 break; 1586 1587 case SO_INCOMING_CPU: 1588 reuseport_update_incoming_cpu(sk, val); 1589 break; 1590 1591 case SO_CNX_ADVICE: 1592 if (val == 1) 1593 dst_negative_advice(sk); 1594 break; 1595 1596 case SO_ZEROCOPY: 1597 if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) { 1598 if (!(sk_is_tcp(sk) || 1599 (sk->sk_type == SOCK_DGRAM && 1600 sk->sk_protocol == IPPROTO_UDP))) 1601 ret = -EOPNOTSUPP; 1602 } else if (sk->sk_family != PF_RDS) { 1603 ret = -EOPNOTSUPP; 1604 } 1605 if (!ret) { 1606 if (val < 0 || val > 1) 1607 ret = -EINVAL; 1608 else 1609 sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool); 1610 } 1611 break; 1612 1613 case SO_TXTIME: 1614 if (optlen != sizeof(struct sock_txtime)) { 1615 ret = -EINVAL; 1616 break; 1617 } else if (copy_from_sockptr(&sk_txtime, optval, 1618 sizeof(struct sock_txtime))) { 1619 ret = -EFAULT; 1620 break; 1621 } else if (sk_txtime.flags & ~SOF_TXTIME_FLAGS_MASK) { 1622 ret = -EINVAL; 1623 break; 1624 } 1625 /* CLOCK_MONOTONIC is only used by sch_fq, and this packet 1626 * scheduler has enough safe guards. 1627 */ 1628 if (sk_txtime.clockid != CLOCK_MONOTONIC && 1629 !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1630 ret = -EPERM; 1631 break; 1632 } 1633 1634 ret = sockopt_validate_clockid(sk_txtime.clockid); 1635 if (ret) 1636 break; 1637 1638 sock_valbool_flag(sk, SOCK_TXTIME, true); 1639 sk->sk_clockid = sk_txtime.clockid; 1640 sk->sk_txtime_deadline_mode = 1641 !!(sk_txtime.flags & SOF_TXTIME_DEADLINE_MODE); 1642 sk->sk_txtime_report_errors = 1643 !!(sk_txtime.flags & SOF_TXTIME_REPORT_ERRORS); 1644 break; 1645 1646 case SO_BINDTOIFINDEX: 1647 ret = sock_bindtoindex_locked(sk, val); 1648 break; 1649 1650 case SO_BUF_LOCK: 1651 if (val & ~SOCK_BUF_LOCK_MASK) { 1652 ret = -EINVAL; 1653 break; 1654 } 1655 sk->sk_userlocks = val | (sk->sk_userlocks & 1656 ~SOCK_BUF_LOCK_MASK); 1657 break; 1658 1659 case SO_RESERVE_MEM: 1660 { 1661 int delta; 1662 1663 if (val < 0) { 1664 ret = -EINVAL; 1665 break; 1666 } 1667 1668 delta = val - sk->sk_reserved_mem; 1669 if (delta < 0) 1670 sock_release_reserved_memory(sk, -delta); 1671 else 1672 ret = sock_reserve_memory(sk, delta); 1673 break; 1674 } 1675 1676 default: 1677 ret = -ENOPROTOOPT; 1678 break; 1679 } 1680 sockopt_release_sock(sk); 1681 return ret; 1682 } 1683 1684 int sock_setsockopt(struct socket *sock, int level, int optname, 1685 sockptr_t optval, unsigned int optlen) 1686 { 1687 return sk_setsockopt(sock->sk, level, optname, 1688 optval, optlen); 1689 } 1690 EXPORT_SYMBOL(sock_setsockopt); 1691 1692 static const struct cred *sk_get_peer_cred(struct sock *sk) 1693 { 1694 const struct cred *cred; 1695 1696 spin_lock(&sk->sk_peer_lock); 1697 cred = get_cred(sk->sk_peer_cred); 1698 spin_unlock(&sk->sk_peer_lock); 1699 1700 return cred; 1701 } 1702 1703 static void cred_to_ucred(struct pid *pid, const struct cred *cred, 1704 struct ucred *ucred) 1705 { 1706 ucred->pid = pid_vnr(pid); 1707 ucred->uid = ucred->gid = -1; 1708 if (cred) { 1709 struct user_namespace *current_ns = current_user_ns(); 1710 1711 ucred->uid = from_kuid_munged(current_ns, cred->euid); 1712 ucred->gid = from_kgid_munged(current_ns, cred->egid); 1713 } 1714 } 1715 1716 static int groups_to_user(sockptr_t dst, const struct group_info *src) 1717 { 1718 struct user_namespace *user_ns = current_user_ns(); 1719 int i; 1720 1721 for (i = 0; i < src->ngroups; i++) { 1722 gid_t gid = from_kgid_munged(user_ns, src->gid[i]); 1723 1724 if (copy_to_sockptr_offset(dst, i * sizeof(gid), &gid, sizeof(gid))) 1725 return -EFAULT; 1726 } 1727 1728 return 0; 1729 } 1730 1731 int sk_getsockopt(struct sock *sk, int level, int optname, 1732 sockptr_t optval, sockptr_t optlen) 1733 { 1734 struct socket *sock = sk->sk_socket; 1735 1736 union { 1737 int val; 1738 u64 val64; 1739 unsigned long ulval; 1740 struct linger ling; 1741 struct old_timeval32 tm32; 1742 struct __kernel_old_timeval tm; 1743 struct __kernel_sock_timeval stm; 1744 struct sock_txtime txtime; 1745 struct so_timestamping timestamping; 1746 } v; 1747 1748 int lv = sizeof(int); 1749 int len; 1750 1751 if (copy_from_sockptr(&len, optlen, sizeof(int))) 1752 return -EFAULT; 1753 if (len < 0) 1754 return -EINVAL; 1755 1756 memset(&v, 0, sizeof(v)); 1757 1758 switch (optname) { 1759 case SO_DEBUG: 1760 v.val = sock_flag(sk, SOCK_DBG); 1761 break; 1762 1763 case SO_DONTROUTE: 1764 v.val = sock_flag(sk, SOCK_LOCALROUTE); 1765 break; 1766 1767 case SO_BROADCAST: 1768 v.val = sock_flag(sk, SOCK_BROADCAST); 1769 break; 1770 1771 case SO_SNDBUF: 1772 v.val = READ_ONCE(sk->sk_sndbuf); 1773 break; 1774 1775 case SO_RCVBUF: 1776 v.val = READ_ONCE(sk->sk_rcvbuf); 1777 break; 1778 1779 case SO_REUSEADDR: 1780 v.val = sk->sk_reuse; 1781 break; 1782 1783 case SO_REUSEPORT: 1784 v.val = sk->sk_reuseport; 1785 break; 1786 1787 case SO_KEEPALIVE: 1788 v.val = sock_flag(sk, SOCK_KEEPOPEN); 1789 break; 1790 1791 case SO_TYPE: 1792 v.val = sk->sk_type; 1793 break; 1794 1795 case SO_PROTOCOL: 1796 v.val = sk->sk_protocol; 1797 break; 1798 1799 case SO_DOMAIN: 1800 v.val = sk->sk_family; 1801 break; 1802 1803 case SO_ERROR: 1804 v.val = -sock_error(sk); 1805 if (v.val == 0) 1806 v.val = xchg(&sk->sk_err_soft, 0); 1807 break; 1808 1809 case SO_OOBINLINE: 1810 v.val = sock_flag(sk, SOCK_URGINLINE); 1811 break; 1812 1813 case SO_NO_CHECK: 1814 v.val = sk->sk_no_check_tx; 1815 break; 1816 1817 case SO_PRIORITY: 1818 v.val = READ_ONCE(sk->sk_priority); 1819 break; 1820 1821 case SO_LINGER: 1822 lv = sizeof(v.ling); 1823 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER); 1824 v.ling.l_linger = READ_ONCE(sk->sk_lingertime) / HZ; 1825 break; 1826 1827 case SO_BSDCOMPAT: 1828 break; 1829 1830 case SO_TIMESTAMP_OLD: 1831 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && 1832 !sock_flag(sk, SOCK_TSTAMP_NEW) && 1833 !sock_flag(sk, SOCK_RCVTSTAMPNS); 1834 break; 1835 1836 case SO_TIMESTAMPNS_OLD: 1837 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && !sock_flag(sk, SOCK_TSTAMP_NEW); 1838 break; 1839 1840 case SO_TIMESTAMP_NEW: 1841 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && sock_flag(sk, SOCK_TSTAMP_NEW); 1842 break; 1843 1844 case SO_TIMESTAMPNS_NEW: 1845 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && sock_flag(sk, SOCK_TSTAMP_NEW); 1846 break; 1847 1848 case SO_TIMESTAMPING_OLD: 1849 case SO_TIMESTAMPING_NEW: 1850 lv = sizeof(v.timestamping); 1851 /* For the later-added case SO_TIMESTAMPING_NEW: Be strict about only 1852 * returning the flags when they were set through the same option. 1853 * Don't change the beviour for the old case SO_TIMESTAMPING_OLD. 1854 */ 1855 if (optname == SO_TIMESTAMPING_OLD || sock_flag(sk, SOCK_TSTAMP_NEW)) { 1856 v.timestamping.flags = READ_ONCE(sk->sk_tsflags); 1857 v.timestamping.bind_phc = READ_ONCE(sk->sk_bind_phc); 1858 } 1859 break; 1860 1861 case SO_RCVTIMEO_OLD: 1862 case SO_RCVTIMEO_NEW: 1863 lv = sock_get_timeout(READ_ONCE(sk->sk_rcvtimeo), &v, 1864 SO_RCVTIMEO_OLD == optname); 1865 break; 1866 1867 case SO_SNDTIMEO_OLD: 1868 case SO_SNDTIMEO_NEW: 1869 lv = sock_get_timeout(READ_ONCE(sk->sk_sndtimeo), &v, 1870 SO_SNDTIMEO_OLD == optname); 1871 break; 1872 1873 case SO_RCVLOWAT: 1874 v.val = READ_ONCE(sk->sk_rcvlowat); 1875 break; 1876 1877 case SO_SNDLOWAT: 1878 v.val = 1; 1879 break; 1880 1881 case SO_PASSCRED: 1882 if (!sk_may_scm_recv(sk)) 1883 return -EOPNOTSUPP; 1884 1885 v.val = sk->sk_scm_credentials; 1886 break; 1887 1888 case SO_PASSPIDFD: 1889 if (!sk_is_unix(sk)) 1890 return -EOPNOTSUPP; 1891 1892 v.val = sk->sk_scm_pidfd; 1893 break; 1894 1895 case SO_PASSRIGHTS: 1896 if (!sk_is_unix(sk)) 1897 return -EOPNOTSUPP; 1898 1899 v.val = sk->sk_scm_rights; 1900 break; 1901 1902 case SO_PEERCRED: 1903 { 1904 struct ucred peercred; 1905 if (len > sizeof(peercred)) 1906 len = sizeof(peercred); 1907 1908 spin_lock(&sk->sk_peer_lock); 1909 cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred); 1910 spin_unlock(&sk->sk_peer_lock); 1911 1912 if (copy_to_sockptr(optval, &peercred, len)) 1913 return -EFAULT; 1914 goto lenout; 1915 } 1916 1917 case SO_PEERPIDFD: 1918 { 1919 struct pid *peer_pid; 1920 struct file *pidfd_file = NULL; 1921 unsigned int flags = 0; 1922 int pidfd; 1923 1924 if (len > sizeof(pidfd)) 1925 len = sizeof(pidfd); 1926 1927 spin_lock(&sk->sk_peer_lock); 1928 peer_pid = get_pid(sk->sk_peer_pid); 1929 spin_unlock(&sk->sk_peer_lock); 1930 1931 if (!peer_pid) 1932 return -ENODATA; 1933 1934 /* The use of PIDFD_STALE requires stashing of struct pid 1935 * on pidfs with pidfs_register_pid() and only AF_UNIX 1936 * were prepared for this. 1937 */ 1938 if (sk->sk_family == AF_UNIX) 1939 flags = PIDFD_STALE; 1940 1941 pidfd = pidfd_prepare(peer_pid, flags, &pidfd_file); 1942 put_pid(peer_pid); 1943 if (pidfd < 0) 1944 return pidfd; 1945 1946 if (copy_to_sockptr(optval, &pidfd, len) || 1947 copy_to_sockptr(optlen, &len, sizeof(int))) { 1948 put_unused_fd(pidfd); 1949 fput(pidfd_file); 1950 1951 return -EFAULT; 1952 } 1953 1954 fd_install(pidfd, pidfd_file); 1955 return 0; 1956 } 1957 1958 case SO_PEERGROUPS: 1959 { 1960 const struct cred *cred; 1961 int ret, n; 1962 1963 cred = sk_get_peer_cred(sk); 1964 if (!cred) 1965 return -ENODATA; 1966 1967 n = cred->group_info->ngroups; 1968 if (len < n * sizeof(gid_t)) { 1969 len = n * sizeof(gid_t); 1970 put_cred(cred); 1971 return copy_to_sockptr(optlen, &len, sizeof(int)) ? -EFAULT : -ERANGE; 1972 } 1973 len = n * sizeof(gid_t); 1974 1975 ret = groups_to_user(optval, cred->group_info); 1976 put_cred(cred); 1977 if (ret) 1978 return ret; 1979 goto lenout; 1980 } 1981 1982 case SO_PEERNAME: 1983 { 1984 struct sockaddr_storage address; 1985 1986 lv = READ_ONCE(sock->ops)->getname(sock, (struct sockaddr *)&address, 2); 1987 if (lv < 0) 1988 return -ENOTCONN; 1989 if (lv < len) 1990 return -EINVAL; 1991 if (copy_to_sockptr(optval, &address, len)) 1992 return -EFAULT; 1993 goto lenout; 1994 } 1995 1996 /* Dubious BSD thing... Probably nobody even uses it, but 1997 * the UNIX standard wants it for whatever reason... -DaveM 1998 */ 1999 case SO_ACCEPTCONN: 2000 v.val = sk->sk_state == TCP_LISTEN; 2001 break; 2002 2003 case SO_PASSSEC: 2004 if (!IS_ENABLED(CONFIG_SECURITY_NETWORK) || !sk_may_scm_recv(sk)) 2005 return -EOPNOTSUPP; 2006 2007 v.val = sk->sk_scm_security; 2008 break; 2009 2010 case SO_PEERSEC: 2011 return security_socket_getpeersec_stream(sock, 2012 optval, optlen, len); 2013 2014 case SO_MARK: 2015 v.val = READ_ONCE(sk->sk_mark); 2016 break; 2017 2018 case SO_RCVMARK: 2019 v.val = sock_flag(sk, SOCK_RCVMARK); 2020 break; 2021 2022 case SO_RCVPRIORITY: 2023 v.val = sock_flag(sk, SOCK_RCVPRIORITY); 2024 break; 2025 2026 case SO_RXQ_OVFL: 2027 v.val = sock_flag(sk, SOCK_RXQ_OVFL); 2028 break; 2029 2030 case SO_WIFI_STATUS: 2031 v.val = sock_flag(sk, SOCK_WIFI_STATUS); 2032 break; 2033 2034 case SO_PEEK_OFF: 2035 if (!READ_ONCE(sock->ops)->set_peek_off) 2036 return -EOPNOTSUPP; 2037 2038 v.val = READ_ONCE(sk->sk_peek_off); 2039 break; 2040 case SO_NOFCS: 2041 v.val = sock_flag(sk, SOCK_NOFCS); 2042 break; 2043 2044 case SO_BINDTODEVICE: 2045 return sock_getbindtodevice(sk, optval, optlen, len); 2046 2047 case SO_GET_FILTER: 2048 len = sk_get_filter(sk, optval, len); 2049 if (len < 0) 2050 return len; 2051 2052 goto lenout; 2053 2054 case SO_LOCK_FILTER: 2055 v.val = sock_flag(sk, SOCK_FILTER_LOCKED); 2056 break; 2057 2058 case SO_BPF_EXTENSIONS: 2059 v.val = bpf_tell_extensions(); 2060 break; 2061 2062 case SO_SELECT_ERR_QUEUE: 2063 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE); 2064 break; 2065 2066 #ifdef CONFIG_NET_RX_BUSY_POLL 2067 case SO_BUSY_POLL: 2068 v.val = READ_ONCE(sk->sk_ll_usec); 2069 break; 2070 case SO_PREFER_BUSY_POLL: 2071 v.val = READ_ONCE(sk->sk_prefer_busy_poll); 2072 break; 2073 #endif 2074 2075 case SO_MAX_PACING_RATE: 2076 /* The READ_ONCE() pair with the WRITE_ONCE() in sk_setsockopt() */ 2077 if (sizeof(v.ulval) != sizeof(v.val) && len >= sizeof(v.ulval)) { 2078 lv = sizeof(v.ulval); 2079 v.ulval = READ_ONCE(sk->sk_max_pacing_rate); 2080 } else { 2081 /* 32bit version */ 2082 v.val = min_t(unsigned long, ~0U, 2083 READ_ONCE(sk->sk_max_pacing_rate)); 2084 } 2085 break; 2086 2087 case SO_INCOMING_CPU: 2088 v.val = READ_ONCE(sk->sk_incoming_cpu); 2089 break; 2090 2091 case SO_MEMINFO: 2092 { 2093 u32 meminfo[SK_MEMINFO_VARS]; 2094 2095 sk_get_meminfo(sk, meminfo); 2096 2097 len = min_t(unsigned int, len, sizeof(meminfo)); 2098 if (copy_to_sockptr(optval, &meminfo, len)) 2099 return -EFAULT; 2100 2101 goto lenout; 2102 } 2103 2104 #ifdef CONFIG_NET_RX_BUSY_POLL 2105 case SO_INCOMING_NAPI_ID: 2106 v.val = READ_ONCE(sk->sk_napi_id); 2107 2108 /* aggregate non-NAPI IDs down to 0 */ 2109 if (!napi_id_valid(v.val)) 2110 v.val = 0; 2111 2112 break; 2113 #endif 2114 2115 case SO_COOKIE: 2116 lv = sizeof(u64); 2117 if (len < lv) 2118 return -EINVAL; 2119 v.val64 = sock_gen_cookie(sk); 2120 break; 2121 2122 case SO_ZEROCOPY: 2123 v.val = sock_flag(sk, SOCK_ZEROCOPY); 2124 break; 2125 2126 case SO_TXTIME: 2127 lv = sizeof(v.txtime); 2128 v.txtime.clockid = sk->sk_clockid; 2129 v.txtime.flags |= sk->sk_txtime_deadline_mode ? 2130 SOF_TXTIME_DEADLINE_MODE : 0; 2131 v.txtime.flags |= sk->sk_txtime_report_errors ? 2132 SOF_TXTIME_REPORT_ERRORS : 0; 2133 break; 2134 2135 case SO_BINDTOIFINDEX: 2136 v.val = READ_ONCE(sk->sk_bound_dev_if); 2137 break; 2138 2139 case SO_NETNS_COOKIE: 2140 lv = sizeof(u64); 2141 if (len != lv) 2142 return -EINVAL; 2143 v.val64 = sock_net(sk)->net_cookie; 2144 break; 2145 2146 case SO_BUF_LOCK: 2147 v.val = sk->sk_userlocks & SOCK_BUF_LOCK_MASK; 2148 break; 2149 2150 case SO_RESERVE_MEM: 2151 v.val = READ_ONCE(sk->sk_reserved_mem); 2152 break; 2153 2154 case SO_TXREHASH: 2155 if (!sk_is_tcp(sk)) 2156 return -EOPNOTSUPP; 2157 2158 /* Paired with WRITE_ONCE() in sk_setsockopt() */ 2159 v.val = READ_ONCE(sk->sk_txrehash); 2160 break; 2161 2162 default: 2163 /* We implement the SO_SNDLOWAT etc to not be settable 2164 * (1003.1g 7). 2165 */ 2166 return -ENOPROTOOPT; 2167 } 2168 2169 if (len > lv) 2170 len = lv; 2171 if (copy_to_sockptr(optval, &v, len)) 2172 return -EFAULT; 2173 lenout: 2174 if (copy_to_sockptr(optlen, &len, sizeof(int))) 2175 return -EFAULT; 2176 return 0; 2177 } 2178 2179 /* 2180 * Initialize an sk_lock. 2181 * 2182 * (We also register the sk_lock with the lock validator.) 2183 */ 2184 static inline void sock_lock_init(struct sock *sk) 2185 { 2186 sk_owner_clear(sk); 2187 2188 if (sk->sk_kern_sock) 2189 sock_lock_init_class_and_name( 2190 sk, 2191 af_family_kern_slock_key_strings[sk->sk_family], 2192 af_family_kern_slock_keys + sk->sk_family, 2193 af_family_kern_key_strings[sk->sk_family], 2194 af_family_kern_keys + sk->sk_family); 2195 else 2196 sock_lock_init_class_and_name( 2197 sk, 2198 af_family_slock_key_strings[sk->sk_family], 2199 af_family_slock_keys + sk->sk_family, 2200 af_family_key_strings[sk->sk_family], 2201 af_family_keys + sk->sk_family); 2202 } 2203 2204 /* 2205 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet, 2206 * even temporarily, because of RCU lookups. sk_node should also be left as is. 2207 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end 2208 */ 2209 static void sock_copy(struct sock *nsk, const struct sock *osk) 2210 { 2211 const struct proto *prot = READ_ONCE(osk->sk_prot); 2212 #ifdef CONFIG_SECURITY_NETWORK 2213 void *sptr = nsk->sk_security; 2214 #endif 2215 2216 /* If we move sk_tx_queue_mapping out of the private section, 2217 * we must check if sk_tx_queue_clear() is called after 2218 * sock_copy() in sk_clone_lock(). 2219 */ 2220 BUILD_BUG_ON(offsetof(struct sock, sk_tx_queue_mapping) < 2221 offsetof(struct sock, sk_dontcopy_begin) || 2222 offsetof(struct sock, sk_tx_queue_mapping) >= 2223 offsetof(struct sock, sk_dontcopy_end)); 2224 2225 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin)); 2226 2227 unsafe_memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end, 2228 prot->obj_size - offsetof(struct sock, sk_dontcopy_end), 2229 /* alloc is larger than struct, see sk_prot_alloc() */); 2230 2231 #ifdef CONFIG_SECURITY_NETWORK 2232 nsk->sk_security = sptr; 2233 security_sk_clone(osk, nsk); 2234 #endif 2235 } 2236 2237 static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority, 2238 int family) 2239 { 2240 struct sock *sk; 2241 struct kmem_cache *slab; 2242 2243 slab = prot->slab; 2244 if (slab != NULL) { 2245 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO); 2246 if (!sk) 2247 return sk; 2248 if (want_init_on_alloc(priority)) 2249 sk_prot_clear_nulls(sk, prot->obj_size); 2250 } else 2251 sk = kmalloc(prot->obj_size, priority); 2252 2253 if (sk != NULL) { 2254 if (security_sk_alloc(sk, family, priority)) 2255 goto out_free; 2256 2257 if (!try_module_get(prot->owner)) 2258 goto out_free_sec; 2259 } 2260 2261 return sk; 2262 2263 out_free_sec: 2264 security_sk_free(sk); 2265 out_free: 2266 if (slab != NULL) 2267 kmem_cache_free(slab, sk); 2268 else 2269 kfree(sk); 2270 return NULL; 2271 } 2272 2273 static void sk_prot_free(struct proto *prot, struct sock *sk) 2274 { 2275 struct kmem_cache *slab; 2276 struct module *owner; 2277 2278 owner = prot->owner; 2279 slab = prot->slab; 2280 2281 cgroup_sk_free(&sk->sk_cgrp_data); 2282 mem_cgroup_sk_free(sk); 2283 security_sk_free(sk); 2284 2285 sk_owner_put(sk); 2286 2287 if (slab != NULL) 2288 kmem_cache_free(slab, sk); 2289 else 2290 kfree(sk); 2291 module_put(owner); 2292 } 2293 2294 /** 2295 * sk_alloc - All socket objects are allocated here 2296 * @net: the applicable net namespace 2297 * @family: protocol family 2298 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 2299 * @prot: struct proto associated with this new sock instance 2300 * @kern: is this to be a kernel socket? 2301 */ 2302 struct sock *sk_alloc(struct net *net, int family, gfp_t priority, 2303 struct proto *prot, int kern) 2304 { 2305 struct sock *sk; 2306 2307 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family); 2308 if (sk) { 2309 sk->sk_family = family; 2310 /* 2311 * See comment in struct sock definition to understand 2312 * why we need sk_prot_creator -acme 2313 */ 2314 sk->sk_prot = sk->sk_prot_creator = prot; 2315 2316 if (READ_ONCE(net->core.sysctl_bypass_prot_mem)) 2317 sk->sk_bypass_prot_mem = 1; 2318 2319 sk->sk_kern_sock = kern; 2320 sock_lock_init(sk); 2321 2322 sk->sk_net_refcnt = kern ? 0 : 1; 2323 if (likely(sk->sk_net_refcnt)) { 2324 get_net_track(net, &sk->ns_tracker, priority); 2325 sock_inuse_add(net, 1); 2326 } else { 2327 net_passive_inc(net); 2328 __netns_tracker_alloc(net, &sk->ns_tracker, 2329 false, priority); 2330 } 2331 2332 sock_net_set(sk, net); 2333 refcount_set(&sk->sk_wmem_alloc, SK_WMEM_ALLOC_BIAS); 2334 2335 mem_cgroup_sk_alloc(sk); 2336 cgroup_sk_alloc(&sk->sk_cgrp_data); 2337 sock_update_classid(&sk->sk_cgrp_data); 2338 sock_update_netprioidx(&sk->sk_cgrp_data); 2339 sk_tx_queue_clear(sk); 2340 } 2341 2342 return sk; 2343 } 2344 EXPORT_SYMBOL(sk_alloc); 2345 2346 /* Sockets having SOCK_RCU_FREE will call this function after one RCU 2347 * grace period. This is the case for UDP sockets and TCP listeners. 2348 */ 2349 static void __sk_destruct(struct rcu_head *head) 2350 { 2351 struct sock *sk = container_of(head, struct sock, sk_rcu); 2352 struct net *net = sock_net(sk); 2353 struct sk_filter *filter; 2354 2355 if (sk->sk_destruct) 2356 sk->sk_destruct(sk); 2357 2358 filter = rcu_dereference_check(sk->sk_filter, 2359 refcount_read(&sk->sk_wmem_alloc) == 0); 2360 if (filter) { 2361 sk_filter_uncharge(sk, filter); 2362 RCU_INIT_POINTER(sk->sk_filter, NULL); 2363 } 2364 2365 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP); 2366 2367 #ifdef CONFIG_BPF_SYSCALL 2368 bpf_sk_storage_free(sk); 2369 #endif 2370 2371 if (atomic_read(&sk->sk_omem_alloc)) 2372 pr_debug("%s: optmem leakage (%d bytes) detected\n", 2373 __func__, atomic_read(&sk->sk_omem_alloc)); 2374 2375 if (sk->sk_frag.page) { 2376 put_page(sk->sk_frag.page); 2377 sk->sk_frag.page = NULL; 2378 } 2379 2380 /* We do not need to acquire sk->sk_peer_lock, we are the last user. */ 2381 put_cred(sk->sk_peer_cred); 2382 put_pid(sk->sk_peer_pid); 2383 2384 if (likely(sk->sk_net_refcnt)) { 2385 put_net_track(net, &sk->ns_tracker); 2386 } else { 2387 __netns_tracker_free(net, &sk->ns_tracker, false); 2388 net_passive_dec(net); 2389 } 2390 sk_prot_free(sk->sk_prot_creator, sk); 2391 } 2392 2393 void sk_net_refcnt_upgrade(struct sock *sk) 2394 { 2395 struct net *net = sock_net(sk); 2396 2397 WARN_ON_ONCE(sk->sk_net_refcnt); 2398 __netns_tracker_free(net, &sk->ns_tracker, false); 2399 net_passive_dec(net); 2400 sk->sk_net_refcnt = 1; 2401 get_net_track(net, &sk->ns_tracker, GFP_KERNEL); 2402 sock_inuse_add(net, 1); 2403 } 2404 EXPORT_SYMBOL_GPL(sk_net_refcnt_upgrade); 2405 2406 void sk_destruct(struct sock *sk) 2407 { 2408 bool use_call_rcu = sock_flag(sk, SOCK_RCU_FREE); 2409 2410 if (rcu_access_pointer(sk->sk_reuseport_cb)) { 2411 reuseport_detach_sock(sk); 2412 use_call_rcu = true; 2413 } 2414 2415 if (use_call_rcu) 2416 call_rcu(&sk->sk_rcu, __sk_destruct); 2417 else 2418 __sk_destruct(&sk->sk_rcu); 2419 } 2420 2421 static void __sk_free(struct sock *sk) 2422 { 2423 if (likely(sk->sk_net_refcnt)) 2424 sock_inuse_add(sock_net(sk), -1); 2425 2426 if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk))) 2427 sock_diag_broadcast_destroy(sk); 2428 else 2429 sk_destruct(sk); 2430 } 2431 2432 void sk_free(struct sock *sk) 2433 { 2434 /* 2435 * We subtract one from sk_wmem_alloc and can know if 2436 * some packets are still in some tx queue. 2437 * If not null, sock_wfree() will call __sk_free(sk) later 2438 */ 2439 if (refcount_dec_and_test(&sk->sk_wmem_alloc)) 2440 __sk_free(sk); 2441 } 2442 EXPORT_SYMBOL(sk_free); 2443 2444 static void sk_init_common(struct sock *sk) 2445 { 2446 skb_queue_head_init(&sk->sk_receive_queue); 2447 skb_queue_head_init(&sk->sk_write_queue); 2448 skb_queue_head_init(&sk->sk_error_queue); 2449 2450 rwlock_init(&sk->sk_callback_lock); 2451 lockdep_set_class_and_name(&sk->sk_receive_queue.lock, 2452 af_rlock_keys + sk->sk_family, 2453 af_family_rlock_key_strings[sk->sk_family]); 2454 lockdep_set_class_and_name(&sk->sk_write_queue.lock, 2455 af_wlock_keys + sk->sk_family, 2456 af_family_wlock_key_strings[sk->sk_family]); 2457 lockdep_set_class_and_name(&sk->sk_error_queue.lock, 2458 af_elock_keys + sk->sk_family, 2459 af_family_elock_key_strings[sk->sk_family]); 2460 if (sk->sk_kern_sock) 2461 lockdep_set_class_and_name(&sk->sk_callback_lock, 2462 af_kern_callback_keys + sk->sk_family, 2463 af_family_kern_clock_key_strings[sk->sk_family]); 2464 else 2465 lockdep_set_class_and_name(&sk->sk_callback_lock, 2466 af_callback_keys + sk->sk_family, 2467 af_family_clock_key_strings[sk->sk_family]); 2468 } 2469 2470 /** 2471 * sk_clone - clone a socket 2472 * @sk: the socket to clone 2473 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 2474 * @lock: if true, lock the cloned sk 2475 * 2476 * If @lock is true, the clone is locked by bh_lock_sock(), and 2477 * caller must unlock socket even in error path by bh_unlock_sock(). 2478 */ 2479 struct sock *sk_clone(const struct sock *sk, const gfp_t priority, 2480 bool lock) 2481 { 2482 struct proto *prot = READ_ONCE(sk->sk_prot); 2483 struct sk_filter *filter; 2484 bool is_charged = true; 2485 struct sock *newsk; 2486 2487 newsk = sk_prot_alloc(prot, priority, sk->sk_family); 2488 if (!newsk) 2489 goto out; 2490 2491 sock_copy(newsk, sk); 2492 2493 newsk->sk_prot_creator = prot; 2494 #ifdef CONFIG_BPF_SYSCALL 2495 RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL); 2496 #endif 2497 #if IS_ENABLED(CONFIG_INET_PSP) 2498 RCU_INIT_POINTER(newsk->psp_assoc, NULL); 2499 #endif 2500 2501 /* SANITY */ 2502 if (likely(newsk->sk_net_refcnt)) { 2503 get_net_track(sock_net(newsk), &newsk->ns_tracker, priority); 2504 sock_inuse_add(sock_net(newsk), 1); 2505 } else { 2506 /* Kernel sockets are not elevating the struct net refcount. 2507 * Instead, use a tracker to more easily detect if a layer 2508 * is not properly dismantling its kernel sockets at netns 2509 * destroy time. 2510 */ 2511 net_passive_inc(sock_net(newsk)); 2512 __netns_tracker_alloc(sock_net(newsk), &newsk->ns_tracker, 2513 false, priority); 2514 } 2515 2516 sk_node_init(&newsk->sk_node); 2517 sock_lock_init(newsk); 2518 2519 if (lock) 2520 bh_lock_sock(newsk); 2521 2522 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL; 2523 newsk->sk_backlog.len = 0; 2524 2525 atomic_set(&newsk->sk_rmem_alloc, 0); 2526 2527 refcount_set(&newsk->sk_wmem_alloc, SK_WMEM_ALLOC_BIAS); 2528 2529 atomic_set(&newsk->sk_omem_alloc, 0); 2530 sk_init_common(newsk); 2531 2532 newsk->sk_dst_cache = NULL; 2533 newsk->sk_dst_pending_confirm = 0; 2534 newsk->sk_wmem_queued = 0; 2535 newsk->sk_forward_alloc = 0; 2536 newsk->sk_reserved_mem = 0; 2537 DEBUG_NET_WARN_ON_ONCE(newsk->sk_drop_counters); 2538 sk_drops_reset(newsk); 2539 newsk->sk_send_head = NULL; 2540 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK; 2541 atomic_set(&newsk->sk_zckey, 0); 2542 2543 sock_reset_flag(newsk, SOCK_DONE); 2544 2545 #ifdef CONFIG_MEMCG 2546 /* sk->sk_memcg will be populated at accept() time */ 2547 newsk->sk_memcg = NULL; 2548 #endif 2549 2550 cgroup_sk_clone(&newsk->sk_cgrp_data); 2551 2552 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL); 2553 2554 if (sock_needs_netstamp(sk) && newsk->sk_flags & SK_FLAGS_TIMESTAMP) 2555 net_enable_timestamp(); 2556 2557 rcu_read_lock(); 2558 filter = rcu_dereference(sk->sk_filter); 2559 if (filter != NULL) 2560 /* though it's an empty new sock, the charging may fail 2561 * if sysctl_optmem_max was changed between creation of 2562 * original socket and cloning 2563 */ 2564 is_charged = sk_filter_charge(newsk, filter); 2565 RCU_INIT_POINTER(newsk->sk_filter, filter); 2566 rcu_read_unlock(); 2567 2568 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) { 2569 /* We need to make sure that we don't uncharge the new 2570 * socket if we couldn't charge it in the first place 2571 * as otherwise we uncharge the parent's filter. 2572 */ 2573 if (!is_charged) 2574 RCU_INIT_POINTER(newsk->sk_filter, NULL); 2575 2576 goto free; 2577 } 2578 2579 if (bpf_sk_storage_clone(sk, newsk)) 2580 goto free; 2581 2582 /* Clear sk_user_data if parent had the pointer tagged 2583 * as not suitable for copying when cloning. 2584 */ 2585 if (sk_user_data_is_nocopy(newsk)) 2586 newsk->sk_user_data = NULL; 2587 2588 newsk->sk_err = 0; 2589 newsk->sk_err_soft = 0; 2590 newsk->sk_priority = 0; 2591 newsk->sk_incoming_cpu = raw_smp_processor_id(); 2592 2593 /* Before updating sk_refcnt, we must commit prior changes to memory 2594 * (Documentation/RCU/rculist_nulls.rst for details) 2595 */ 2596 smp_wmb(); 2597 refcount_set(&newsk->sk_refcnt, 2); 2598 2599 sk_set_socket(newsk, NULL); 2600 sk_tx_queue_clear(newsk); 2601 sk_rx_queue_clear(newsk); 2602 RCU_INIT_POINTER(newsk->sk_wq, NULL); 2603 2604 if (newsk->sk_prot->sockets_allocated) 2605 sk_sockets_allocated_inc(newsk); 2606 out: 2607 return newsk; 2608 free: 2609 /* It is still raw copy of parent, so invalidate 2610 * destructor and make plain sk_free() 2611 */ 2612 newsk->sk_destruct = NULL; 2613 if (lock) 2614 bh_unlock_sock(newsk); 2615 sk_free(newsk); 2616 newsk = NULL; 2617 goto out; 2618 } 2619 EXPORT_SYMBOL_GPL(sk_clone); 2620 2621 static u32 sk_dst_gso_max_size(struct sock *sk, const struct net_device *dev) 2622 { 2623 bool is_ipv6 = false; 2624 u32 max_size; 2625 2626 #if IS_ENABLED(CONFIG_IPV6) 2627 is_ipv6 = (sk->sk_family == AF_INET6 && 2628 !ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)); 2629 #endif 2630 /* pairs with the WRITE_ONCE() in netif_set_gso(_ipv4)_max_size() */ 2631 max_size = is_ipv6 ? READ_ONCE(dev->gso_max_size) : 2632 READ_ONCE(dev->gso_ipv4_max_size); 2633 if (max_size > GSO_LEGACY_MAX_SIZE && !sk_is_tcp(sk)) 2634 max_size = GSO_LEGACY_MAX_SIZE; 2635 2636 return max_size - (MAX_TCP_HEADER + 1); 2637 } 2638 2639 void sk_setup_caps(struct sock *sk, struct dst_entry *dst) 2640 { 2641 const struct net_device *dev; 2642 u32 max_segs = 1; 2643 2644 rcu_read_lock(); 2645 dev = dst_dev_rcu(dst); 2646 sk->sk_route_caps = dev->features; 2647 if (sk_is_tcp(sk)) { 2648 struct inet_connection_sock *icsk = inet_csk(sk); 2649 2650 sk->sk_route_caps |= NETIF_F_GSO; 2651 icsk->icsk_ack.dst_quick_ack = dst_metric(dst, RTAX_QUICKACK); 2652 } 2653 if (sk->sk_route_caps & NETIF_F_GSO) 2654 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE; 2655 if (unlikely(sk->sk_gso_disabled)) 2656 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 2657 if (sk_can_gso(sk)) { 2658 if (dst->header_len && !xfrm_dst_offload_ok(dst)) { 2659 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 2660 } else { 2661 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM; 2662 sk->sk_gso_max_size = sk_dst_gso_max_size(sk, dev); 2663 /* pairs with the WRITE_ONCE() in netif_set_gso_max_segs() */ 2664 max_segs = max_t(u32, READ_ONCE(dev->gso_max_segs), 1); 2665 } 2666 } 2667 sk->sk_gso_max_segs = max_segs; 2668 sk_dst_set(sk, dst); 2669 rcu_read_unlock(); 2670 } 2671 EXPORT_SYMBOL_GPL(sk_setup_caps); 2672 2673 /* 2674 * Simple resource managers for sockets. 2675 */ 2676 2677 2678 /* 2679 * Write buffer destructor automatically called from kfree_skb. 2680 */ 2681 void sock_wfree(struct sk_buff *skb) 2682 { 2683 unsigned int len = skb->truesize; 2684 struct sock *sk = skb->sk; 2685 bool free; 2686 int old; 2687 2688 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) { 2689 void (*sk_write_space)(struct sock *sk); 2690 2691 sk_write_space = READ_ONCE(sk->sk_write_space); 2692 2693 if (sock_flag(sk, SOCK_RCU_FREE) && 2694 sk_write_space == sock_def_write_space) { 2695 rcu_read_lock(); 2696 free = __refcount_sub_and_test(len, &sk->sk_wmem_alloc, 2697 &old); 2698 sock_def_write_space_wfree(sk, old - len); 2699 rcu_read_unlock(); 2700 if (unlikely(free)) 2701 __sk_free(sk); 2702 return; 2703 } 2704 2705 /* 2706 * Keep a reference on sk_wmem_alloc, this will be released 2707 * after sk_write_space() call 2708 */ 2709 WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc)); 2710 sk_write_space(sk); 2711 len = 1; 2712 } 2713 /* 2714 * if sk_wmem_alloc reaches 0, we must finish what sk_free() 2715 * could not do because of in-flight packets 2716 */ 2717 if (refcount_sub_and_test(len, &sk->sk_wmem_alloc)) 2718 __sk_free(sk); 2719 } 2720 EXPORT_SYMBOL(sock_wfree); 2721 2722 /* This variant of sock_wfree() is used by TCP, 2723 * since it sets SOCK_USE_WRITE_QUEUE. 2724 */ 2725 #ifdef CONFIG_INET 2726 void __sock_wfree(struct sk_buff *skb) 2727 { 2728 struct sock *sk = skb->sk; 2729 2730 if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)) 2731 __sk_free(sk); 2732 } 2733 EXPORT_SYMBOL_GPL(__sock_wfree); 2734 #endif 2735 2736 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk) 2737 { 2738 int old_wmem; 2739 2740 skb_orphan(skb); 2741 #ifdef CONFIG_INET 2742 if (unlikely(!sk_fullsock(sk))) 2743 return skb_set_owner_edemux(skb, sk); 2744 #endif 2745 skb->sk = sk; 2746 skb->destructor = sock_wfree; 2747 skb_set_hash_from_sk(skb, sk); 2748 /* 2749 * We used to take a refcount on sk, but following operation 2750 * is enough to guarantee sk_free() won't free this sock until 2751 * all in-flight packets are completed 2752 */ 2753 __refcount_add(skb->truesize, &sk->sk_wmem_alloc, &old_wmem); 2754 2755 /* (old_wmem == SK_WMEM_ALLOC_BIAS) if no other TX packet for this socket 2756 * is in a host queue (qdisc, NIC queue). 2757 * Set skb->ooo_okay so that netdev_pick_tx() can choose a TX queue 2758 * based on XPS for better performance. 2759 * Otherwise clear ooo_okay to not risk Out Of Order delivery. 2760 */ 2761 skb->ooo_okay = (old_wmem == SK_WMEM_ALLOC_BIAS); 2762 } 2763 EXPORT_SYMBOL(skb_set_owner_w); 2764 2765 static bool can_skb_orphan_partial(const struct sk_buff *skb) 2766 { 2767 /* Drivers depend on in-order delivery for crypto offload, 2768 * partial orphan breaks out-of-order-OK logic. 2769 */ 2770 if (skb_is_decrypted(skb)) 2771 return false; 2772 2773 return (skb->destructor == sock_wfree || 2774 (IS_ENABLED(CONFIG_INET) && skb->destructor == tcp_wfree)); 2775 } 2776 2777 /* This helper is used by netem, as it can hold packets in its 2778 * delay queue. We want to allow the owner socket to send more 2779 * packets, as if they were already TX completed by a typical driver. 2780 * But we also want to keep skb->sk set because some packet schedulers 2781 * rely on it (sch_fq for example). 2782 */ 2783 void skb_orphan_partial(struct sk_buff *skb) 2784 { 2785 if (skb_is_tcp_pure_ack(skb)) 2786 return; 2787 2788 if (can_skb_orphan_partial(skb) && skb_set_owner_sk_safe(skb, skb->sk)) 2789 return; 2790 2791 skb_orphan(skb); 2792 } 2793 EXPORT_SYMBOL(skb_orphan_partial); 2794 2795 /* 2796 * Read buffer destructor automatically called from kfree_skb. 2797 */ 2798 void sock_rfree(struct sk_buff *skb) 2799 { 2800 struct sock *sk = skb->sk; 2801 unsigned int len = skb->truesize; 2802 2803 atomic_sub(len, &sk->sk_rmem_alloc); 2804 sk_mem_uncharge(sk, len); 2805 } 2806 EXPORT_SYMBOL(sock_rfree); 2807 2808 /* 2809 * Buffer destructor for skbs that are not used directly in read or write 2810 * path, e.g. for error handler skbs. Automatically called from kfree_skb. 2811 */ 2812 void sock_efree(struct sk_buff *skb) 2813 { 2814 sock_put(skb->sk); 2815 } 2816 EXPORT_SYMBOL(sock_efree); 2817 2818 /* Buffer destructor for prefetch/receive path where reference count may 2819 * not be held, e.g. for listen sockets. 2820 */ 2821 #ifdef CONFIG_INET 2822 void sock_pfree(struct sk_buff *skb) 2823 { 2824 struct sock *sk = skb->sk; 2825 2826 if (!sk_is_refcounted(sk)) 2827 return; 2828 2829 if (sk->sk_state == TCP_NEW_SYN_RECV && inet_reqsk(sk)->syncookie) { 2830 inet_reqsk(sk)->rsk_listener = NULL; 2831 reqsk_free(inet_reqsk(sk)); 2832 return; 2833 } 2834 2835 sock_gen_put(sk); 2836 } 2837 EXPORT_SYMBOL(sock_pfree); 2838 #endif /* CONFIG_INET */ 2839 2840 /* 2841 * Allocate a skb from the socket's send buffer. 2842 */ 2843 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force, 2844 gfp_t priority) 2845 { 2846 if (force || 2847 refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) { 2848 struct sk_buff *skb = alloc_skb(size, priority); 2849 2850 if (skb) { 2851 skb_set_owner_w(skb, sk); 2852 return skb; 2853 } 2854 } 2855 return NULL; 2856 } 2857 EXPORT_SYMBOL(sock_wmalloc); 2858 2859 static void sock_ofree(struct sk_buff *skb) 2860 { 2861 struct sock *sk = skb->sk; 2862 2863 atomic_sub(skb->truesize, &sk->sk_omem_alloc); 2864 } 2865 2866 struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size, 2867 gfp_t priority) 2868 { 2869 struct sk_buff *skb; 2870 2871 /* small safe race: SKB_TRUESIZE may differ from final skb->truesize */ 2872 if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) > 2873 READ_ONCE(sock_net(sk)->core.sysctl_optmem_max)) 2874 return NULL; 2875 2876 skb = alloc_skb(size, priority); 2877 if (!skb) 2878 return NULL; 2879 2880 atomic_add(skb->truesize, &sk->sk_omem_alloc); 2881 skb->sk = sk; 2882 skb->destructor = sock_ofree; 2883 return skb; 2884 } 2885 2886 /* 2887 * Allocate a memory block from the socket's option memory buffer. 2888 */ 2889 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority) 2890 { 2891 int optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max); 2892 2893 if ((unsigned int)size <= optmem_max && 2894 atomic_read(&sk->sk_omem_alloc) + size < optmem_max) { 2895 void *mem; 2896 /* First do the add, to avoid the race if kmalloc 2897 * might sleep. 2898 */ 2899 atomic_add(size, &sk->sk_omem_alloc); 2900 mem = kmalloc(size, priority); 2901 if (mem) 2902 return mem; 2903 atomic_sub(size, &sk->sk_omem_alloc); 2904 } 2905 return NULL; 2906 } 2907 EXPORT_SYMBOL(sock_kmalloc); 2908 2909 /* 2910 * Duplicate the input "src" memory block using the socket's 2911 * option memory buffer. 2912 */ 2913 void *sock_kmemdup(struct sock *sk, const void *src, 2914 int size, gfp_t priority) 2915 { 2916 void *mem; 2917 2918 mem = sock_kmalloc(sk, size, priority); 2919 if (mem) 2920 memcpy(mem, src, size); 2921 return mem; 2922 } 2923 EXPORT_SYMBOL(sock_kmemdup); 2924 2925 /* Free an option memory block. Note, we actually want the inline 2926 * here as this allows gcc to detect the nullify and fold away the 2927 * condition entirely. 2928 */ 2929 static inline void __sock_kfree_s(struct sock *sk, void *mem, int size, 2930 const bool nullify) 2931 { 2932 if (WARN_ON_ONCE(!mem)) 2933 return; 2934 if (nullify) 2935 kfree_sensitive(mem); 2936 else 2937 kfree(mem); 2938 atomic_sub(size, &sk->sk_omem_alloc); 2939 } 2940 2941 void sock_kfree_s(struct sock *sk, void *mem, int size) 2942 { 2943 __sock_kfree_s(sk, mem, size, false); 2944 } 2945 EXPORT_SYMBOL(sock_kfree_s); 2946 2947 void sock_kzfree_s(struct sock *sk, void *mem, int size) 2948 { 2949 __sock_kfree_s(sk, mem, size, true); 2950 } 2951 EXPORT_SYMBOL(sock_kzfree_s); 2952 2953 /* It is almost wait_for_tcp_memory minus release_sock/lock_sock. 2954 I think, these locks should be removed for datagram sockets. 2955 */ 2956 static long sock_wait_for_wmem(struct sock *sk, long timeo) 2957 { 2958 DEFINE_WAIT(wait); 2959 2960 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 2961 for (;;) { 2962 if (!timeo) 2963 break; 2964 if (signal_pending(current)) 2965 break; 2966 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2967 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); 2968 if (refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) 2969 break; 2970 if (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN) 2971 break; 2972 if (READ_ONCE(sk->sk_err)) 2973 break; 2974 timeo = schedule_timeout(timeo); 2975 } 2976 finish_wait(sk_sleep(sk), &wait); 2977 return timeo; 2978 } 2979 2980 2981 /* 2982 * Generic send/receive buffer handlers 2983 */ 2984 2985 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len, 2986 unsigned long data_len, int noblock, 2987 int *errcode, int max_page_order) 2988 { 2989 struct sk_buff *skb; 2990 long timeo; 2991 int err; 2992 2993 timeo = sock_sndtimeo(sk, noblock); 2994 for (;;) { 2995 err = sock_error(sk); 2996 if (err != 0) 2997 goto failure; 2998 2999 err = -EPIPE; 3000 if (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN) 3001 goto failure; 3002 3003 if (sk_wmem_alloc_get(sk) < READ_ONCE(sk->sk_sndbuf)) 3004 break; 3005 3006 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 3007 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 3008 err = -EAGAIN; 3009 if (!timeo) 3010 goto failure; 3011 if (signal_pending(current)) 3012 goto interrupted; 3013 timeo = sock_wait_for_wmem(sk, timeo); 3014 } 3015 skb = alloc_skb_with_frags(header_len, data_len, max_page_order, 3016 errcode, sk->sk_allocation); 3017 if (skb) 3018 skb_set_owner_w(skb, sk); 3019 return skb; 3020 3021 interrupted: 3022 err = sock_intr_errno(timeo); 3023 failure: 3024 *errcode = err; 3025 return NULL; 3026 } 3027 EXPORT_SYMBOL(sock_alloc_send_pskb); 3028 3029 int __sock_cmsg_send(struct sock *sk, struct cmsghdr *cmsg, 3030 struct sockcm_cookie *sockc) 3031 { 3032 u32 tsflags; 3033 3034 BUILD_BUG_ON(SOF_TIMESTAMPING_LAST == (1 << 31)); 3035 3036 switch (cmsg->cmsg_type) { 3037 case SO_MARK: 3038 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && 3039 !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 3040 return -EPERM; 3041 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3042 return -EINVAL; 3043 sockc->mark = *(u32 *)CMSG_DATA(cmsg); 3044 break; 3045 case SO_TIMESTAMPING_OLD: 3046 case SO_TIMESTAMPING_NEW: 3047 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3048 return -EINVAL; 3049 3050 tsflags = *(u32 *)CMSG_DATA(cmsg); 3051 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK) 3052 return -EINVAL; 3053 3054 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK; 3055 sockc->tsflags |= tsflags; 3056 break; 3057 case SCM_TXTIME: 3058 { 3059 ktime_t tmin; 3060 u64 txtime; 3061 3062 if (!sock_flag(sk, SOCK_TXTIME)) 3063 return -EINVAL; 3064 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64))) 3065 return -EINVAL; 3066 3067 txtime = get_unaligned((u64 *)CMSG_DATA(cmsg)); 3068 3069 /* Allow sending without a delivery time: zero special case */ 3070 if (!txtime) { 3071 sockc->transmit_time = 0; 3072 break; 3073 } 3074 3075 switch (sk->sk_clockid) { 3076 case CLOCK_MONOTONIC: 3077 tmin = 1; 3078 break; 3079 case CLOCK_REALTIME: 3080 tmin = max(ktime_mono_to_real(0), 1); 3081 break; 3082 case CLOCK_TAI: 3083 tmin = max(ktime_mono_to_any(0, TK_OFFS_TAI), 1); 3084 break; 3085 default: 3086 tmin = 1; 3087 WARN_ON_ONCE(1); 3088 break; 3089 } 3090 3091 sockc->transmit_time = max_t(ktime_t, txtime, tmin); 3092 break; 3093 } 3094 case SCM_TS_OPT_ID: 3095 if (sk_is_tcp(sk)) 3096 return -EINVAL; 3097 tsflags = READ_ONCE(sk->sk_tsflags); 3098 if (!(tsflags & SOF_TIMESTAMPING_OPT_ID)) 3099 return -EINVAL; 3100 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3101 return -EINVAL; 3102 sockc->ts_opt_id = *(u32 *)CMSG_DATA(cmsg); 3103 sockc->tsflags |= SOCKCM_FLAG_TS_OPT_ID; 3104 break; 3105 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */ 3106 case SCM_RIGHTS: 3107 case SCM_CREDENTIALS: 3108 break; 3109 case SO_PRIORITY: 3110 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3111 return -EINVAL; 3112 if (!sk_set_prio_allowed(sk, *(u32 *)CMSG_DATA(cmsg))) 3113 return -EPERM; 3114 sockc->priority = *(u32 *)CMSG_DATA(cmsg); 3115 break; 3116 case SCM_DEVMEM_DMABUF: 3117 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3118 return -EINVAL; 3119 sockc->dmabuf_id = *(u32 *)CMSG_DATA(cmsg); 3120 break; 3121 default: 3122 return -EINVAL; 3123 } 3124 return 0; 3125 } 3126 EXPORT_SYMBOL(__sock_cmsg_send); 3127 3128 int sock_cmsg_send(struct sock *sk, struct msghdr *msg, 3129 struct sockcm_cookie *sockc) 3130 { 3131 struct cmsghdr *cmsg; 3132 int ret; 3133 3134 for_each_cmsghdr(cmsg, msg) { 3135 if (!CMSG_OK(msg, cmsg)) 3136 return -EINVAL; 3137 if (cmsg->cmsg_level != SOL_SOCKET) 3138 continue; 3139 ret = __sock_cmsg_send(sk, cmsg, sockc); 3140 if (ret) 3141 return ret; 3142 } 3143 return 0; 3144 } 3145 EXPORT_SYMBOL(sock_cmsg_send); 3146 3147 static void sk_enter_memory_pressure(struct sock *sk) 3148 { 3149 if (!sk->sk_prot->enter_memory_pressure) 3150 return; 3151 3152 sk->sk_prot->enter_memory_pressure(sk); 3153 } 3154 3155 static void sk_leave_memory_pressure(struct sock *sk) 3156 { 3157 if (sk->sk_prot->leave_memory_pressure) { 3158 INDIRECT_CALL_INET_1(sk->sk_prot->leave_memory_pressure, 3159 tcp_leave_memory_pressure, sk); 3160 } else { 3161 unsigned long *memory_pressure = sk->sk_prot->memory_pressure; 3162 3163 if (memory_pressure && READ_ONCE(*memory_pressure)) 3164 WRITE_ONCE(*memory_pressure, 0); 3165 } 3166 } 3167 3168 DEFINE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key); 3169 3170 /** 3171 * skb_page_frag_refill - check that a page_frag contains enough room 3172 * @sz: minimum size of the fragment we want to get 3173 * @pfrag: pointer to page_frag 3174 * @gfp: priority for memory allocation 3175 * 3176 * Note: While this allocator tries to use high order pages, there is 3177 * no guarantee that allocations succeed. Therefore, @sz MUST be 3178 * less or equal than PAGE_SIZE. 3179 */ 3180 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp) 3181 { 3182 if (pfrag->page) { 3183 if (page_ref_count(pfrag->page) == 1) { 3184 pfrag->offset = 0; 3185 return true; 3186 } 3187 if (pfrag->offset + sz <= pfrag->size) 3188 return true; 3189 put_page(pfrag->page); 3190 } 3191 3192 pfrag->offset = 0; 3193 if (SKB_FRAG_PAGE_ORDER && 3194 !static_branch_unlikely(&net_high_order_alloc_disable_key)) { 3195 /* Avoid direct reclaim but allow kswapd to wake */ 3196 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) | 3197 __GFP_COMP | __GFP_NOWARN | 3198 __GFP_NORETRY, 3199 SKB_FRAG_PAGE_ORDER); 3200 if (likely(pfrag->page)) { 3201 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER; 3202 return true; 3203 } 3204 } 3205 pfrag->page = alloc_page(gfp); 3206 if (likely(pfrag->page)) { 3207 pfrag->size = PAGE_SIZE; 3208 return true; 3209 } 3210 return false; 3211 } 3212 EXPORT_SYMBOL(skb_page_frag_refill); 3213 3214 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 3215 { 3216 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation))) 3217 return true; 3218 3219 if (!sk->sk_bypass_prot_mem) 3220 sk_enter_memory_pressure(sk); 3221 3222 sk_stream_moderate_sndbuf(sk); 3223 3224 return false; 3225 } 3226 EXPORT_SYMBOL(sk_page_frag_refill); 3227 3228 static void __lock_sock(struct sock *sk) 3229 __releases(&sk->sk_lock.slock) 3230 __acquires(&sk->sk_lock.slock) 3231 { 3232 DEFINE_WAIT(wait); 3233 3234 for (;;) { 3235 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait, 3236 TASK_UNINTERRUPTIBLE); 3237 spin_unlock_bh(&sk->sk_lock.slock); 3238 schedule(); 3239 spin_lock_bh(&sk->sk_lock.slock); 3240 if (!sock_owned_by_user(sk)) 3241 break; 3242 } 3243 finish_wait(&sk->sk_lock.wq, &wait); 3244 } 3245 3246 void __release_sock(struct sock *sk) 3247 __releases(&sk->sk_lock.slock) 3248 __acquires(&sk->sk_lock.slock) 3249 { 3250 struct sk_buff *skb, *next; 3251 int nb = 0; 3252 3253 while ((skb = sk->sk_backlog.head) != NULL) { 3254 sk->sk_backlog.head = sk->sk_backlog.tail = NULL; 3255 3256 spin_unlock_bh(&sk->sk_lock.slock); 3257 3258 while (1) { 3259 next = skb->next; 3260 prefetch(next); 3261 DEBUG_NET_WARN_ON_ONCE(skb_dst_is_noref(skb)); 3262 skb_mark_not_on_list(skb); 3263 sk_backlog_rcv(sk, skb); 3264 3265 skb = next; 3266 if (!skb) 3267 break; 3268 3269 if (!(++nb & 15)) 3270 cond_resched(); 3271 } 3272 3273 spin_lock_bh(&sk->sk_lock.slock); 3274 } 3275 3276 /* 3277 * Doing the zeroing here guarantee we can not loop forever 3278 * while a wild producer attempts to flood us. 3279 */ 3280 sk->sk_backlog.len = 0; 3281 } 3282 3283 void __sk_flush_backlog(struct sock *sk) 3284 { 3285 spin_lock_bh(&sk->sk_lock.slock); 3286 __release_sock(sk); 3287 3288 if (sk->sk_prot->release_cb) 3289 INDIRECT_CALL_INET_1(sk->sk_prot->release_cb, 3290 tcp_release_cb, sk); 3291 3292 spin_unlock_bh(&sk->sk_lock.slock); 3293 } 3294 EXPORT_SYMBOL_GPL(__sk_flush_backlog); 3295 3296 /** 3297 * sk_wait_data - wait for data to arrive at sk_receive_queue 3298 * @sk: sock to wait on 3299 * @timeo: for how long 3300 * @skb: last skb seen on sk_receive_queue 3301 * 3302 * Now socket state including sk->sk_err is changed only under lock, 3303 * hence we may omit checks after joining wait queue. 3304 * We check receive queue before schedule() only as optimization; 3305 * it is very likely that release_sock() added new data. 3306 */ 3307 int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb) 3308 { 3309 DEFINE_WAIT_FUNC(wait, woken_wake_function); 3310 int rc; 3311 3312 add_wait_queue(sk_sleep(sk), &wait); 3313 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 3314 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait); 3315 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 3316 remove_wait_queue(sk_sleep(sk), &wait); 3317 return rc; 3318 } 3319 EXPORT_SYMBOL(sk_wait_data); 3320 3321 /** 3322 * __sk_mem_raise_allocated - increase memory_allocated 3323 * @sk: socket 3324 * @size: memory size to allocate 3325 * @amt: pages to allocate 3326 * @kind: allocation type 3327 * 3328 * Similar to __sk_mem_schedule(), but does not update sk_forward_alloc. 3329 * 3330 * Unlike the globally shared limits among the sockets under same protocol, 3331 * consuming the budget of a memcg won't have direct effect on other ones. 3332 * So be optimistic about memcg's tolerance, and leave the callers to decide 3333 * whether or not to raise allocated through sk_under_memory_pressure() or 3334 * its variants. 3335 */ 3336 int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind) 3337 { 3338 bool memcg_enabled = false, charged = false; 3339 struct proto *prot = sk->sk_prot; 3340 long allocated = 0; 3341 3342 if (!sk->sk_bypass_prot_mem) { 3343 sk_memory_allocated_add(sk, amt); 3344 allocated = sk_memory_allocated(sk); 3345 } 3346 3347 if (mem_cgroup_sk_enabled(sk)) { 3348 memcg_enabled = true; 3349 charged = mem_cgroup_sk_charge(sk, amt, gfp_memcg_charge()); 3350 if (!charged) 3351 goto suppress_allocation; 3352 } 3353 3354 if (!allocated) 3355 return 1; 3356 3357 /* Under limit. */ 3358 if (allocated <= sk_prot_mem_limits(sk, 0)) { 3359 sk_leave_memory_pressure(sk); 3360 return 1; 3361 } 3362 3363 /* Under pressure. */ 3364 if (allocated > sk_prot_mem_limits(sk, 1)) 3365 sk_enter_memory_pressure(sk); 3366 3367 /* Over hard limit. */ 3368 if (allocated > sk_prot_mem_limits(sk, 2)) 3369 goto suppress_allocation; 3370 3371 /* Guarantee minimum buffer size under pressure (either global 3372 * or memcg) to make sure features described in RFC 7323 (TCP 3373 * Extensions for High Performance) work properly. 3374 * 3375 * This rule does NOT stand when exceeds global or memcg's hard 3376 * limit, or else a DoS attack can be taken place by spawning 3377 * lots of sockets whose usage are under minimum buffer size. 3378 */ 3379 if (kind == SK_MEM_RECV) { 3380 if (atomic_read(&sk->sk_rmem_alloc) < sk_get_rmem0(sk, prot)) 3381 return 1; 3382 3383 } else { /* SK_MEM_SEND */ 3384 int wmem0 = sk_get_wmem0(sk, prot); 3385 3386 if (sk->sk_type == SOCK_STREAM) { 3387 if (sk->sk_wmem_queued < wmem0) 3388 return 1; 3389 } else if (refcount_read(&sk->sk_wmem_alloc) < wmem0) { 3390 return 1; 3391 } 3392 } 3393 3394 if (sk_has_memory_pressure(sk)) { 3395 u64 alloc; 3396 3397 /* The following 'average' heuristic is within the 3398 * scope of global accounting, so it only makes 3399 * sense for global memory pressure. 3400 */ 3401 if (!sk_under_global_memory_pressure(sk)) 3402 return 1; 3403 3404 /* Try to be fair among all the sockets under global 3405 * pressure by allowing the ones that below average 3406 * usage to raise. 3407 */ 3408 alloc = sk_sockets_allocated_read_positive(sk); 3409 if (sk_prot_mem_limits(sk, 2) > alloc * 3410 sk_mem_pages(sk->sk_wmem_queued + 3411 atomic_read(&sk->sk_rmem_alloc) + 3412 sk->sk_forward_alloc)) 3413 return 1; 3414 } 3415 3416 suppress_allocation: 3417 3418 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) { 3419 sk_stream_moderate_sndbuf(sk); 3420 3421 /* Fail only if socket is _under_ its sndbuf. 3422 * In this case we cannot block, so that we have to fail. 3423 */ 3424 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf) { 3425 /* Force charge with __GFP_NOFAIL */ 3426 if (memcg_enabled && !charged) 3427 mem_cgroup_sk_charge(sk, amt, 3428 gfp_memcg_charge() | __GFP_NOFAIL); 3429 return 1; 3430 } 3431 } 3432 3433 trace_sock_exceed_buf_limit(sk, prot, allocated, kind); 3434 3435 if (allocated) 3436 sk_memory_allocated_sub(sk, amt); 3437 3438 if (charged) 3439 mem_cgroup_sk_uncharge(sk, amt); 3440 3441 return 0; 3442 } 3443 3444 /** 3445 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated 3446 * @sk: socket 3447 * @size: memory size to allocate 3448 * @kind: allocation type 3449 * 3450 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means 3451 * rmem allocation. This function assumes that protocols which have 3452 * memory_pressure use sk_wmem_queued as write buffer accounting. 3453 */ 3454 int __sk_mem_schedule(struct sock *sk, int size, int kind) 3455 { 3456 int ret, amt = sk_mem_pages(size); 3457 3458 sk_forward_alloc_add(sk, amt << PAGE_SHIFT); 3459 ret = __sk_mem_raise_allocated(sk, size, amt, kind); 3460 if (!ret) 3461 sk_forward_alloc_add(sk, -(amt << PAGE_SHIFT)); 3462 return ret; 3463 } 3464 EXPORT_SYMBOL(__sk_mem_schedule); 3465 3466 /** 3467 * __sk_mem_reduce_allocated - reclaim memory_allocated 3468 * @sk: socket 3469 * @amount: number of quanta 3470 * 3471 * Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc 3472 */ 3473 void __sk_mem_reduce_allocated(struct sock *sk, int amount) 3474 { 3475 if (mem_cgroup_sk_enabled(sk)) 3476 mem_cgroup_sk_uncharge(sk, amount); 3477 3478 if (sk->sk_bypass_prot_mem) 3479 return; 3480 3481 sk_memory_allocated_sub(sk, amount); 3482 3483 if (sk_under_global_memory_pressure(sk) && 3484 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0))) 3485 sk_leave_memory_pressure(sk); 3486 } 3487 3488 /** 3489 * __sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated 3490 * @sk: socket 3491 * @amount: number of bytes (rounded down to a PAGE_SIZE multiple) 3492 */ 3493 void __sk_mem_reclaim(struct sock *sk, int amount) 3494 { 3495 amount >>= PAGE_SHIFT; 3496 sk_forward_alloc_add(sk, -(amount << PAGE_SHIFT)); 3497 __sk_mem_reduce_allocated(sk, amount); 3498 } 3499 EXPORT_SYMBOL(__sk_mem_reclaim); 3500 3501 void __sk_charge(struct sock *sk, gfp_t gfp) 3502 { 3503 int amt; 3504 3505 gfp |= __GFP_NOFAIL; 3506 if (mem_cgroup_from_sk(sk)) { 3507 /* The socket has not been accepted yet, no need 3508 * to look at newsk->sk_wmem_queued. 3509 */ 3510 amt = sk_mem_pages(sk->sk_forward_alloc + 3511 atomic_read(&sk->sk_rmem_alloc)); 3512 if (amt) 3513 mem_cgroup_sk_charge(sk, amt, gfp); 3514 } 3515 3516 kmem_cache_charge(sk, gfp); 3517 } 3518 3519 int sk_set_peek_off(struct sock *sk, int val) 3520 { 3521 WRITE_ONCE(sk->sk_peek_off, val); 3522 return 0; 3523 } 3524 EXPORT_SYMBOL_GPL(sk_set_peek_off); 3525 3526 /* 3527 * Set of default routines for initialising struct proto_ops when 3528 * the protocol does not support a particular function. In certain 3529 * cases where it makes no sense for a protocol to have a "do nothing" 3530 * function, some default processing is provided. 3531 */ 3532 3533 int sock_no_bind(struct socket *sock, struct sockaddr_unsized *saddr, int len) 3534 { 3535 return -EOPNOTSUPP; 3536 } 3537 EXPORT_SYMBOL(sock_no_bind); 3538 3539 int sock_no_connect(struct socket *sock, struct sockaddr_unsized *saddr, 3540 int len, int flags) 3541 { 3542 return -EOPNOTSUPP; 3543 } 3544 EXPORT_SYMBOL(sock_no_connect); 3545 3546 int sock_no_socketpair(struct socket *sock1, struct socket *sock2) 3547 { 3548 return -EOPNOTSUPP; 3549 } 3550 EXPORT_SYMBOL(sock_no_socketpair); 3551 3552 int sock_no_accept(struct socket *sock, struct socket *newsock, 3553 struct proto_accept_arg *arg) 3554 { 3555 return -EOPNOTSUPP; 3556 } 3557 EXPORT_SYMBOL(sock_no_accept); 3558 3559 int sock_no_getname(struct socket *sock, struct sockaddr *saddr, 3560 int peer) 3561 { 3562 return -EOPNOTSUPP; 3563 } 3564 EXPORT_SYMBOL(sock_no_getname); 3565 3566 int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) 3567 { 3568 return -EOPNOTSUPP; 3569 } 3570 EXPORT_SYMBOL(sock_no_ioctl); 3571 3572 int sock_no_listen(struct socket *sock, int backlog) 3573 { 3574 return -EOPNOTSUPP; 3575 } 3576 EXPORT_SYMBOL(sock_no_listen); 3577 3578 int sock_no_shutdown(struct socket *sock, int how) 3579 { 3580 return -EOPNOTSUPP; 3581 } 3582 EXPORT_SYMBOL(sock_no_shutdown); 3583 3584 int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len) 3585 { 3586 return -EOPNOTSUPP; 3587 } 3588 EXPORT_SYMBOL(sock_no_sendmsg); 3589 3590 int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len) 3591 { 3592 return -EOPNOTSUPP; 3593 } 3594 EXPORT_SYMBOL(sock_no_sendmsg_locked); 3595 3596 int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len, 3597 int flags) 3598 { 3599 return -EOPNOTSUPP; 3600 } 3601 EXPORT_SYMBOL(sock_no_recvmsg); 3602 3603 int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma) 3604 { 3605 /* Mirror missing mmap method error code */ 3606 return -ENODEV; 3607 } 3608 EXPORT_SYMBOL(sock_no_mmap); 3609 3610 /* 3611 * When a file is received (via SCM_RIGHTS, etc), we must bump the 3612 * various sock-based usage counts. 3613 */ 3614 void __receive_sock(struct file *file) 3615 { 3616 struct socket *sock; 3617 3618 sock = sock_from_file(file); 3619 if (sock) { 3620 sock_update_netprioidx(&sock->sk->sk_cgrp_data); 3621 sock_update_classid(&sock->sk->sk_cgrp_data); 3622 } 3623 } 3624 3625 /* 3626 * Default Socket Callbacks 3627 */ 3628 3629 static void sock_def_wakeup(struct sock *sk) 3630 { 3631 struct socket_wq *wq; 3632 3633 rcu_read_lock(); 3634 wq = rcu_dereference(sk->sk_wq); 3635 if (skwq_has_sleeper(wq)) 3636 wake_up_interruptible_all(&wq->wait); 3637 rcu_read_unlock(); 3638 } 3639 3640 static void sock_def_error_report(struct sock *sk) 3641 { 3642 struct socket_wq *wq; 3643 3644 rcu_read_lock(); 3645 wq = rcu_dereference(sk->sk_wq); 3646 if (skwq_has_sleeper(wq)) 3647 wake_up_interruptible_poll(&wq->wait, EPOLLERR); 3648 sk_wake_async_rcu(sk, SOCK_WAKE_IO, POLL_ERR); 3649 rcu_read_unlock(); 3650 } 3651 3652 void sock_def_readable(struct sock *sk) 3653 { 3654 struct socket_wq *wq; 3655 3656 trace_sk_data_ready(sk); 3657 3658 rcu_read_lock(); 3659 wq = rcu_dereference(sk->sk_wq); 3660 if (skwq_has_sleeper(wq)) 3661 wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI | 3662 EPOLLRDNORM | EPOLLRDBAND); 3663 sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_IN); 3664 rcu_read_unlock(); 3665 } 3666 3667 static void sock_def_write_space(struct sock *sk) 3668 { 3669 struct socket_wq *wq; 3670 3671 rcu_read_lock(); 3672 3673 /* Do not wake up a writer until he can make "significant" 3674 * progress. --DaveM 3675 */ 3676 if (sock_writeable(sk)) { 3677 wq = rcu_dereference(sk->sk_wq); 3678 if (skwq_has_sleeper(wq)) 3679 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 3680 EPOLLWRNORM | EPOLLWRBAND); 3681 3682 /* Should agree with poll, otherwise some programs break */ 3683 sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT); 3684 } 3685 3686 rcu_read_unlock(); 3687 } 3688 3689 /* An optimised version of sock_def_write_space(), should only be called 3690 * for SOCK_RCU_FREE sockets under RCU read section and after putting 3691 * ->sk_wmem_alloc. 3692 */ 3693 static void sock_def_write_space_wfree(struct sock *sk, int wmem_alloc) 3694 { 3695 /* Do not wake up a writer until he can make "significant" 3696 * progress. --DaveM 3697 */ 3698 if (__sock_writeable(sk, wmem_alloc)) { 3699 struct socket_wq *wq = rcu_dereference(sk->sk_wq); 3700 3701 /* rely on refcount_sub from sock_wfree() */ 3702 smp_mb__after_atomic(); 3703 if (wq && waitqueue_active(&wq->wait)) 3704 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 3705 EPOLLWRNORM | EPOLLWRBAND); 3706 3707 /* Should agree with poll, otherwise some programs break */ 3708 sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT); 3709 } 3710 } 3711 3712 static void sock_def_destruct(struct sock *sk) 3713 { 3714 } 3715 3716 void sk_send_sigurg(struct sock *sk) 3717 { 3718 if (sk->sk_socket && sk->sk_socket->file) 3719 if (send_sigurg(sk->sk_socket->file)) 3720 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI); 3721 } 3722 EXPORT_SYMBOL(sk_send_sigurg); 3723 3724 void sk_reset_timer(struct sock *sk, struct timer_list* timer, 3725 unsigned long expires) 3726 { 3727 if (!mod_timer(timer, expires)) 3728 sock_hold(sk); 3729 } 3730 EXPORT_SYMBOL(sk_reset_timer); 3731 3732 void sk_stop_timer(struct sock *sk, struct timer_list* timer) 3733 { 3734 if (timer_delete(timer)) 3735 __sock_put(sk); 3736 } 3737 EXPORT_SYMBOL(sk_stop_timer); 3738 3739 void sk_stop_timer_sync(struct sock *sk, struct timer_list *timer) 3740 { 3741 if (timer_delete_sync(timer)) 3742 __sock_put(sk); 3743 } 3744 EXPORT_SYMBOL(sk_stop_timer_sync); 3745 3746 void sock_init_data_uid(struct socket *sock, struct sock *sk, kuid_t uid) 3747 { 3748 sk_init_common(sk); 3749 sk->sk_send_head = NULL; 3750 3751 timer_setup(&sk->sk_timer, NULL, 0); 3752 3753 sk->sk_allocation = GFP_KERNEL; 3754 sk->sk_rcvbuf = READ_ONCE(sysctl_rmem_default); 3755 sk->sk_sndbuf = READ_ONCE(sysctl_wmem_default); 3756 sk->sk_state = TCP_CLOSE; 3757 sk->sk_use_task_frag = true; 3758 sk_set_socket(sk, sock); 3759 3760 sock_set_flag(sk, SOCK_ZAPPED); 3761 3762 if (sock) { 3763 sk->sk_type = sock->type; 3764 RCU_INIT_POINTER(sk->sk_wq, &sock->wq); 3765 sock->sk = sk; 3766 } else { 3767 RCU_INIT_POINTER(sk->sk_wq, NULL); 3768 } 3769 sk->sk_uid = uid; 3770 3771 sk->sk_state_change = sock_def_wakeup; 3772 sk->sk_data_ready = sock_def_readable; 3773 sk->sk_write_space = sock_def_write_space; 3774 sk->sk_error_report = sock_def_error_report; 3775 sk->sk_destruct = sock_def_destruct; 3776 3777 sk->sk_frag.page = NULL; 3778 sk->sk_frag.offset = 0; 3779 sk->sk_peek_off = -1; 3780 3781 sk->sk_peer_pid = NULL; 3782 sk->sk_peer_cred = NULL; 3783 spin_lock_init(&sk->sk_peer_lock); 3784 3785 sk->sk_write_pending = 0; 3786 sk->sk_rcvlowat = 1; 3787 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT; 3788 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT; 3789 3790 sk->sk_stamp = SK_DEFAULT_STAMP; 3791 #if BITS_PER_LONG==32 3792 seqlock_init(&sk->sk_stamp_seq); 3793 #endif 3794 atomic_set(&sk->sk_zckey, 0); 3795 3796 #ifdef CONFIG_NET_RX_BUSY_POLL 3797 sk->sk_napi_id = 0; 3798 sk->sk_ll_usec = READ_ONCE(sysctl_net_busy_read); 3799 #endif 3800 3801 sk->sk_max_pacing_rate = ~0UL; 3802 sk->sk_pacing_rate = ~0UL; 3803 WRITE_ONCE(sk->sk_pacing_shift, 10); 3804 sk->sk_incoming_cpu = -1; 3805 3806 sk_rx_queue_clear(sk); 3807 /* 3808 * Before updating sk_refcnt, we must commit prior changes to memory 3809 * (Documentation/RCU/rculist_nulls.rst for details) 3810 */ 3811 smp_wmb(); 3812 refcount_set(&sk->sk_refcnt, 1); 3813 sk_drops_reset(sk); 3814 } 3815 EXPORT_SYMBOL(sock_init_data_uid); 3816 3817 void sock_init_data(struct socket *sock, struct sock *sk) 3818 { 3819 kuid_t uid = sock ? 3820 SOCK_INODE(sock)->i_uid : 3821 make_kuid(sock_net(sk)->user_ns, 0); 3822 3823 sock_init_data_uid(sock, sk, uid); 3824 } 3825 EXPORT_SYMBOL(sock_init_data); 3826 3827 void noinline lock_sock_nested(struct sock *sk, int subclass) 3828 { 3829 /* The sk_lock has mutex_lock() semantics here. */ 3830 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_); 3831 3832 might_sleep(); 3833 #ifdef CONFIG_64BIT 3834 if (sizeof(struct slock_owned) == sizeof(long)) { 3835 socket_lock_t tmp = { 3836 .slock = __SPIN_LOCK_UNLOCKED(tmp.slock), 3837 .owned = 1, 3838 }; 3839 socket_lock_t old = { 3840 .slock = __SPIN_LOCK_UNLOCKED(old.slock), 3841 .owned = 0, 3842 }; 3843 3844 if (likely(try_cmpxchg(&sk->sk_lock.combined, 3845 &old.combined, tmp.combined))) 3846 return; 3847 } 3848 #endif 3849 spin_lock_bh(&sk->sk_lock.slock); 3850 if (unlikely(sock_owned_by_user_nocheck(sk))) 3851 __lock_sock(sk); 3852 sk->sk_lock.owned = 1; 3853 spin_unlock_bh(&sk->sk_lock.slock); 3854 } 3855 EXPORT_SYMBOL(lock_sock_nested); 3856 3857 void release_sock(struct sock *sk) 3858 { 3859 spin_lock_bh(&sk->sk_lock.slock); 3860 3861 if (unlikely(sk->sk_backlog.tail)) 3862 __release_sock(sk); 3863 3864 if (sk->sk_prot->release_cb) { 3865 if (!tcp_release_cb_cond(sk)) 3866 sk->sk_prot->release_cb(sk); 3867 } 3868 sock_release_ownership(sk); 3869 if (unlikely(waitqueue_active(&sk->sk_lock.wq))) 3870 wake_up(&sk->sk_lock.wq); 3871 3872 spin_unlock_bh(&sk->sk_lock.slock); 3873 } 3874 EXPORT_SYMBOL(release_sock); 3875 3876 bool __lock_sock_fast(struct sock *sk) __acquires(&sk->sk_lock.slock) 3877 { 3878 might_sleep(); 3879 spin_lock_bh(&sk->sk_lock.slock); 3880 3881 if (likely(!sock_owned_by_user_nocheck(sk))) { 3882 /* 3883 * Fast path return with bottom halves disabled and 3884 * sock::sk_lock.slock held. 3885 * 3886 * The 'mutex' is not contended and holding 3887 * sock::sk_lock.slock prevents all other lockers to 3888 * proceed so the corresponding unlock_sock_fast() can 3889 * avoid the slow path of release_sock() completely and 3890 * just release slock. 3891 * 3892 * From a semantical POV this is equivalent to 'acquiring' 3893 * the 'mutex', hence the corresponding lockdep 3894 * mutex_release() has to happen in the fast path of 3895 * unlock_sock_fast(). 3896 */ 3897 return false; 3898 } 3899 3900 __lock_sock(sk); 3901 sk->sk_lock.owned = 1; 3902 __acquire(&sk->sk_lock.slock); 3903 spin_unlock_bh(&sk->sk_lock.slock); 3904 return true; 3905 } 3906 EXPORT_SYMBOL(__lock_sock_fast); 3907 3908 int sock_gettstamp(struct socket *sock, void __user *userstamp, 3909 bool timeval, bool time32) 3910 { 3911 struct sock *sk = sock->sk; 3912 struct timespec64 ts; 3913 3914 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 3915 ts = ktime_to_timespec64(sock_read_timestamp(sk)); 3916 if (ts.tv_sec == -1) 3917 return -ENOENT; 3918 if (ts.tv_sec == 0) { 3919 ktime_t kt = ktime_get_real(); 3920 sock_write_timestamp(sk, kt); 3921 ts = ktime_to_timespec64(kt); 3922 } 3923 3924 if (timeval) 3925 ts.tv_nsec /= 1000; 3926 3927 #ifdef CONFIG_COMPAT_32BIT_TIME 3928 if (time32) 3929 return put_old_timespec32(&ts, userstamp); 3930 #endif 3931 #ifdef CONFIG_SPARC64 3932 /* beware of padding in sparc64 timeval */ 3933 if (timeval && !in_compat_syscall()) { 3934 struct __kernel_old_timeval __user tv = { 3935 .tv_sec = ts.tv_sec, 3936 .tv_usec = ts.tv_nsec, 3937 }; 3938 if (copy_to_user(userstamp, &tv, sizeof(tv))) 3939 return -EFAULT; 3940 return 0; 3941 } 3942 #endif 3943 return put_timespec64(&ts, userstamp); 3944 } 3945 EXPORT_SYMBOL(sock_gettstamp); 3946 3947 void sock_enable_timestamp(struct sock *sk, enum sock_flags flag) 3948 { 3949 if (!sock_flag(sk, flag)) { 3950 unsigned long previous_flags = sk->sk_flags; 3951 3952 sock_set_flag(sk, flag); 3953 /* 3954 * we just set one of the two flags which require net 3955 * time stamping, but time stamping might have been on 3956 * already because of the other one 3957 */ 3958 if (sock_needs_netstamp(sk) && 3959 !(previous_flags & SK_FLAGS_TIMESTAMP)) 3960 net_enable_timestamp(); 3961 } 3962 } 3963 3964 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, 3965 int level, int type) 3966 { 3967 struct sock_extended_err ee; 3968 struct sk_buff *skb; 3969 int copied, err; 3970 3971 err = -EAGAIN; 3972 skb = sock_dequeue_err_skb(sk); 3973 if (skb == NULL) 3974 goto out; 3975 3976 copied = skb->len; 3977 if (copied > len) { 3978 msg->msg_flags |= MSG_TRUNC; 3979 copied = len; 3980 } 3981 err = skb_copy_datagram_msg(skb, 0, msg, copied); 3982 if (err) 3983 goto out_free_skb; 3984 3985 sock_recv_timestamp(msg, sk, skb); 3986 3987 /* We must use a bounce buffer for CONFIG_HARDENED_USERCOPY=y */ 3988 ee = SKB_EXT_ERR(skb)->ee; 3989 put_cmsg(msg, level, type, sizeof(ee), &ee); 3990 3991 msg->msg_flags |= MSG_ERRQUEUE; 3992 err = copied; 3993 3994 out_free_skb: 3995 kfree_skb(skb); 3996 out: 3997 return err; 3998 } 3999 EXPORT_SYMBOL(sock_recv_errqueue); 4000 4001 /* 4002 * Get a socket option on an socket. 4003 * 4004 * FIX: POSIX 1003.1g is very ambiguous here. It states that 4005 * asynchronous errors should be reported by getsockopt. We assume 4006 * this means if you specify SO_ERROR (otherwise what is the point of it). 4007 */ 4008 int sock_common_getsockopt(struct socket *sock, int level, int optname, 4009 char __user *optval, int __user *optlen) 4010 { 4011 struct sock *sk = sock->sk; 4012 4013 /* IPV6_ADDRFORM can change sk->sk_prot under us. */ 4014 return READ_ONCE(sk->sk_prot)->getsockopt(sk, level, optname, optval, optlen); 4015 } 4016 EXPORT_SYMBOL(sock_common_getsockopt); 4017 4018 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, 4019 int flags) 4020 { 4021 struct sock *sk = sock->sk; 4022 4023 return sk->sk_prot->recvmsg(sk, msg, size, flags); 4024 } 4025 EXPORT_SYMBOL(sock_common_recvmsg); 4026 4027 /* 4028 * Set socket options on an inet socket. 4029 */ 4030 int sock_common_setsockopt(struct socket *sock, int level, int optname, 4031 sockptr_t optval, unsigned int optlen) 4032 { 4033 struct sock *sk = sock->sk; 4034 4035 /* IPV6_ADDRFORM can change sk->sk_prot under us. */ 4036 return READ_ONCE(sk->sk_prot)->setsockopt(sk, level, optname, optval, optlen); 4037 } 4038 EXPORT_SYMBOL(sock_common_setsockopt); 4039 4040 void sk_common_release(struct sock *sk) 4041 { 4042 if (sk->sk_prot->destroy) 4043 sk->sk_prot->destroy(sk); 4044 4045 /* 4046 * Observation: when sk_common_release is called, processes have 4047 * no access to socket. But net still has. 4048 * Step one, detach it from networking: 4049 * 4050 * A. Remove from hash tables. 4051 */ 4052 4053 sk->sk_prot->unhash(sk); 4054 4055 /* 4056 * In this point socket cannot receive new packets, but it is possible 4057 * that some packets are in flight because some CPU runs receiver and 4058 * did hash table lookup before we unhashed socket. They will achieve 4059 * receive queue and will be purged by socket destructor. 4060 * 4061 * Also we still have packets pending on receive queue and probably, 4062 * our own packets waiting in device queues. sock_destroy will drain 4063 * receive queue, but transmitted packets will delay socket destruction 4064 * until the last reference will be released. 4065 */ 4066 4067 sock_orphan(sk); 4068 4069 xfrm_sk_free_policy(sk); 4070 4071 sock_put(sk); 4072 } 4073 EXPORT_SYMBOL(sk_common_release); 4074 4075 void sk_get_meminfo(const struct sock *sk, u32 *mem) 4076 { 4077 memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS); 4078 4079 mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk); 4080 mem[SK_MEMINFO_RCVBUF] = READ_ONCE(sk->sk_rcvbuf); 4081 mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk); 4082 mem[SK_MEMINFO_SNDBUF] = READ_ONCE(sk->sk_sndbuf); 4083 mem[SK_MEMINFO_FWD_ALLOC] = READ_ONCE(sk->sk_forward_alloc); 4084 mem[SK_MEMINFO_WMEM_QUEUED] = READ_ONCE(sk->sk_wmem_queued); 4085 mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc); 4086 mem[SK_MEMINFO_BACKLOG] = READ_ONCE(sk->sk_backlog.len); 4087 mem[SK_MEMINFO_DROPS] = sk_drops_read(sk); 4088 } 4089 4090 #ifdef CONFIG_PROC_FS 4091 static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR); 4092 4093 int sock_prot_inuse_get(struct net *net, struct proto *prot) 4094 { 4095 int cpu, idx = prot->inuse_idx; 4096 int res = 0; 4097 4098 for_each_possible_cpu(cpu) 4099 res += per_cpu_ptr(net->core.prot_inuse, cpu)->val[idx]; 4100 4101 return res >= 0 ? res : 0; 4102 } 4103 EXPORT_SYMBOL_GPL(sock_prot_inuse_get); 4104 4105 int sock_inuse_get(struct net *net) 4106 { 4107 int cpu, res = 0; 4108 4109 for_each_possible_cpu(cpu) 4110 res += per_cpu_ptr(net->core.prot_inuse, cpu)->all; 4111 4112 return res; 4113 } 4114 4115 EXPORT_SYMBOL_GPL(sock_inuse_get); 4116 4117 static int __net_init sock_inuse_init_net(struct net *net) 4118 { 4119 net->core.prot_inuse = alloc_percpu(struct prot_inuse); 4120 if (net->core.prot_inuse == NULL) 4121 return -ENOMEM; 4122 return 0; 4123 } 4124 4125 static void __net_exit sock_inuse_exit_net(struct net *net) 4126 { 4127 free_percpu(net->core.prot_inuse); 4128 } 4129 4130 static struct pernet_operations net_inuse_ops = { 4131 .init = sock_inuse_init_net, 4132 .exit = sock_inuse_exit_net, 4133 }; 4134 4135 static __init int net_inuse_init(void) 4136 { 4137 if (register_pernet_subsys(&net_inuse_ops)) 4138 panic("Cannot initialize net inuse counters"); 4139 4140 return 0; 4141 } 4142 4143 core_initcall(net_inuse_init); 4144 4145 static int assign_proto_idx(struct proto *prot) 4146 { 4147 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR); 4148 4149 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR)) { 4150 pr_err("PROTO_INUSE_NR exhausted\n"); 4151 return -ENOSPC; 4152 } 4153 4154 set_bit(prot->inuse_idx, proto_inuse_idx); 4155 return 0; 4156 } 4157 4158 static void release_proto_idx(struct proto *prot) 4159 { 4160 if (prot->inuse_idx != PROTO_INUSE_NR) 4161 clear_bit(prot->inuse_idx, proto_inuse_idx); 4162 } 4163 #else 4164 static inline int assign_proto_idx(struct proto *prot) 4165 { 4166 return 0; 4167 } 4168 4169 static inline void release_proto_idx(struct proto *prot) 4170 { 4171 } 4172 4173 #endif 4174 4175 static void tw_prot_cleanup(struct timewait_sock_ops *twsk_prot) 4176 { 4177 if (!twsk_prot) 4178 return; 4179 kfree(twsk_prot->twsk_slab_name); 4180 twsk_prot->twsk_slab_name = NULL; 4181 kmem_cache_destroy(twsk_prot->twsk_slab); 4182 twsk_prot->twsk_slab = NULL; 4183 } 4184 4185 static int tw_prot_init(const struct proto *prot) 4186 { 4187 struct timewait_sock_ops *twsk_prot = prot->twsk_prot; 4188 4189 if (!twsk_prot) 4190 return 0; 4191 4192 twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", 4193 prot->name); 4194 if (!twsk_prot->twsk_slab_name) 4195 return -ENOMEM; 4196 4197 twsk_prot->twsk_slab = 4198 kmem_cache_create(twsk_prot->twsk_slab_name, 4199 twsk_prot->twsk_obj_size, 0, 4200 SLAB_ACCOUNT | prot->slab_flags, 4201 NULL); 4202 if (!twsk_prot->twsk_slab) { 4203 pr_crit("%s: Can't create timewait sock SLAB cache!\n", 4204 prot->name); 4205 return -ENOMEM; 4206 } 4207 4208 return 0; 4209 } 4210 4211 static void req_prot_cleanup(struct request_sock_ops *rsk_prot) 4212 { 4213 if (!rsk_prot) 4214 return; 4215 kfree(rsk_prot->slab_name); 4216 rsk_prot->slab_name = NULL; 4217 kmem_cache_destroy(rsk_prot->slab); 4218 rsk_prot->slab = NULL; 4219 } 4220 4221 static int req_prot_init(const struct proto *prot) 4222 { 4223 struct request_sock_ops *rsk_prot = prot->rsk_prot; 4224 4225 if (!rsk_prot) 4226 return 0; 4227 4228 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s", 4229 prot->name); 4230 if (!rsk_prot->slab_name) 4231 return -ENOMEM; 4232 4233 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name, 4234 rsk_prot->obj_size, 0, 4235 SLAB_ACCOUNT | prot->slab_flags, 4236 NULL); 4237 4238 if (!rsk_prot->slab) { 4239 pr_crit("%s: Can't create request sock SLAB cache!\n", 4240 prot->name); 4241 return -ENOMEM; 4242 } 4243 return 0; 4244 } 4245 4246 int proto_register(struct proto *prot, int alloc_slab) 4247 { 4248 int ret = -ENOBUFS; 4249 4250 if (prot->memory_allocated && !prot->sysctl_mem) { 4251 pr_err("%s: missing sysctl_mem\n", prot->name); 4252 return -EINVAL; 4253 } 4254 if (prot->memory_allocated && !prot->per_cpu_fw_alloc) { 4255 pr_err("%s: missing per_cpu_fw_alloc\n", prot->name); 4256 return -EINVAL; 4257 } 4258 if (alloc_slab) { 4259 struct kmem_cache_args args = { 4260 .useroffset = prot->useroffset, 4261 .usersize = prot->usersize, 4262 .freeptr_offset = prot->freeptr_offset, 4263 .use_freeptr_offset = !!prot->freeptr_offset, 4264 }; 4265 4266 prot->slab = kmem_cache_create(prot->name, prot->obj_size, 4267 &args, 4268 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT | 4269 prot->slab_flags); 4270 if (prot->slab == NULL) { 4271 pr_crit("%s: Can't create sock SLAB cache!\n", 4272 prot->name); 4273 goto out; 4274 } 4275 4276 if (req_prot_init(prot)) 4277 goto out_free_request_sock_slab; 4278 4279 if (tw_prot_init(prot)) 4280 goto out_free_timewait_sock_slab; 4281 } 4282 4283 mutex_lock(&proto_list_mutex); 4284 ret = assign_proto_idx(prot); 4285 if (ret) { 4286 mutex_unlock(&proto_list_mutex); 4287 goto out_free_timewait_sock_slab; 4288 } 4289 list_add(&prot->node, &proto_list); 4290 mutex_unlock(&proto_list_mutex); 4291 return ret; 4292 4293 out_free_timewait_sock_slab: 4294 if (alloc_slab) 4295 tw_prot_cleanup(prot->twsk_prot); 4296 out_free_request_sock_slab: 4297 if (alloc_slab) { 4298 req_prot_cleanup(prot->rsk_prot); 4299 4300 kmem_cache_destroy(prot->slab); 4301 prot->slab = NULL; 4302 } 4303 out: 4304 return ret; 4305 } 4306 EXPORT_SYMBOL(proto_register); 4307 4308 void proto_unregister(struct proto *prot) 4309 { 4310 mutex_lock(&proto_list_mutex); 4311 release_proto_idx(prot); 4312 list_del(&prot->node); 4313 mutex_unlock(&proto_list_mutex); 4314 4315 kmem_cache_destroy(prot->slab); 4316 prot->slab = NULL; 4317 4318 req_prot_cleanup(prot->rsk_prot); 4319 tw_prot_cleanup(prot->twsk_prot); 4320 } 4321 EXPORT_SYMBOL(proto_unregister); 4322 4323 int sock_load_diag_module(int family, int protocol) 4324 { 4325 if (!protocol) { 4326 if (!sock_is_registered(family)) 4327 return -ENOENT; 4328 4329 return request_module("net-pf-%d-proto-%d-type-%d", PF_NETLINK, 4330 NETLINK_SOCK_DIAG, family); 4331 } 4332 4333 #ifdef CONFIG_INET 4334 if (family == AF_INET && 4335 protocol != IPPROTO_RAW && 4336 protocol < MAX_INET_PROTOS && 4337 !rcu_access_pointer(inet_protos[protocol])) 4338 return -ENOENT; 4339 #endif 4340 4341 return request_module("net-pf-%d-proto-%d-type-%d-%d", PF_NETLINK, 4342 NETLINK_SOCK_DIAG, family, protocol); 4343 } 4344 EXPORT_SYMBOL(sock_load_diag_module); 4345 4346 #ifdef CONFIG_PROC_FS 4347 static void *proto_seq_start(struct seq_file *seq, loff_t *pos) 4348 __acquires(proto_list_mutex) 4349 { 4350 mutex_lock(&proto_list_mutex); 4351 return seq_list_start_head(&proto_list, *pos); 4352 } 4353 4354 static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos) 4355 { 4356 return seq_list_next(v, &proto_list, pos); 4357 } 4358 4359 static void proto_seq_stop(struct seq_file *seq, void *v) 4360 __releases(proto_list_mutex) 4361 { 4362 mutex_unlock(&proto_list_mutex); 4363 } 4364 4365 static char proto_method_implemented(const void *method) 4366 { 4367 return method == NULL ? 'n' : 'y'; 4368 } 4369 static long sock_prot_memory_allocated(struct proto *proto) 4370 { 4371 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L; 4372 } 4373 4374 static const char *sock_prot_memory_pressure(struct proto *proto) 4375 { 4376 return proto->memory_pressure != NULL ? 4377 proto_memory_pressure(proto) ? "yes" : "no" : "NI"; 4378 } 4379 4380 static void proto_seq_printf(struct seq_file *seq, struct proto *proto) 4381 { 4382 4383 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s " 4384 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n", 4385 proto->name, 4386 proto->obj_size, 4387 sock_prot_inuse_get(seq_file_net(seq), proto), 4388 sock_prot_memory_allocated(proto), 4389 sock_prot_memory_pressure(proto), 4390 proto->max_header, 4391 proto->slab == NULL ? "no" : "yes", 4392 module_name(proto->owner), 4393 proto_method_implemented(proto->close), 4394 proto_method_implemented(proto->connect), 4395 proto_method_implemented(proto->disconnect), 4396 proto_method_implemented(proto->accept), 4397 proto_method_implemented(proto->ioctl), 4398 proto_method_implemented(proto->init), 4399 proto_method_implemented(proto->destroy), 4400 proto_method_implemented(proto->shutdown), 4401 proto_method_implemented(proto->setsockopt), 4402 proto_method_implemented(proto->getsockopt), 4403 proto_method_implemented(proto->sendmsg), 4404 proto_method_implemented(proto->recvmsg), 4405 proto_method_implemented(proto->bind), 4406 proto_method_implemented(proto->backlog_rcv), 4407 proto_method_implemented(proto->hash), 4408 proto_method_implemented(proto->unhash), 4409 proto_method_implemented(proto->get_port), 4410 proto_method_implemented(proto->enter_memory_pressure)); 4411 } 4412 4413 static int proto_seq_show(struct seq_file *seq, void *v) 4414 { 4415 if (v == &proto_list) 4416 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s", 4417 "protocol", 4418 "size", 4419 "sockets", 4420 "memory", 4421 "press", 4422 "maxhdr", 4423 "slab", 4424 "module", 4425 "cl co di ac io in de sh ss gs se re bi br ha uh gp em\n"); 4426 else 4427 proto_seq_printf(seq, list_entry(v, struct proto, node)); 4428 return 0; 4429 } 4430 4431 static const struct seq_operations proto_seq_ops = { 4432 .start = proto_seq_start, 4433 .next = proto_seq_next, 4434 .stop = proto_seq_stop, 4435 .show = proto_seq_show, 4436 }; 4437 4438 static __net_init int proto_init_net(struct net *net) 4439 { 4440 if (!proc_create_net("protocols", 0444, net->proc_net, &proto_seq_ops, 4441 sizeof(struct seq_net_private))) 4442 return -ENOMEM; 4443 4444 return 0; 4445 } 4446 4447 static __net_exit void proto_exit_net(struct net *net) 4448 { 4449 remove_proc_entry("protocols", net->proc_net); 4450 } 4451 4452 4453 static __net_initdata struct pernet_operations proto_net_ops = { 4454 .init = proto_init_net, 4455 .exit = proto_exit_net, 4456 }; 4457 4458 static int __init proto_init(void) 4459 { 4460 return register_pernet_subsys(&proto_net_ops); 4461 } 4462 4463 subsys_initcall(proto_init); 4464 4465 #endif /* PROC_FS */ 4466 4467 #ifdef CONFIG_NET_RX_BUSY_POLL 4468 bool sk_busy_loop_end(void *p, unsigned long start_time) 4469 { 4470 struct sock *sk = p; 4471 4472 if (!skb_queue_empty_lockless(&sk->sk_receive_queue)) 4473 return true; 4474 4475 if (sk_is_udp(sk) && 4476 !skb_queue_empty_lockless(&udp_sk(sk)->reader_queue)) 4477 return true; 4478 4479 return sk_busy_loop_timeout(sk, start_time); 4480 } 4481 EXPORT_SYMBOL(sk_busy_loop_end); 4482 #endif /* CONFIG_NET_RX_BUSY_POLL */ 4483 4484 int sock_bind_add(struct sock *sk, struct sockaddr_unsized *addr, int addr_len) 4485 { 4486 if (!sk->sk_prot->bind_add) 4487 return -EOPNOTSUPP; 4488 return sk->sk_prot->bind_add(sk, addr, addr_len); 4489 } 4490 EXPORT_SYMBOL(sock_bind_add); 4491 4492 /* Copy 'size' bytes from userspace and return `size` back to userspace */ 4493 int sock_ioctl_inout(struct sock *sk, unsigned int cmd, 4494 void __user *arg, void *karg, size_t size) 4495 { 4496 int ret; 4497 4498 if (copy_from_user(karg, arg, size)) 4499 return -EFAULT; 4500 4501 ret = READ_ONCE(sk->sk_prot)->ioctl(sk, cmd, karg); 4502 if (ret) 4503 return ret; 4504 4505 if (copy_to_user(arg, karg, size)) 4506 return -EFAULT; 4507 4508 return 0; 4509 } 4510 EXPORT_SYMBOL(sock_ioctl_inout); 4511 4512 /* This is the most common ioctl prep function, where the result (4 bytes) is 4513 * copied back to userspace if the ioctl() returns successfully. No input is 4514 * copied from userspace as input argument. 4515 */ 4516 static int sock_ioctl_out(struct sock *sk, unsigned int cmd, void __user *arg) 4517 { 4518 int ret, karg = 0; 4519 4520 ret = READ_ONCE(sk->sk_prot)->ioctl(sk, cmd, &karg); 4521 if (ret) 4522 return ret; 4523 4524 return put_user(karg, (int __user *)arg); 4525 } 4526 4527 /* A wrapper around sock ioctls, which copies the data from userspace 4528 * (depending on the protocol/ioctl), and copies back the result to userspace. 4529 * The main motivation for this function is to pass kernel memory to the 4530 * protocol ioctl callbacks, instead of userspace memory. 4531 */ 4532 int sk_ioctl(struct sock *sk, unsigned int cmd, void __user *arg) 4533 { 4534 int rc = 1; 4535 4536 if (sk->sk_type == SOCK_RAW && sk->sk_family == AF_INET) 4537 rc = ipmr_sk_ioctl(sk, cmd, arg); 4538 else if (sk->sk_type == SOCK_RAW && sk->sk_family == AF_INET6) 4539 rc = ip6mr_sk_ioctl(sk, cmd, arg); 4540 else if (sk_is_phonet(sk)) 4541 rc = phonet_sk_ioctl(sk, cmd, arg); 4542 4543 /* If ioctl was processed, returns its value */ 4544 if (rc <= 0) 4545 return rc; 4546 4547 /* Otherwise call the default handler */ 4548 return sock_ioctl_out(sk, cmd, arg); 4549 } 4550 EXPORT_SYMBOL(sk_ioctl); 4551 4552 static int __init sock_struct_check(void) 4553 { 4554 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_drops); 4555 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_peek_off); 4556 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_error_queue); 4557 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_receive_queue); 4558 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_backlog); 4559 4560 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rx_dst); 4561 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rx_dst_ifindex); 4562 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rx_dst_cookie); 4563 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rcvbuf); 4564 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_filter); 4565 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_wq); 4566 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_data_ready); 4567 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rcvtimeo); 4568 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rcvlowat); 4569 4570 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rxtx, sk_err); 4571 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rxtx, sk_socket); 4572 #ifdef CONFIG_MEMCG 4573 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rxtx, sk_memcg); 4574 #endif 4575 4576 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_lock); 4577 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_reserved_mem); 4578 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_forward_alloc); 4579 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_tsflags); 4580 4581 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_omem_alloc); 4582 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_omem_alloc); 4583 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_err_soft); 4584 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_wmem_queued); 4585 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_wmem_alloc); 4586 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_tsq_flags); 4587 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_send_head); 4588 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_write_queue); 4589 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_write_pending); 4590 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_frag); 4591 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_timer); 4592 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_pacing_rate); 4593 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_zckey); 4594 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_tskey); 4595 4596 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_dst_pending_confirm); 4597 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_pacing_status); 4598 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_max_pacing_rate); 4599 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_sndtimeo); 4600 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_priority); 4601 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_mark); 4602 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_uid); 4603 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_protocol); 4604 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_dst_cache); 4605 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_route_caps); 4606 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_gso_type); 4607 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_gso_max_size); 4608 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_allocation); 4609 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_txhash); 4610 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_sndbuf); 4611 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_gso_max_segs); 4612 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_pacing_shift); 4613 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_use_task_frag); 4614 return 0; 4615 } 4616 4617 core_initcall(sock_struct_check); 4618