1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * Generic socket support routines. Memory allocators, socket lock/release 8 * handler for protocols to use and generic option handler. 9 * 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Florian La Roche, <flla@stud.uni-sb.de> 13 * Alan Cox, <A.Cox@swansea.ac.uk> 14 * 15 * Fixes: 16 * Alan Cox : Numerous verify_area() problems 17 * Alan Cox : Connecting on a connecting socket 18 * now returns an error for tcp. 19 * Alan Cox : sock->protocol is set correctly. 20 * and is not sometimes left as 0. 21 * Alan Cox : connect handles icmp errors on a 22 * connect properly. Unfortunately there 23 * is a restart syscall nasty there. I 24 * can't match BSD without hacking the C 25 * library. Ideas urgently sought! 26 * Alan Cox : Disallow bind() to addresses that are 27 * not ours - especially broadcast ones!! 28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost) 29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets, 30 * instead they leave that for the DESTROY timer. 31 * Alan Cox : Clean up error flag in accept 32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer 33 * was buggy. Put a remove_sock() in the handler 34 * for memory when we hit 0. Also altered the timer 35 * code. The ACK stuff can wait and needs major 36 * TCP layer surgery. 37 * Alan Cox : Fixed TCP ack bug, removed remove sock 38 * and fixed timer/inet_bh race. 39 * Alan Cox : Added zapped flag for TCP 40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code 41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb 42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources 43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing. 44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so... 45 * Rick Sladkey : Relaxed UDP rules for matching packets. 46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support 47 * Pauline Middelink : identd support 48 * Alan Cox : Fixed connect() taking signals I think. 49 * Alan Cox : SO_LINGER supported 50 * Alan Cox : Error reporting fixes 51 * Anonymous : inet_create tidied up (sk->reuse setting) 52 * Alan Cox : inet sockets don't set sk->type! 53 * Alan Cox : Split socket option code 54 * Alan Cox : Callbacks 55 * Alan Cox : Nagle flag for Charles & Johannes stuff 56 * Alex : Removed restriction on inet fioctl 57 * Alan Cox : Splitting INET from NET core 58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt() 59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code 60 * Alan Cox : Split IP from generic code 61 * Alan Cox : New kfree_skbmem() 62 * Alan Cox : Make SO_DEBUG superuser only. 63 * Alan Cox : Allow anyone to clear SO_DEBUG 64 * (compatibility fix) 65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput. 66 * Alan Cox : Allocator for a socket is settable. 67 * Alan Cox : SO_ERROR includes soft errors. 68 * Alan Cox : Allow NULL arguments on some SO_ opts 69 * Alan Cox : Generic socket allocation to make hooks 70 * easier (suggested by Craig Metz). 71 * Michael Pall : SO_ERROR returns positive errno again 72 * Steve Whitehouse: Added default destructor to free 73 * protocol private data. 74 * Steve Whitehouse: Added various other default routines 75 * common to several socket families. 76 * Chris Evans : Call suser() check last on F_SETOWN 77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER. 78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s() 79 * Andi Kleen : Fix write_space callback 80 * Chris Evans : Security fixes - signedness again 81 * Arnaldo C. Melo : cleanups, use skb_queue_purge 82 * 83 * To Fix: 84 */ 85 86 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 87 88 #include <linux/unaligned.h> 89 #include <linux/capability.h> 90 #include <linux/errno.h> 91 #include <linux/errqueue.h> 92 #include <linux/types.h> 93 #include <linux/socket.h> 94 #include <linux/in.h> 95 #include <linux/kernel.h> 96 #include <linux/module.h> 97 #include <linux/proc_fs.h> 98 #include <linux/seq_file.h> 99 #include <linux/sched.h> 100 #include <linux/sched/mm.h> 101 #include <linux/timer.h> 102 #include <linux/string.h> 103 #include <linux/sockios.h> 104 #include <linux/net.h> 105 #include <linux/mm.h> 106 #include <linux/slab.h> 107 #include <linux/interrupt.h> 108 #include <linux/poll.h> 109 #include <linux/tcp.h> 110 #include <linux/udp.h> 111 #include <linux/init.h> 112 #include <linux/highmem.h> 113 #include <linux/user_namespace.h> 114 #include <linux/static_key.h> 115 #include <linux/memcontrol.h> 116 #include <linux/prefetch.h> 117 #include <linux/compat.h> 118 #include <linux/mroute.h> 119 #include <linux/mroute6.h> 120 #include <linux/icmpv6.h> 121 122 #include <linux/uaccess.h> 123 124 #include <linux/netdevice.h> 125 #include <net/protocol.h> 126 #include <linux/skbuff.h> 127 #include <linux/skbuff_ref.h> 128 #include <net/net_namespace.h> 129 #include <net/request_sock.h> 130 #include <net/sock.h> 131 #include <net/proto_memory.h> 132 #include <linux/net_tstamp.h> 133 #include <net/xfrm.h> 134 #include <linux/ipsec.h> 135 #include <net/cls_cgroup.h> 136 #include <net/netprio_cgroup.h> 137 #include <linux/sock_diag.h> 138 139 #include <linux/filter.h> 140 #include <net/sock_reuseport.h> 141 #include <net/bpf_sk_storage.h> 142 143 #include <trace/events/sock.h> 144 145 #include <net/tcp.h> 146 #include <net/busy_poll.h> 147 #include <net/phonet/phonet.h> 148 149 #include <linux/ethtool.h> 150 151 #include <uapi/linux/pidfd.h> 152 153 #include "dev.h" 154 155 static DEFINE_MUTEX(proto_list_mutex); 156 static LIST_HEAD(proto_list); 157 158 static void sock_def_write_space_wfree(struct sock *sk, int wmem_alloc); 159 static void sock_def_write_space(struct sock *sk); 160 161 /** 162 * sk_ns_capable - General socket capability test 163 * @sk: Socket to use a capability on or through 164 * @user_ns: The user namespace of the capability to use 165 * @cap: The capability to use 166 * 167 * Test to see if the opener of the socket had when the socket was 168 * created and the current process has the capability @cap in the user 169 * namespace @user_ns. 170 */ 171 bool sk_ns_capable(const struct sock *sk, 172 struct user_namespace *user_ns, int cap) 173 { 174 return file_ns_capable(sk->sk_socket->file, user_ns, cap) && 175 ns_capable(user_ns, cap); 176 } 177 EXPORT_SYMBOL(sk_ns_capable); 178 179 /** 180 * sk_capable - Socket global capability test 181 * @sk: Socket to use a capability on or through 182 * @cap: The global capability to use 183 * 184 * Test to see if the opener of the socket had when the socket was 185 * created and the current process has the capability @cap in all user 186 * namespaces. 187 */ 188 bool sk_capable(const struct sock *sk, int cap) 189 { 190 return sk_ns_capable(sk, &init_user_ns, cap); 191 } 192 EXPORT_SYMBOL(sk_capable); 193 194 /** 195 * sk_net_capable - Network namespace socket capability test 196 * @sk: Socket to use a capability on or through 197 * @cap: The capability to use 198 * 199 * Test to see if the opener of the socket had when the socket was created 200 * and the current process has the capability @cap over the network namespace 201 * the socket is a member of. 202 */ 203 bool sk_net_capable(const struct sock *sk, int cap) 204 { 205 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap); 206 } 207 EXPORT_SYMBOL(sk_net_capable); 208 209 /* 210 * Each address family might have different locking rules, so we have 211 * one slock key per address family and separate keys for internal and 212 * userspace sockets. 213 */ 214 static struct lock_class_key af_family_keys[AF_MAX]; 215 static struct lock_class_key af_family_kern_keys[AF_MAX]; 216 static struct lock_class_key af_family_slock_keys[AF_MAX]; 217 static struct lock_class_key af_family_kern_slock_keys[AF_MAX]; 218 219 /* 220 * Make lock validator output more readable. (we pre-construct these 221 * strings build-time, so that runtime initialization of socket 222 * locks is fast): 223 */ 224 225 #define _sock_locks(x) \ 226 x "AF_UNSPEC", x "AF_UNIX" , x "AF_INET" , \ 227 x "AF_AX25" , x "AF_IPX" , x "AF_APPLETALK", \ 228 x "AF_NETROM", x "AF_BRIDGE" , x "AF_ATMPVC" , \ 229 x "AF_X25" , x "AF_INET6" , x "AF_ROSE" , \ 230 x "AF_DECnet", x "AF_NETBEUI" , x "AF_SECURITY" , \ 231 x "AF_KEY" , x "AF_NETLINK" , x "AF_PACKET" , \ 232 x "AF_ASH" , x "AF_ECONET" , x "AF_ATMSVC" , \ 233 x "AF_RDS" , x "AF_SNA" , x "AF_IRDA" , \ 234 x "AF_PPPOX" , x "AF_WANPIPE" , x "AF_LLC" , \ 235 x "27" , x "28" , x "AF_CAN" , \ 236 x "AF_TIPC" , x "AF_BLUETOOTH", x "IUCV" , \ 237 x "AF_RXRPC" , x "AF_ISDN" , x "AF_PHONET" , \ 238 x "AF_IEEE802154", x "AF_CAIF" , x "AF_ALG" , \ 239 x "AF_NFC" , x "AF_VSOCK" , x "AF_KCM" , \ 240 x "AF_QIPCRTR", x "AF_SMC" , x "AF_XDP" , \ 241 x "AF_MCTP" , \ 242 x "AF_MAX" 243 244 static const char *const af_family_key_strings[AF_MAX+1] = { 245 _sock_locks("sk_lock-") 246 }; 247 static const char *const af_family_slock_key_strings[AF_MAX+1] = { 248 _sock_locks("slock-") 249 }; 250 static const char *const af_family_clock_key_strings[AF_MAX+1] = { 251 _sock_locks("clock-") 252 }; 253 254 static const char *const af_family_kern_key_strings[AF_MAX+1] = { 255 _sock_locks("k-sk_lock-") 256 }; 257 static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = { 258 _sock_locks("k-slock-") 259 }; 260 static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = { 261 _sock_locks("k-clock-") 262 }; 263 static const char *const af_family_rlock_key_strings[AF_MAX+1] = { 264 _sock_locks("rlock-") 265 }; 266 static const char *const af_family_wlock_key_strings[AF_MAX+1] = { 267 _sock_locks("wlock-") 268 }; 269 static const char *const af_family_elock_key_strings[AF_MAX+1] = { 270 _sock_locks("elock-") 271 }; 272 273 /* 274 * sk_callback_lock and sk queues locking rules are per-address-family, 275 * so split the lock classes by using a per-AF key: 276 */ 277 static struct lock_class_key af_callback_keys[AF_MAX]; 278 static struct lock_class_key af_rlock_keys[AF_MAX]; 279 static struct lock_class_key af_wlock_keys[AF_MAX]; 280 static struct lock_class_key af_elock_keys[AF_MAX]; 281 static struct lock_class_key af_kern_callback_keys[AF_MAX]; 282 283 /* Run time adjustable parameters. */ 284 __u32 sysctl_wmem_max __read_mostly = 4 << 20; 285 EXPORT_SYMBOL(sysctl_wmem_max); 286 __u32 sysctl_rmem_max __read_mostly = 4 << 20; 287 EXPORT_SYMBOL(sysctl_rmem_max); 288 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_DEFAULT; 289 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_DEFAULT; 290 291 DEFINE_STATIC_KEY_FALSE(memalloc_socks_key); 292 EXPORT_SYMBOL_GPL(memalloc_socks_key); 293 294 /** 295 * sk_set_memalloc - sets %SOCK_MEMALLOC 296 * @sk: socket to set it on 297 * 298 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves. 299 * It's the responsibility of the admin to adjust min_free_kbytes 300 * to meet the requirements 301 */ 302 void sk_set_memalloc(struct sock *sk) 303 { 304 sock_set_flag(sk, SOCK_MEMALLOC); 305 sk->sk_allocation |= __GFP_MEMALLOC; 306 static_branch_inc(&memalloc_socks_key); 307 } 308 EXPORT_SYMBOL_GPL(sk_set_memalloc); 309 310 void sk_clear_memalloc(struct sock *sk) 311 { 312 sock_reset_flag(sk, SOCK_MEMALLOC); 313 sk->sk_allocation &= ~__GFP_MEMALLOC; 314 static_branch_dec(&memalloc_socks_key); 315 316 /* 317 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward 318 * progress of swapping. SOCK_MEMALLOC may be cleared while 319 * it has rmem allocations due to the last swapfile being deactivated 320 * but there is a risk that the socket is unusable due to exceeding 321 * the rmem limits. Reclaim the reserves and obey rmem limits again. 322 */ 323 sk_mem_reclaim(sk); 324 } 325 EXPORT_SYMBOL_GPL(sk_clear_memalloc); 326 327 int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb) 328 { 329 int ret; 330 unsigned int noreclaim_flag; 331 332 /* these should have been dropped before queueing */ 333 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC)); 334 335 noreclaim_flag = memalloc_noreclaim_save(); 336 ret = INDIRECT_CALL_INET(sk->sk_backlog_rcv, 337 tcp_v6_do_rcv, 338 tcp_v4_do_rcv, 339 sk, skb); 340 memalloc_noreclaim_restore(noreclaim_flag); 341 342 return ret; 343 } 344 EXPORT_SYMBOL(__sk_backlog_rcv); 345 346 void sk_error_report(struct sock *sk) 347 { 348 sk->sk_error_report(sk); 349 350 switch (sk->sk_family) { 351 case AF_INET: 352 fallthrough; 353 case AF_INET6: 354 trace_inet_sk_error_report(sk); 355 break; 356 default: 357 break; 358 } 359 } 360 EXPORT_SYMBOL(sk_error_report); 361 362 int sock_get_timeout(long timeo, void *optval, bool old_timeval) 363 { 364 struct __kernel_sock_timeval tv; 365 366 if (timeo == MAX_SCHEDULE_TIMEOUT) { 367 tv.tv_sec = 0; 368 tv.tv_usec = 0; 369 } else { 370 tv.tv_sec = timeo / HZ; 371 tv.tv_usec = ((timeo % HZ) * USEC_PER_SEC) / HZ; 372 } 373 374 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) { 375 struct old_timeval32 tv32 = { tv.tv_sec, tv.tv_usec }; 376 *(struct old_timeval32 *)optval = tv32; 377 return sizeof(tv32); 378 } 379 380 if (old_timeval) { 381 struct __kernel_old_timeval old_tv; 382 old_tv.tv_sec = tv.tv_sec; 383 old_tv.tv_usec = tv.tv_usec; 384 *(struct __kernel_old_timeval *)optval = old_tv; 385 return sizeof(old_tv); 386 } 387 388 *(struct __kernel_sock_timeval *)optval = tv; 389 return sizeof(tv); 390 } 391 EXPORT_SYMBOL(sock_get_timeout); 392 393 int sock_copy_user_timeval(struct __kernel_sock_timeval *tv, 394 sockptr_t optval, int optlen, bool old_timeval) 395 { 396 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) { 397 struct old_timeval32 tv32; 398 399 if (optlen < sizeof(tv32)) 400 return -EINVAL; 401 402 if (copy_from_sockptr(&tv32, optval, sizeof(tv32))) 403 return -EFAULT; 404 tv->tv_sec = tv32.tv_sec; 405 tv->tv_usec = tv32.tv_usec; 406 } else if (old_timeval) { 407 struct __kernel_old_timeval old_tv; 408 409 if (optlen < sizeof(old_tv)) 410 return -EINVAL; 411 if (copy_from_sockptr(&old_tv, optval, sizeof(old_tv))) 412 return -EFAULT; 413 tv->tv_sec = old_tv.tv_sec; 414 tv->tv_usec = old_tv.tv_usec; 415 } else { 416 if (optlen < sizeof(*tv)) 417 return -EINVAL; 418 if (copy_from_sockptr(tv, optval, sizeof(*tv))) 419 return -EFAULT; 420 } 421 422 return 0; 423 } 424 EXPORT_SYMBOL(sock_copy_user_timeval); 425 426 static int sock_set_timeout(long *timeo_p, sockptr_t optval, int optlen, 427 bool old_timeval) 428 { 429 struct __kernel_sock_timeval tv; 430 int err = sock_copy_user_timeval(&tv, optval, optlen, old_timeval); 431 long val; 432 433 if (err) 434 return err; 435 436 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC) 437 return -EDOM; 438 439 if (tv.tv_sec < 0) { 440 static int warned __read_mostly; 441 442 WRITE_ONCE(*timeo_p, 0); 443 if (warned < 10 && net_ratelimit()) { 444 warned++; 445 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n", 446 __func__, current->comm, task_pid_nr(current)); 447 } 448 return 0; 449 } 450 val = MAX_SCHEDULE_TIMEOUT; 451 if ((tv.tv_sec || tv.tv_usec) && 452 (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT / HZ - 1))) 453 val = tv.tv_sec * HZ + DIV_ROUND_UP((unsigned long)tv.tv_usec, 454 USEC_PER_SEC / HZ); 455 WRITE_ONCE(*timeo_p, val); 456 return 0; 457 } 458 459 static bool sk_set_prio_allowed(const struct sock *sk, int val) 460 { 461 return ((val >= TC_PRIO_BESTEFFORT && val <= TC_PRIO_INTERACTIVE) || 462 sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) || 463 sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)); 464 } 465 466 static bool sock_needs_netstamp(const struct sock *sk) 467 { 468 switch (sk->sk_family) { 469 case AF_UNSPEC: 470 case AF_UNIX: 471 return false; 472 default: 473 return true; 474 } 475 } 476 477 static void sock_disable_timestamp(struct sock *sk, unsigned long flags) 478 { 479 if (sk->sk_flags & flags) { 480 sk->sk_flags &= ~flags; 481 if (sock_needs_netstamp(sk) && 482 !(sk->sk_flags & SK_FLAGS_TIMESTAMP)) 483 net_disable_timestamp(); 484 } 485 } 486 487 488 int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 489 { 490 unsigned long flags; 491 struct sk_buff_head *list = &sk->sk_receive_queue; 492 493 if (atomic_read(&sk->sk_rmem_alloc) >= READ_ONCE(sk->sk_rcvbuf)) { 494 sk_drops_inc(sk); 495 trace_sock_rcvqueue_full(sk, skb); 496 return -ENOMEM; 497 } 498 499 if (!sk_rmem_schedule(sk, skb, skb->truesize)) { 500 sk_drops_inc(sk); 501 return -ENOBUFS; 502 } 503 504 skb->dev = NULL; 505 skb_set_owner_r(skb, sk); 506 507 /* we escape from rcu protected region, make sure we dont leak 508 * a norefcounted dst 509 */ 510 skb_dst_force(skb); 511 512 spin_lock_irqsave(&list->lock, flags); 513 sock_skb_set_dropcount(sk, skb); 514 __skb_queue_tail(list, skb); 515 spin_unlock_irqrestore(&list->lock, flags); 516 517 if (!sock_flag(sk, SOCK_DEAD)) 518 sk->sk_data_ready(sk); 519 return 0; 520 } 521 EXPORT_SYMBOL(__sock_queue_rcv_skb); 522 523 enum skb_drop_reason 524 sock_queue_rcv_skb_reason(struct sock *sk, struct sk_buff *skb) 525 { 526 enum skb_drop_reason drop_reason; 527 int err; 528 529 drop_reason = sk_filter_reason(sk, skb); 530 if (drop_reason) 531 return drop_reason; 532 533 err = __sock_queue_rcv_skb(sk, skb); 534 switch (err) { 535 case -ENOMEM: 536 return SKB_DROP_REASON_SOCKET_RCVBUFF; 537 case -ENOBUFS: 538 return SKB_DROP_REASON_PROTO_MEM; 539 } 540 return SKB_NOT_DROPPED_YET; 541 } 542 EXPORT_SYMBOL(sock_queue_rcv_skb_reason); 543 544 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, 545 const int nested, unsigned int trim_cap, bool refcounted) 546 { 547 enum skb_drop_reason reason; 548 int rc = NET_RX_SUCCESS; 549 int err; 550 551 reason = sk_filter_trim_cap(sk, skb, trim_cap); 552 if (reason) 553 goto discard_and_relse; 554 555 skb->dev = NULL; 556 557 if (sk_rcvqueues_full(sk, READ_ONCE(sk->sk_rcvbuf))) { 558 sk_drops_inc(sk); 559 reason = SKB_DROP_REASON_SOCKET_RCVBUFF; 560 goto discard_and_relse; 561 } 562 if (nested) 563 bh_lock_sock_nested(sk); 564 else 565 bh_lock_sock(sk); 566 if (!sock_owned_by_user(sk)) { 567 /* 568 * trylock + unlock semantics: 569 */ 570 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_); 571 572 rc = sk_backlog_rcv(sk, skb); 573 574 mutex_release(&sk->sk_lock.dep_map, _RET_IP_); 575 } else if ((err = sk_add_backlog(sk, skb, READ_ONCE(sk->sk_rcvbuf)))) { 576 bh_unlock_sock(sk); 577 if (err == -ENOMEM) 578 reason = SKB_DROP_REASON_PFMEMALLOC; 579 if (err == -ENOBUFS) 580 reason = SKB_DROP_REASON_SOCKET_BACKLOG; 581 sk_drops_inc(sk); 582 goto discard_and_relse; 583 } 584 585 bh_unlock_sock(sk); 586 out: 587 if (refcounted) 588 sock_put(sk); 589 return rc; 590 discard_and_relse: 591 sk_skb_reason_drop(sk, skb, reason); 592 goto out; 593 } 594 EXPORT_SYMBOL(__sk_receive_skb); 595 596 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ip6_dst_check(struct dst_entry *, 597 u32)); 598 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *, 599 u32)); 600 struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie) 601 { 602 struct dst_entry *dst = __sk_dst_get(sk); 603 604 if (dst && READ_ONCE(dst->obsolete) && 605 INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check, 606 dst, cookie) == NULL) { 607 sk_tx_queue_clear(sk); 608 WRITE_ONCE(sk->sk_dst_pending_confirm, 0); 609 RCU_INIT_POINTER(sk->sk_dst_cache, NULL); 610 dst_release(dst); 611 return NULL; 612 } 613 614 return dst; 615 } 616 EXPORT_SYMBOL(__sk_dst_check); 617 618 struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie) 619 { 620 struct dst_entry *dst = sk_dst_get(sk); 621 622 if (dst && READ_ONCE(dst->obsolete) && 623 INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check, 624 dst, cookie) == NULL) { 625 sk_dst_reset(sk); 626 dst_release(dst); 627 return NULL; 628 } 629 630 return dst; 631 } 632 EXPORT_SYMBOL(sk_dst_check); 633 634 static int sock_bindtoindex_locked(struct sock *sk, int ifindex) 635 { 636 int ret = -ENOPROTOOPT; 637 #ifdef CONFIG_NETDEVICES 638 struct net *net = sock_net(sk); 639 640 /* Sorry... */ 641 ret = -EPERM; 642 if (sk->sk_bound_dev_if && !ns_capable(net->user_ns, CAP_NET_RAW)) 643 goto out; 644 645 ret = -EINVAL; 646 if (ifindex < 0) 647 goto out; 648 649 /* Paired with all READ_ONCE() done locklessly. */ 650 WRITE_ONCE(sk->sk_bound_dev_if, ifindex); 651 652 if (sk->sk_prot->rehash) 653 sk->sk_prot->rehash(sk); 654 sk_dst_reset(sk); 655 656 ret = 0; 657 658 out: 659 #endif 660 661 return ret; 662 } 663 664 int sock_bindtoindex(struct sock *sk, int ifindex, bool lock_sk) 665 { 666 int ret; 667 668 if (lock_sk) 669 lock_sock(sk); 670 ret = sock_bindtoindex_locked(sk, ifindex); 671 if (lock_sk) 672 release_sock(sk); 673 674 return ret; 675 } 676 EXPORT_SYMBOL(sock_bindtoindex); 677 678 static int sock_setbindtodevice(struct sock *sk, sockptr_t optval, int optlen) 679 { 680 int ret = -ENOPROTOOPT; 681 #ifdef CONFIG_NETDEVICES 682 struct net *net = sock_net(sk); 683 char devname[IFNAMSIZ]; 684 int index; 685 686 ret = -EINVAL; 687 if (optlen < 0) 688 goto out; 689 690 /* Bind this socket to a particular device like "eth0", 691 * as specified in the passed interface name. If the 692 * name is "" or the option length is zero the socket 693 * is not bound. 694 */ 695 if (optlen > IFNAMSIZ - 1) 696 optlen = IFNAMSIZ - 1; 697 memset(devname, 0, sizeof(devname)); 698 699 ret = -EFAULT; 700 if (copy_from_sockptr(devname, optval, optlen)) 701 goto out; 702 703 index = 0; 704 if (devname[0] != '\0') { 705 struct net_device *dev; 706 707 rcu_read_lock(); 708 dev = dev_get_by_name_rcu(net, devname); 709 if (dev) 710 index = dev->ifindex; 711 rcu_read_unlock(); 712 ret = -ENODEV; 713 if (!dev) 714 goto out; 715 } 716 717 sockopt_lock_sock(sk); 718 ret = sock_bindtoindex_locked(sk, index); 719 sockopt_release_sock(sk); 720 out: 721 #endif 722 723 return ret; 724 } 725 726 static int sock_getbindtodevice(struct sock *sk, sockptr_t optval, 727 sockptr_t optlen, int len) 728 { 729 int ret = -ENOPROTOOPT; 730 #ifdef CONFIG_NETDEVICES 731 int bound_dev_if = READ_ONCE(sk->sk_bound_dev_if); 732 struct net *net = sock_net(sk); 733 char devname[IFNAMSIZ]; 734 735 if (bound_dev_if == 0) { 736 len = 0; 737 goto zero; 738 } 739 740 ret = -EINVAL; 741 if (len < IFNAMSIZ) 742 goto out; 743 744 ret = netdev_get_name(net, devname, bound_dev_if); 745 if (ret) 746 goto out; 747 748 len = strlen(devname) + 1; 749 750 ret = -EFAULT; 751 if (copy_to_sockptr(optval, devname, len)) 752 goto out; 753 754 zero: 755 ret = -EFAULT; 756 if (copy_to_sockptr(optlen, &len, sizeof(int))) 757 goto out; 758 759 ret = 0; 760 761 out: 762 #endif 763 764 return ret; 765 } 766 767 bool sk_mc_loop(const struct sock *sk) 768 { 769 if (dev_recursion_level()) 770 return false; 771 if (!sk) 772 return true; 773 /* IPV6_ADDRFORM can change sk->sk_family under us. */ 774 switch (READ_ONCE(sk->sk_family)) { 775 case AF_INET: 776 return inet_test_bit(MC_LOOP, sk); 777 #if IS_ENABLED(CONFIG_IPV6) 778 case AF_INET6: 779 return inet6_test_bit(MC6_LOOP, sk); 780 #endif 781 } 782 return true; 783 } 784 EXPORT_SYMBOL(sk_mc_loop); 785 786 void sock_set_reuseaddr(struct sock *sk) 787 { 788 lock_sock(sk); 789 sk->sk_reuse = SK_CAN_REUSE; 790 release_sock(sk); 791 } 792 EXPORT_SYMBOL(sock_set_reuseaddr); 793 794 void sock_set_reuseport(struct sock *sk) 795 { 796 lock_sock(sk); 797 sk->sk_reuseport = true; 798 release_sock(sk); 799 } 800 EXPORT_SYMBOL(sock_set_reuseport); 801 802 void sock_no_linger(struct sock *sk) 803 { 804 lock_sock(sk); 805 WRITE_ONCE(sk->sk_lingertime, 0); 806 sock_set_flag(sk, SOCK_LINGER); 807 release_sock(sk); 808 } 809 EXPORT_SYMBOL(sock_no_linger); 810 811 void sock_set_priority(struct sock *sk, u32 priority) 812 { 813 WRITE_ONCE(sk->sk_priority, priority); 814 } 815 EXPORT_SYMBOL(sock_set_priority); 816 817 void sock_set_sndtimeo(struct sock *sk, s64 secs) 818 { 819 if (secs && secs < MAX_SCHEDULE_TIMEOUT / HZ - 1) 820 WRITE_ONCE(sk->sk_sndtimeo, secs * HZ); 821 else 822 WRITE_ONCE(sk->sk_sndtimeo, MAX_SCHEDULE_TIMEOUT); 823 } 824 EXPORT_SYMBOL(sock_set_sndtimeo); 825 826 static void __sock_set_timestamps(struct sock *sk, bool val, bool new, bool ns) 827 { 828 sock_valbool_flag(sk, SOCK_RCVTSTAMP, val); 829 sock_valbool_flag(sk, SOCK_RCVTSTAMPNS, val && ns); 830 if (val) { 831 sock_valbool_flag(sk, SOCK_TSTAMP_NEW, new); 832 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 833 } 834 } 835 836 void sock_set_timestamp(struct sock *sk, int optname, bool valbool) 837 { 838 switch (optname) { 839 case SO_TIMESTAMP_OLD: 840 __sock_set_timestamps(sk, valbool, false, false); 841 break; 842 case SO_TIMESTAMP_NEW: 843 __sock_set_timestamps(sk, valbool, true, false); 844 break; 845 case SO_TIMESTAMPNS_OLD: 846 __sock_set_timestamps(sk, valbool, false, true); 847 break; 848 case SO_TIMESTAMPNS_NEW: 849 __sock_set_timestamps(sk, valbool, true, true); 850 break; 851 } 852 } 853 854 static int sock_timestamping_bind_phc(struct sock *sk, int phc_index) 855 { 856 struct net *net = sock_net(sk); 857 struct net_device *dev = NULL; 858 bool match = false; 859 int *vclock_index; 860 int i, num; 861 862 if (sk->sk_bound_dev_if) 863 dev = dev_get_by_index(net, sk->sk_bound_dev_if); 864 865 if (!dev) { 866 pr_err("%s: sock not bind to device\n", __func__); 867 return -EOPNOTSUPP; 868 } 869 870 num = ethtool_get_phc_vclocks(dev, &vclock_index); 871 dev_put(dev); 872 873 for (i = 0; i < num; i++) { 874 if (*(vclock_index + i) == phc_index) { 875 match = true; 876 break; 877 } 878 } 879 880 if (num > 0) 881 kfree(vclock_index); 882 883 if (!match) 884 return -EINVAL; 885 886 WRITE_ONCE(sk->sk_bind_phc, phc_index); 887 888 return 0; 889 } 890 891 int sock_set_timestamping(struct sock *sk, int optname, 892 struct so_timestamping timestamping) 893 { 894 int val = timestamping.flags; 895 int ret; 896 897 if (val & ~SOF_TIMESTAMPING_MASK) 898 return -EINVAL; 899 900 if (val & SOF_TIMESTAMPING_OPT_ID_TCP && 901 !(val & SOF_TIMESTAMPING_OPT_ID)) 902 return -EINVAL; 903 904 if (val & SOF_TIMESTAMPING_OPT_ID && 905 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) { 906 if (sk_is_tcp(sk)) { 907 if ((1 << sk->sk_state) & 908 (TCPF_CLOSE | TCPF_LISTEN)) 909 return -EINVAL; 910 if (val & SOF_TIMESTAMPING_OPT_ID_TCP) 911 atomic_set(&sk->sk_tskey, tcp_sk(sk)->write_seq); 912 else 913 atomic_set(&sk->sk_tskey, tcp_sk(sk)->snd_una); 914 } else { 915 atomic_set(&sk->sk_tskey, 0); 916 } 917 } 918 919 if (val & SOF_TIMESTAMPING_OPT_STATS && 920 !(val & SOF_TIMESTAMPING_OPT_TSONLY)) 921 return -EINVAL; 922 923 if (val & SOF_TIMESTAMPING_BIND_PHC) { 924 ret = sock_timestamping_bind_phc(sk, timestamping.bind_phc); 925 if (ret) 926 return ret; 927 } 928 929 WRITE_ONCE(sk->sk_tsflags, val); 930 sock_valbool_flag(sk, SOCK_TSTAMP_NEW, optname == SO_TIMESTAMPING_NEW); 931 sock_valbool_flag(sk, SOCK_TIMESTAMPING_ANY, !!(val & TSFLAGS_ANY)); 932 933 if (val & SOF_TIMESTAMPING_RX_SOFTWARE) 934 sock_enable_timestamp(sk, 935 SOCK_TIMESTAMPING_RX_SOFTWARE); 936 else 937 sock_disable_timestamp(sk, 938 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE)); 939 return 0; 940 } 941 942 #if defined(CONFIG_CGROUP_BPF) 943 void bpf_skops_tx_timestamping(struct sock *sk, struct sk_buff *skb, int op) 944 { 945 struct bpf_sock_ops_kern sock_ops; 946 947 memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); 948 sock_ops.op = op; 949 sock_ops.is_fullsock = 1; 950 sock_ops.sk = sk; 951 bpf_skops_init_skb(&sock_ops, skb, 0); 952 __cgroup_bpf_run_filter_sock_ops(sk, &sock_ops, CGROUP_SOCK_OPS); 953 } 954 #endif 955 956 void sock_set_keepalive(struct sock *sk) 957 { 958 lock_sock(sk); 959 if (sk->sk_prot->keepalive) 960 sk->sk_prot->keepalive(sk, true); 961 sock_valbool_flag(sk, SOCK_KEEPOPEN, true); 962 release_sock(sk); 963 } 964 EXPORT_SYMBOL(sock_set_keepalive); 965 966 static void __sock_set_rcvbuf(struct sock *sk, int val) 967 { 968 struct socket *sock = sk->sk_socket; 969 970 /* Ensure val * 2 fits into an int, to prevent max_t() from treating it 971 * as a negative value. 972 */ 973 val = min_t(int, val, INT_MAX / 2); 974 sk->sk_userlocks |= SOCK_RCVBUF_LOCK; 975 976 /* We double it on the way in to account for "struct sk_buff" etc. 977 * overhead. Applications assume that the SO_RCVBUF setting they make 978 * will allow that much actual data to be received on that socket. 979 * 980 * Applications are unaware that "struct sk_buff" and other overheads 981 * allocate from the receive buffer during socket buffer allocation. 982 * 983 * And after considering the possible alternatives, returning the value 984 * we actually used in getsockopt is the most desirable behavior. 985 */ 986 WRITE_ONCE(sk->sk_rcvbuf, max_t(int, val * 2, SOCK_MIN_RCVBUF)); 987 988 if (sock) { 989 const struct proto_ops *ops = READ_ONCE(sock->ops); 990 991 if (ops->set_rcvbuf) 992 ops->set_rcvbuf(sk, sk->sk_rcvbuf); 993 } 994 } 995 996 void sock_set_rcvbuf(struct sock *sk, int val) 997 { 998 lock_sock(sk); 999 __sock_set_rcvbuf(sk, val); 1000 release_sock(sk); 1001 } 1002 EXPORT_SYMBOL(sock_set_rcvbuf); 1003 1004 static void __sock_set_mark(struct sock *sk, u32 val) 1005 { 1006 if (val != sk->sk_mark) { 1007 WRITE_ONCE(sk->sk_mark, val); 1008 sk_dst_reset(sk); 1009 } 1010 } 1011 1012 void sock_set_mark(struct sock *sk, u32 val) 1013 { 1014 lock_sock(sk); 1015 __sock_set_mark(sk, val); 1016 release_sock(sk); 1017 } 1018 EXPORT_SYMBOL(sock_set_mark); 1019 1020 static void sock_release_reserved_memory(struct sock *sk, int bytes) 1021 { 1022 /* Round down bytes to multiple of pages */ 1023 bytes = round_down(bytes, PAGE_SIZE); 1024 1025 WARN_ON(bytes > sk->sk_reserved_mem); 1026 WRITE_ONCE(sk->sk_reserved_mem, sk->sk_reserved_mem - bytes); 1027 sk_mem_reclaim(sk); 1028 } 1029 1030 static int sock_reserve_memory(struct sock *sk, int bytes) 1031 { 1032 long allocated; 1033 bool charged; 1034 int pages; 1035 1036 if (!mem_cgroup_sk_enabled(sk) || !sk_has_account(sk)) 1037 return -EOPNOTSUPP; 1038 1039 if (!bytes) 1040 return 0; 1041 1042 pages = sk_mem_pages(bytes); 1043 1044 /* pre-charge to memcg */ 1045 charged = mem_cgroup_sk_charge(sk, pages, 1046 GFP_KERNEL | __GFP_RETRY_MAYFAIL); 1047 if (!charged) 1048 return -ENOMEM; 1049 1050 if (sk->sk_bypass_prot_mem) 1051 goto success; 1052 1053 /* pre-charge to forward_alloc */ 1054 sk_memory_allocated_add(sk, pages); 1055 allocated = sk_memory_allocated(sk); 1056 1057 /* If the system goes into memory pressure with this 1058 * precharge, give up and return error. 1059 */ 1060 if (allocated > sk_prot_mem_limits(sk, 1)) { 1061 sk_memory_allocated_sub(sk, pages); 1062 mem_cgroup_sk_uncharge(sk, pages); 1063 return -ENOMEM; 1064 } 1065 1066 success: 1067 sk_forward_alloc_add(sk, pages << PAGE_SHIFT); 1068 1069 WRITE_ONCE(sk->sk_reserved_mem, 1070 sk->sk_reserved_mem + (pages << PAGE_SHIFT)); 1071 1072 return 0; 1073 } 1074 1075 #ifdef CONFIG_PAGE_POOL 1076 1077 /* This is the number of tokens and frags that the user can SO_DEVMEM_DONTNEED 1078 * in 1 syscall. The limit exists to limit the amount of memory the kernel 1079 * allocates to copy these tokens, and to prevent looping over the frags for 1080 * too long. 1081 */ 1082 #define MAX_DONTNEED_TOKENS 128 1083 #define MAX_DONTNEED_FRAGS 1024 1084 1085 static noinline_for_stack int 1086 sock_devmem_dontneed(struct sock *sk, sockptr_t optval, unsigned int optlen) 1087 { 1088 unsigned int num_tokens, i, j, k, netmem_num = 0; 1089 struct dmabuf_token *tokens; 1090 int ret = 0, num_frags = 0; 1091 netmem_ref netmems[16]; 1092 1093 if (!sk_is_tcp(sk)) 1094 return -EBADF; 1095 1096 if (optlen % sizeof(*tokens) || 1097 optlen > sizeof(*tokens) * MAX_DONTNEED_TOKENS) 1098 return -EINVAL; 1099 1100 num_tokens = optlen / sizeof(*tokens); 1101 tokens = kvmalloc_objs(*tokens, num_tokens); 1102 if (!tokens) 1103 return -ENOMEM; 1104 1105 if (copy_from_sockptr(tokens, optval, optlen)) { 1106 kvfree(tokens); 1107 return -EFAULT; 1108 } 1109 1110 xa_lock_bh(&sk->sk_user_frags); 1111 for (i = 0; i < num_tokens; i++) { 1112 for (j = 0; j < tokens[i].token_count; j++) { 1113 if (++num_frags > MAX_DONTNEED_FRAGS) 1114 goto frag_limit_reached; 1115 1116 netmem_ref netmem = (__force netmem_ref)__xa_erase( 1117 &sk->sk_user_frags, tokens[i].token_start + j); 1118 1119 if (!netmem || WARN_ON_ONCE(!netmem_is_net_iov(netmem))) 1120 continue; 1121 1122 netmems[netmem_num++] = netmem; 1123 if (netmem_num == ARRAY_SIZE(netmems)) { 1124 xa_unlock_bh(&sk->sk_user_frags); 1125 for (k = 0; k < netmem_num; k++) 1126 WARN_ON_ONCE(!napi_pp_put_page(netmems[k])); 1127 netmem_num = 0; 1128 xa_lock_bh(&sk->sk_user_frags); 1129 } 1130 ret++; 1131 } 1132 } 1133 1134 frag_limit_reached: 1135 xa_unlock_bh(&sk->sk_user_frags); 1136 for (k = 0; k < netmem_num; k++) 1137 WARN_ON_ONCE(!napi_pp_put_page(netmems[k])); 1138 1139 kvfree(tokens); 1140 return ret; 1141 } 1142 #endif 1143 1144 void sockopt_lock_sock(struct sock *sk) 1145 { 1146 /* When current->bpf_ctx is set, the setsockopt is called from 1147 * a bpf prog. bpf has ensured the sk lock has been 1148 * acquired before calling setsockopt(). 1149 */ 1150 if (has_current_bpf_ctx()) 1151 return; 1152 1153 lock_sock(sk); 1154 } 1155 EXPORT_SYMBOL(sockopt_lock_sock); 1156 1157 void sockopt_release_sock(struct sock *sk) 1158 { 1159 if (has_current_bpf_ctx()) 1160 return; 1161 1162 release_sock(sk); 1163 } 1164 EXPORT_SYMBOL(sockopt_release_sock); 1165 1166 bool sockopt_ns_capable(struct user_namespace *ns, int cap) 1167 { 1168 return has_current_bpf_ctx() || ns_capable(ns, cap); 1169 } 1170 EXPORT_SYMBOL(sockopt_ns_capable); 1171 1172 bool sockopt_capable(int cap) 1173 { 1174 return has_current_bpf_ctx() || capable(cap); 1175 } 1176 EXPORT_SYMBOL(sockopt_capable); 1177 1178 static int sockopt_validate_clockid(__kernel_clockid_t value) 1179 { 1180 switch (value) { 1181 case CLOCK_REALTIME: 1182 case CLOCK_MONOTONIC: 1183 case CLOCK_TAI: 1184 return 0; 1185 } 1186 return -EINVAL; 1187 } 1188 1189 /* 1190 * This is meant for all protocols to use and covers goings on 1191 * at the socket level. Everything here is generic. 1192 */ 1193 1194 int sk_setsockopt(struct sock *sk, int level, int optname, 1195 sockptr_t optval, unsigned int optlen) 1196 { 1197 struct so_timestamping timestamping; 1198 struct socket *sock = sk->sk_socket; 1199 struct sock_txtime sk_txtime; 1200 int val; 1201 int valbool; 1202 struct linger ling; 1203 int ret = 0; 1204 1205 /* 1206 * Options without arguments 1207 */ 1208 1209 if (optname == SO_BINDTODEVICE) 1210 return sock_setbindtodevice(sk, optval, optlen); 1211 1212 if (optlen < sizeof(int)) 1213 return -EINVAL; 1214 1215 if (copy_from_sockptr(&val, optval, sizeof(val))) 1216 return -EFAULT; 1217 1218 valbool = val ? 1 : 0; 1219 1220 /* handle options which do not require locking the socket. */ 1221 switch (optname) { 1222 case SO_PRIORITY: 1223 if (sk_set_prio_allowed(sk, val)) { 1224 sock_set_priority(sk, val); 1225 return 0; 1226 } 1227 return -EPERM; 1228 case SO_TYPE: 1229 case SO_PROTOCOL: 1230 case SO_DOMAIN: 1231 case SO_ERROR: 1232 return -ENOPROTOOPT; 1233 #ifdef CONFIG_NET_RX_BUSY_POLL 1234 case SO_BUSY_POLL: 1235 if (val < 0) 1236 return -EINVAL; 1237 WRITE_ONCE(sk->sk_ll_usec, val); 1238 return 0; 1239 case SO_PREFER_BUSY_POLL: 1240 if (valbool && !sockopt_capable(CAP_NET_ADMIN)) 1241 return -EPERM; 1242 WRITE_ONCE(sk->sk_prefer_busy_poll, valbool); 1243 return 0; 1244 case SO_BUSY_POLL_BUDGET: 1245 if (val > READ_ONCE(sk->sk_busy_poll_budget) && 1246 !sockopt_capable(CAP_NET_ADMIN)) 1247 return -EPERM; 1248 if (val < 0 || val > U16_MAX) 1249 return -EINVAL; 1250 WRITE_ONCE(sk->sk_busy_poll_budget, val); 1251 return 0; 1252 #endif 1253 case SO_MAX_PACING_RATE: 1254 { 1255 unsigned long ulval = (val == ~0U) ? ~0UL : (unsigned int)val; 1256 unsigned long pacing_rate; 1257 1258 if (sizeof(ulval) != sizeof(val) && 1259 optlen >= sizeof(ulval) && 1260 copy_from_sockptr(&ulval, optval, sizeof(ulval))) { 1261 return -EFAULT; 1262 } 1263 if (ulval != ~0UL) 1264 cmpxchg(&sk->sk_pacing_status, 1265 SK_PACING_NONE, 1266 SK_PACING_NEEDED); 1267 /* Pairs with READ_ONCE() from sk_getsockopt() */ 1268 WRITE_ONCE(sk->sk_max_pacing_rate, ulval); 1269 pacing_rate = READ_ONCE(sk->sk_pacing_rate); 1270 if (ulval < pacing_rate) 1271 WRITE_ONCE(sk->sk_pacing_rate, ulval); 1272 return 0; 1273 } 1274 case SO_TXREHASH: 1275 if (!sk_is_tcp(sk)) 1276 return -EOPNOTSUPP; 1277 if (val < -1 || val > 1) 1278 return -EINVAL; 1279 if ((u8)val == SOCK_TXREHASH_DEFAULT) 1280 val = READ_ONCE(sock_net(sk)->core.sysctl_txrehash); 1281 /* Paired with READ_ONCE() in tcp_rtx_synack() 1282 * and sk_getsockopt(). 1283 */ 1284 WRITE_ONCE(sk->sk_txrehash, (u8)val); 1285 return 0; 1286 case SO_PEEK_OFF: 1287 { 1288 int (*set_peek_off)(struct sock *sk, int val); 1289 1290 set_peek_off = READ_ONCE(sock->ops)->set_peek_off; 1291 if (set_peek_off) 1292 ret = set_peek_off(sk, val); 1293 else 1294 ret = -EOPNOTSUPP; 1295 return ret; 1296 } 1297 #ifdef CONFIG_PAGE_POOL 1298 case SO_DEVMEM_DONTNEED: 1299 return sock_devmem_dontneed(sk, optval, optlen); 1300 #endif 1301 case SO_SNDTIMEO_OLD: 1302 case SO_SNDTIMEO_NEW: 1303 return sock_set_timeout(&sk->sk_sndtimeo, optval, 1304 optlen, optname == SO_SNDTIMEO_OLD); 1305 case SO_RCVTIMEO_OLD: 1306 case SO_RCVTIMEO_NEW: 1307 return sock_set_timeout(&sk->sk_rcvtimeo, optval, 1308 optlen, optname == SO_RCVTIMEO_OLD); 1309 } 1310 1311 sockopt_lock_sock(sk); 1312 1313 switch (optname) { 1314 case SO_DEBUG: 1315 if (val && !sockopt_capable(CAP_NET_ADMIN)) 1316 ret = -EACCES; 1317 else 1318 sock_valbool_flag(sk, SOCK_DBG, valbool); 1319 break; 1320 case SO_REUSEADDR: 1321 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE); 1322 break; 1323 case SO_REUSEPORT: 1324 if (valbool && !sk_is_inet(sk)) 1325 ret = -EOPNOTSUPP; 1326 else 1327 sk->sk_reuseport = valbool; 1328 break; 1329 case SO_DONTROUTE: 1330 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool); 1331 sk_dst_reset(sk); 1332 break; 1333 case SO_BROADCAST: 1334 sock_valbool_flag(sk, SOCK_BROADCAST, valbool); 1335 break; 1336 case SO_SNDBUF: 1337 /* Don't error on this BSD doesn't and if you think 1338 * about it this is right. Otherwise apps have to 1339 * play 'guess the biggest size' games. RCVBUF/SNDBUF 1340 * are treated in BSD as hints 1341 */ 1342 val = min_t(u32, val, READ_ONCE(sysctl_wmem_max)); 1343 set_sndbuf: 1344 /* Ensure val * 2 fits into an int, to prevent max_t() 1345 * from treating it as a negative value. 1346 */ 1347 val = min_t(int, val, INT_MAX / 2); 1348 sk->sk_userlocks |= SOCK_SNDBUF_LOCK; 1349 WRITE_ONCE(sk->sk_sndbuf, 1350 max_t(int, val * 2, SOCK_MIN_SNDBUF)); 1351 /* Wake up sending tasks if we upped the value. */ 1352 sk->sk_write_space(sk); 1353 break; 1354 1355 case SO_SNDBUFFORCE: 1356 if (!sockopt_capable(CAP_NET_ADMIN)) { 1357 ret = -EPERM; 1358 break; 1359 } 1360 1361 /* No negative values (to prevent underflow, as val will be 1362 * multiplied by 2). 1363 */ 1364 if (val < 0) 1365 val = 0; 1366 goto set_sndbuf; 1367 1368 case SO_RCVBUF: 1369 /* Don't error on this BSD doesn't and if you think 1370 * about it this is right. Otherwise apps have to 1371 * play 'guess the biggest size' games. RCVBUF/SNDBUF 1372 * are treated in BSD as hints 1373 */ 1374 __sock_set_rcvbuf(sk, min_t(u32, val, READ_ONCE(sysctl_rmem_max))); 1375 break; 1376 1377 case SO_RCVBUFFORCE: 1378 if (!sockopt_capable(CAP_NET_ADMIN)) { 1379 ret = -EPERM; 1380 break; 1381 } 1382 1383 /* No negative values (to prevent underflow, as val will be 1384 * multiplied by 2). 1385 */ 1386 __sock_set_rcvbuf(sk, max(val, 0)); 1387 break; 1388 1389 case SO_KEEPALIVE: 1390 if (sk->sk_prot->keepalive) 1391 sk->sk_prot->keepalive(sk, valbool); 1392 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool); 1393 break; 1394 1395 case SO_OOBINLINE: 1396 sock_valbool_flag(sk, SOCK_URGINLINE, valbool); 1397 break; 1398 1399 case SO_NO_CHECK: 1400 sk->sk_no_check_tx = valbool; 1401 break; 1402 1403 case SO_LINGER: 1404 if (optlen < sizeof(ling)) { 1405 ret = -EINVAL; /* 1003.1g */ 1406 break; 1407 } 1408 if (copy_from_sockptr(&ling, optval, sizeof(ling))) { 1409 ret = -EFAULT; 1410 break; 1411 } 1412 if (!ling.l_onoff) { 1413 sock_reset_flag(sk, SOCK_LINGER); 1414 } else { 1415 unsigned long t_sec = ling.l_linger; 1416 1417 if (t_sec >= MAX_SCHEDULE_TIMEOUT / HZ) 1418 WRITE_ONCE(sk->sk_lingertime, MAX_SCHEDULE_TIMEOUT); 1419 else 1420 WRITE_ONCE(sk->sk_lingertime, t_sec * HZ); 1421 sock_set_flag(sk, SOCK_LINGER); 1422 } 1423 break; 1424 1425 case SO_BSDCOMPAT: 1426 break; 1427 1428 case SO_TIMESTAMP_OLD: 1429 case SO_TIMESTAMP_NEW: 1430 case SO_TIMESTAMPNS_OLD: 1431 case SO_TIMESTAMPNS_NEW: 1432 sock_set_timestamp(sk, optname, valbool); 1433 break; 1434 1435 case SO_TIMESTAMPING_NEW: 1436 case SO_TIMESTAMPING_OLD: 1437 if (optlen == sizeof(timestamping)) { 1438 if (copy_from_sockptr(×tamping, optval, 1439 sizeof(timestamping))) { 1440 ret = -EFAULT; 1441 break; 1442 } 1443 } else { 1444 memset(×tamping, 0, sizeof(timestamping)); 1445 timestamping.flags = val; 1446 } 1447 ret = sock_set_timestamping(sk, optname, timestamping); 1448 break; 1449 1450 case SO_RCVLOWAT: 1451 { 1452 int (*set_rcvlowat)(struct sock *sk, int val) = NULL; 1453 1454 if (val < 0) 1455 val = INT_MAX; 1456 if (sock) 1457 set_rcvlowat = READ_ONCE(sock->ops)->set_rcvlowat; 1458 if (set_rcvlowat) 1459 ret = set_rcvlowat(sk, val); 1460 else 1461 WRITE_ONCE(sk->sk_rcvlowat, val ? : 1); 1462 break; 1463 } 1464 case SO_ATTACH_FILTER: { 1465 struct sock_fprog fprog; 1466 1467 if (sk_is_tcp(sk) && 1468 !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1469 ret = -EPERM; 1470 break; 1471 } 1472 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen); 1473 if (!ret) 1474 ret = sk_attach_filter(&fprog, sk); 1475 break; 1476 } 1477 case SO_ATTACH_BPF: 1478 ret = -EINVAL; 1479 if (optlen == sizeof(u32)) { 1480 u32 ufd; 1481 1482 ret = -EFAULT; 1483 if (copy_from_sockptr(&ufd, optval, sizeof(ufd))) 1484 break; 1485 1486 ret = sk_attach_bpf(ufd, sk); 1487 } 1488 break; 1489 1490 case SO_ATTACH_REUSEPORT_CBPF: { 1491 struct sock_fprog fprog; 1492 1493 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen); 1494 if (!ret) 1495 ret = sk_reuseport_attach_filter(&fprog, sk); 1496 break; 1497 } 1498 case SO_ATTACH_REUSEPORT_EBPF: 1499 ret = -EINVAL; 1500 if (optlen == sizeof(u32)) { 1501 u32 ufd; 1502 1503 ret = -EFAULT; 1504 if (copy_from_sockptr(&ufd, optval, sizeof(ufd))) 1505 break; 1506 1507 ret = sk_reuseport_attach_bpf(ufd, sk); 1508 } 1509 break; 1510 1511 case SO_DETACH_REUSEPORT_BPF: 1512 ret = reuseport_detach_prog(sk); 1513 break; 1514 1515 case SO_DETACH_FILTER: 1516 ret = sk_detach_filter(sk); 1517 break; 1518 1519 case SO_LOCK_FILTER: 1520 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool) 1521 ret = -EPERM; 1522 else 1523 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool); 1524 break; 1525 1526 case SO_MARK: 1527 if (!sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && 1528 !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1529 ret = -EPERM; 1530 break; 1531 } 1532 1533 __sock_set_mark(sk, val); 1534 break; 1535 case SO_RCVMARK: 1536 sock_valbool_flag(sk, SOCK_RCVMARK, valbool); 1537 break; 1538 1539 case SO_RCVPRIORITY: 1540 sock_valbool_flag(sk, SOCK_RCVPRIORITY, valbool); 1541 break; 1542 1543 case SO_RXQ_OVFL: 1544 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool); 1545 break; 1546 1547 case SO_WIFI_STATUS: 1548 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool); 1549 break; 1550 1551 case SO_NOFCS: 1552 sock_valbool_flag(sk, SOCK_NOFCS, valbool); 1553 break; 1554 1555 case SO_SELECT_ERR_QUEUE: 1556 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool); 1557 break; 1558 1559 case SO_PASSCRED: 1560 if (sk_may_scm_recv(sk)) 1561 sk->sk_scm_credentials = valbool; 1562 else 1563 ret = -EOPNOTSUPP; 1564 break; 1565 1566 case SO_PASSSEC: 1567 if (IS_ENABLED(CONFIG_SECURITY_NETWORK) && sk_may_scm_recv(sk)) 1568 sk->sk_scm_security = valbool; 1569 else 1570 ret = -EOPNOTSUPP; 1571 break; 1572 1573 case SO_PASSPIDFD: 1574 if (sk_is_unix(sk)) 1575 sk->sk_scm_pidfd = valbool; 1576 else 1577 ret = -EOPNOTSUPP; 1578 break; 1579 1580 case SO_PASSRIGHTS: 1581 if (sk_is_unix(sk)) 1582 sk->sk_scm_rights = valbool; 1583 else 1584 ret = -EOPNOTSUPP; 1585 break; 1586 1587 case SO_INCOMING_CPU: 1588 reuseport_update_incoming_cpu(sk, val); 1589 break; 1590 1591 case SO_CNX_ADVICE: 1592 if (val == 1) 1593 dst_negative_advice(sk); 1594 break; 1595 1596 case SO_ZEROCOPY: 1597 if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) { 1598 if (!(sk_is_tcp(sk) || 1599 (sk->sk_type == SOCK_DGRAM && 1600 sk->sk_protocol == IPPROTO_UDP))) 1601 ret = -EOPNOTSUPP; 1602 } else if (sk->sk_family != PF_RDS) { 1603 ret = -EOPNOTSUPP; 1604 } 1605 if (!ret) { 1606 if (val < 0 || val > 1) 1607 ret = -EINVAL; 1608 else 1609 sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool); 1610 } 1611 break; 1612 1613 case SO_TXTIME: 1614 if (optlen != sizeof(struct sock_txtime)) { 1615 ret = -EINVAL; 1616 break; 1617 } else if (copy_from_sockptr(&sk_txtime, optval, 1618 sizeof(struct sock_txtime))) { 1619 ret = -EFAULT; 1620 break; 1621 } else if (sk_txtime.flags & ~SOF_TXTIME_FLAGS_MASK) { 1622 ret = -EINVAL; 1623 break; 1624 } 1625 /* CLOCK_MONOTONIC is only used by sch_fq, and this packet 1626 * scheduler has enough safe guards. 1627 */ 1628 if (sk_txtime.clockid != CLOCK_MONOTONIC && 1629 !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1630 ret = -EPERM; 1631 break; 1632 } 1633 1634 ret = sockopt_validate_clockid(sk_txtime.clockid); 1635 if (ret) 1636 break; 1637 1638 sock_valbool_flag(sk, SOCK_TXTIME, true); 1639 sk->sk_clockid = sk_txtime.clockid; 1640 sk->sk_txtime_deadline_mode = 1641 !!(sk_txtime.flags & SOF_TXTIME_DEADLINE_MODE); 1642 sk->sk_txtime_report_errors = 1643 !!(sk_txtime.flags & SOF_TXTIME_REPORT_ERRORS); 1644 break; 1645 1646 case SO_BINDTOIFINDEX: 1647 ret = sock_bindtoindex_locked(sk, val); 1648 break; 1649 1650 case SO_BUF_LOCK: 1651 if (val & ~SOCK_BUF_LOCK_MASK) { 1652 ret = -EINVAL; 1653 break; 1654 } 1655 sk->sk_userlocks = val | (sk->sk_userlocks & 1656 ~SOCK_BUF_LOCK_MASK); 1657 break; 1658 1659 case SO_RESERVE_MEM: 1660 { 1661 int delta; 1662 1663 if (val < 0) { 1664 ret = -EINVAL; 1665 break; 1666 } 1667 1668 delta = val - sk->sk_reserved_mem; 1669 if (delta < 0) 1670 sock_release_reserved_memory(sk, -delta); 1671 else 1672 ret = sock_reserve_memory(sk, delta); 1673 break; 1674 } 1675 1676 default: 1677 ret = -ENOPROTOOPT; 1678 break; 1679 } 1680 sockopt_release_sock(sk); 1681 return ret; 1682 } 1683 1684 int sock_setsockopt(struct socket *sock, int level, int optname, 1685 sockptr_t optval, unsigned int optlen) 1686 { 1687 return sk_setsockopt(sock->sk, level, optname, 1688 optval, optlen); 1689 } 1690 EXPORT_SYMBOL(sock_setsockopt); 1691 1692 static const struct cred *sk_get_peer_cred(struct sock *sk) 1693 { 1694 const struct cred *cred; 1695 1696 spin_lock(&sk->sk_peer_lock); 1697 cred = get_cred(sk->sk_peer_cred); 1698 spin_unlock(&sk->sk_peer_lock); 1699 1700 return cred; 1701 } 1702 1703 static void cred_to_ucred(struct pid *pid, const struct cred *cred, 1704 struct ucred *ucred) 1705 { 1706 ucred->pid = pid_vnr(pid); 1707 ucred->uid = ucred->gid = -1; 1708 if (cred) { 1709 struct user_namespace *current_ns = current_user_ns(); 1710 1711 ucred->uid = from_kuid_munged(current_ns, cred->euid); 1712 ucred->gid = from_kgid_munged(current_ns, cred->egid); 1713 } 1714 } 1715 1716 static int groups_to_user(sockptr_t dst, const struct group_info *src) 1717 { 1718 struct user_namespace *user_ns = current_user_ns(); 1719 int i; 1720 1721 for (i = 0; i < src->ngroups; i++) { 1722 gid_t gid = from_kgid_munged(user_ns, src->gid[i]); 1723 1724 if (copy_to_sockptr_offset(dst, i * sizeof(gid), &gid, sizeof(gid))) 1725 return -EFAULT; 1726 } 1727 1728 return 0; 1729 } 1730 1731 int sk_getsockopt(struct sock *sk, int level, int optname, 1732 sockptr_t optval, sockptr_t optlen) 1733 { 1734 struct socket *sock = sk->sk_socket; 1735 1736 union { 1737 int val; 1738 u64 val64; 1739 unsigned long ulval; 1740 struct linger ling; 1741 struct old_timeval32 tm32; 1742 struct __kernel_old_timeval tm; 1743 struct __kernel_sock_timeval stm; 1744 struct sock_txtime txtime; 1745 struct so_timestamping timestamping; 1746 } v; 1747 1748 int lv = sizeof(int); 1749 int len; 1750 1751 if (copy_from_sockptr(&len, optlen, sizeof(int))) 1752 return -EFAULT; 1753 if (len < 0) 1754 return -EINVAL; 1755 1756 memset(&v, 0, sizeof(v)); 1757 1758 switch (optname) { 1759 case SO_DEBUG: 1760 v.val = sock_flag(sk, SOCK_DBG); 1761 break; 1762 1763 case SO_DONTROUTE: 1764 v.val = sock_flag(sk, SOCK_LOCALROUTE); 1765 break; 1766 1767 case SO_BROADCAST: 1768 v.val = sock_flag(sk, SOCK_BROADCAST); 1769 break; 1770 1771 case SO_SNDBUF: 1772 v.val = READ_ONCE(sk->sk_sndbuf); 1773 break; 1774 1775 case SO_RCVBUF: 1776 v.val = READ_ONCE(sk->sk_rcvbuf); 1777 break; 1778 1779 case SO_REUSEADDR: 1780 v.val = sk->sk_reuse; 1781 break; 1782 1783 case SO_REUSEPORT: 1784 v.val = sk->sk_reuseport; 1785 break; 1786 1787 case SO_KEEPALIVE: 1788 v.val = sock_flag(sk, SOCK_KEEPOPEN); 1789 break; 1790 1791 case SO_TYPE: 1792 v.val = sk->sk_type; 1793 break; 1794 1795 case SO_PROTOCOL: 1796 v.val = sk->sk_protocol; 1797 break; 1798 1799 case SO_DOMAIN: 1800 v.val = sk->sk_family; 1801 break; 1802 1803 case SO_ERROR: 1804 v.val = -sock_error(sk); 1805 if (v.val == 0) 1806 v.val = xchg(&sk->sk_err_soft, 0); 1807 break; 1808 1809 case SO_OOBINLINE: 1810 v.val = sock_flag(sk, SOCK_URGINLINE); 1811 break; 1812 1813 case SO_NO_CHECK: 1814 v.val = sk->sk_no_check_tx; 1815 break; 1816 1817 case SO_PRIORITY: 1818 v.val = READ_ONCE(sk->sk_priority); 1819 break; 1820 1821 case SO_LINGER: 1822 lv = sizeof(v.ling); 1823 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER); 1824 v.ling.l_linger = READ_ONCE(sk->sk_lingertime) / HZ; 1825 break; 1826 1827 case SO_BSDCOMPAT: 1828 break; 1829 1830 case SO_TIMESTAMP_OLD: 1831 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && 1832 !sock_flag(sk, SOCK_TSTAMP_NEW) && 1833 !sock_flag(sk, SOCK_RCVTSTAMPNS); 1834 break; 1835 1836 case SO_TIMESTAMPNS_OLD: 1837 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && !sock_flag(sk, SOCK_TSTAMP_NEW); 1838 break; 1839 1840 case SO_TIMESTAMP_NEW: 1841 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && sock_flag(sk, SOCK_TSTAMP_NEW); 1842 break; 1843 1844 case SO_TIMESTAMPNS_NEW: 1845 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && sock_flag(sk, SOCK_TSTAMP_NEW); 1846 break; 1847 1848 case SO_TIMESTAMPING_OLD: 1849 case SO_TIMESTAMPING_NEW: 1850 lv = sizeof(v.timestamping); 1851 /* For the later-added case SO_TIMESTAMPING_NEW: Be strict about only 1852 * returning the flags when they were set through the same option. 1853 * Don't change the beviour for the old case SO_TIMESTAMPING_OLD. 1854 */ 1855 if (optname == SO_TIMESTAMPING_OLD || sock_flag(sk, SOCK_TSTAMP_NEW)) { 1856 v.timestamping.flags = READ_ONCE(sk->sk_tsflags); 1857 v.timestamping.bind_phc = READ_ONCE(sk->sk_bind_phc); 1858 } 1859 break; 1860 1861 case SO_RCVTIMEO_OLD: 1862 case SO_RCVTIMEO_NEW: 1863 lv = sock_get_timeout(READ_ONCE(sk->sk_rcvtimeo), &v, 1864 SO_RCVTIMEO_OLD == optname); 1865 break; 1866 1867 case SO_SNDTIMEO_OLD: 1868 case SO_SNDTIMEO_NEW: 1869 lv = sock_get_timeout(READ_ONCE(sk->sk_sndtimeo), &v, 1870 SO_SNDTIMEO_OLD == optname); 1871 break; 1872 1873 case SO_RCVLOWAT: 1874 v.val = READ_ONCE(sk->sk_rcvlowat); 1875 break; 1876 1877 case SO_SNDLOWAT: 1878 v.val = 1; 1879 break; 1880 1881 case SO_PASSCRED: 1882 if (!sk_may_scm_recv(sk)) 1883 return -EOPNOTSUPP; 1884 1885 v.val = sk->sk_scm_credentials; 1886 break; 1887 1888 case SO_PASSPIDFD: 1889 if (!sk_is_unix(sk)) 1890 return -EOPNOTSUPP; 1891 1892 v.val = sk->sk_scm_pidfd; 1893 break; 1894 1895 case SO_PASSRIGHTS: 1896 if (!sk_is_unix(sk)) 1897 return -EOPNOTSUPP; 1898 1899 v.val = sk->sk_scm_rights; 1900 break; 1901 1902 case SO_PEERCRED: 1903 { 1904 struct ucred peercred; 1905 if (len > sizeof(peercred)) 1906 len = sizeof(peercred); 1907 1908 spin_lock(&sk->sk_peer_lock); 1909 cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred); 1910 spin_unlock(&sk->sk_peer_lock); 1911 1912 if (copy_to_sockptr(optval, &peercred, len)) 1913 return -EFAULT; 1914 goto lenout; 1915 } 1916 1917 case SO_PEERPIDFD: 1918 { 1919 struct pid *peer_pid; 1920 struct file *pidfd_file = NULL; 1921 unsigned int flags = 0; 1922 int pidfd; 1923 1924 if (len > sizeof(pidfd)) 1925 len = sizeof(pidfd); 1926 1927 spin_lock(&sk->sk_peer_lock); 1928 peer_pid = get_pid(sk->sk_peer_pid); 1929 spin_unlock(&sk->sk_peer_lock); 1930 1931 if (!peer_pid) 1932 return -ENODATA; 1933 1934 /* The use of PIDFD_STALE requires stashing of struct pid 1935 * on pidfs with pidfs_register_pid() and only AF_UNIX 1936 * were prepared for this. 1937 */ 1938 if (sk->sk_family == AF_UNIX) 1939 flags = PIDFD_STALE; 1940 1941 pidfd = pidfd_prepare(peer_pid, flags, &pidfd_file); 1942 put_pid(peer_pid); 1943 if (pidfd < 0) 1944 return pidfd; 1945 1946 if (copy_to_sockptr(optval, &pidfd, len) || 1947 copy_to_sockptr(optlen, &len, sizeof(int))) { 1948 put_unused_fd(pidfd); 1949 fput(pidfd_file); 1950 1951 return -EFAULT; 1952 } 1953 1954 fd_install(pidfd, pidfd_file); 1955 return 0; 1956 } 1957 1958 case SO_PEERGROUPS: 1959 { 1960 const struct cred *cred; 1961 int ret, n; 1962 1963 cred = sk_get_peer_cred(sk); 1964 if (!cred) 1965 return -ENODATA; 1966 1967 n = cred->group_info->ngroups; 1968 if (len < n * sizeof(gid_t)) { 1969 len = n * sizeof(gid_t); 1970 put_cred(cred); 1971 return copy_to_sockptr(optlen, &len, sizeof(int)) ? -EFAULT : -ERANGE; 1972 } 1973 len = n * sizeof(gid_t); 1974 1975 ret = groups_to_user(optval, cred->group_info); 1976 put_cred(cred); 1977 if (ret) 1978 return ret; 1979 goto lenout; 1980 } 1981 1982 case SO_PEERNAME: 1983 { 1984 struct sockaddr_storage address; 1985 1986 lv = READ_ONCE(sock->ops)->getname(sock, (struct sockaddr *)&address, 2); 1987 if (lv < 0) 1988 return -ENOTCONN; 1989 if (lv < len) 1990 return -EINVAL; 1991 if (copy_to_sockptr(optval, &address, len)) 1992 return -EFAULT; 1993 goto lenout; 1994 } 1995 1996 /* Dubious BSD thing... Probably nobody even uses it, but 1997 * the UNIX standard wants it for whatever reason... -DaveM 1998 */ 1999 case SO_ACCEPTCONN: 2000 v.val = sk->sk_state == TCP_LISTEN; 2001 break; 2002 2003 case SO_PASSSEC: 2004 if (!IS_ENABLED(CONFIG_SECURITY_NETWORK) || !sk_may_scm_recv(sk)) 2005 return -EOPNOTSUPP; 2006 2007 v.val = sk->sk_scm_security; 2008 break; 2009 2010 case SO_PEERSEC: 2011 return security_socket_getpeersec_stream(sock, 2012 optval, optlen, len); 2013 2014 case SO_MARK: 2015 v.val = READ_ONCE(sk->sk_mark); 2016 break; 2017 2018 case SO_RCVMARK: 2019 v.val = sock_flag(sk, SOCK_RCVMARK); 2020 break; 2021 2022 case SO_RCVPRIORITY: 2023 v.val = sock_flag(sk, SOCK_RCVPRIORITY); 2024 break; 2025 2026 case SO_RXQ_OVFL: 2027 v.val = sock_flag(sk, SOCK_RXQ_OVFL); 2028 break; 2029 2030 case SO_WIFI_STATUS: 2031 v.val = sock_flag(sk, SOCK_WIFI_STATUS); 2032 break; 2033 2034 case SO_PEEK_OFF: 2035 if (!READ_ONCE(sock->ops)->set_peek_off) 2036 return -EOPNOTSUPP; 2037 2038 v.val = READ_ONCE(sk->sk_peek_off); 2039 break; 2040 case SO_NOFCS: 2041 v.val = sock_flag(sk, SOCK_NOFCS); 2042 break; 2043 2044 case SO_BINDTODEVICE: 2045 return sock_getbindtodevice(sk, optval, optlen, len); 2046 2047 case SO_GET_FILTER: 2048 len = sk_get_filter(sk, optval, len); 2049 if (len < 0) 2050 return len; 2051 2052 goto lenout; 2053 2054 case SO_LOCK_FILTER: 2055 v.val = sock_flag(sk, SOCK_FILTER_LOCKED); 2056 break; 2057 2058 case SO_BPF_EXTENSIONS: 2059 v.val = bpf_tell_extensions(); 2060 break; 2061 2062 case SO_SELECT_ERR_QUEUE: 2063 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE); 2064 break; 2065 2066 #ifdef CONFIG_NET_RX_BUSY_POLL 2067 case SO_BUSY_POLL: 2068 v.val = READ_ONCE(sk->sk_ll_usec); 2069 break; 2070 case SO_PREFER_BUSY_POLL: 2071 v.val = READ_ONCE(sk->sk_prefer_busy_poll); 2072 break; 2073 #endif 2074 2075 case SO_MAX_PACING_RATE: 2076 /* The READ_ONCE() pair with the WRITE_ONCE() in sk_setsockopt() */ 2077 if (sizeof(v.ulval) != sizeof(v.val) && len >= sizeof(v.ulval)) { 2078 lv = sizeof(v.ulval); 2079 v.ulval = READ_ONCE(sk->sk_max_pacing_rate); 2080 } else { 2081 /* 32bit version */ 2082 v.val = min_t(unsigned long, ~0U, 2083 READ_ONCE(sk->sk_max_pacing_rate)); 2084 } 2085 break; 2086 2087 case SO_INCOMING_CPU: 2088 v.val = READ_ONCE(sk->sk_incoming_cpu); 2089 break; 2090 2091 case SO_MEMINFO: 2092 { 2093 u32 meminfo[SK_MEMINFO_VARS]; 2094 2095 sk_get_meminfo(sk, meminfo); 2096 2097 len = min_t(unsigned int, len, sizeof(meminfo)); 2098 if (copy_to_sockptr(optval, &meminfo, len)) 2099 return -EFAULT; 2100 2101 goto lenout; 2102 } 2103 2104 #ifdef CONFIG_NET_RX_BUSY_POLL 2105 case SO_INCOMING_NAPI_ID: 2106 v.val = READ_ONCE(sk->sk_napi_id); 2107 2108 /* aggregate non-NAPI IDs down to 0 */ 2109 if (!napi_id_valid(v.val)) 2110 v.val = 0; 2111 2112 break; 2113 #endif 2114 2115 case SO_COOKIE: 2116 lv = sizeof(u64); 2117 if (len < lv) 2118 return -EINVAL; 2119 v.val64 = sock_gen_cookie(sk); 2120 break; 2121 2122 case SO_ZEROCOPY: 2123 v.val = sock_flag(sk, SOCK_ZEROCOPY); 2124 break; 2125 2126 case SO_TXTIME: 2127 lv = sizeof(v.txtime); 2128 v.txtime.clockid = sk->sk_clockid; 2129 v.txtime.flags |= sk->sk_txtime_deadline_mode ? 2130 SOF_TXTIME_DEADLINE_MODE : 0; 2131 v.txtime.flags |= sk->sk_txtime_report_errors ? 2132 SOF_TXTIME_REPORT_ERRORS : 0; 2133 break; 2134 2135 case SO_BINDTOIFINDEX: 2136 v.val = READ_ONCE(sk->sk_bound_dev_if); 2137 break; 2138 2139 case SO_NETNS_COOKIE: 2140 lv = sizeof(u64); 2141 if (len != lv) 2142 return -EINVAL; 2143 v.val64 = sock_net(sk)->net_cookie; 2144 break; 2145 2146 case SO_BUF_LOCK: 2147 v.val = sk->sk_userlocks & SOCK_BUF_LOCK_MASK; 2148 break; 2149 2150 case SO_RESERVE_MEM: 2151 v.val = READ_ONCE(sk->sk_reserved_mem); 2152 break; 2153 2154 case SO_TXREHASH: 2155 if (!sk_is_tcp(sk)) 2156 return -EOPNOTSUPP; 2157 2158 /* Paired with WRITE_ONCE() in sk_setsockopt() */ 2159 v.val = READ_ONCE(sk->sk_txrehash); 2160 break; 2161 2162 default: 2163 /* We implement the SO_SNDLOWAT etc to not be settable 2164 * (1003.1g 7). 2165 */ 2166 return -ENOPROTOOPT; 2167 } 2168 2169 if (len > lv) 2170 len = lv; 2171 if (copy_to_sockptr(optval, &v, len)) 2172 return -EFAULT; 2173 lenout: 2174 if (copy_to_sockptr(optlen, &len, sizeof(int))) 2175 return -EFAULT; 2176 return 0; 2177 } 2178 2179 /* 2180 * Initialize an sk_lock. 2181 * 2182 * (We also register the sk_lock with the lock validator.) 2183 */ 2184 static inline void sock_lock_init(struct sock *sk) 2185 { 2186 sk_owner_clear(sk); 2187 2188 if (sk->sk_kern_sock) 2189 sock_lock_init_class_and_name( 2190 sk, 2191 af_family_kern_slock_key_strings[sk->sk_family], 2192 af_family_kern_slock_keys + sk->sk_family, 2193 af_family_kern_key_strings[sk->sk_family], 2194 af_family_kern_keys + sk->sk_family); 2195 else 2196 sock_lock_init_class_and_name( 2197 sk, 2198 af_family_slock_key_strings[sk->sk_family], 2199 af_family_slock_keys + sk->sk_family, 2200 af_family_key_strings[sk->sk_family], 2201 af_family_keys + sk->sk_family); 2202 } 2203 2204 /* 2205 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet, 2206 * even temporarily, because of RCU lookups. sk_node should also be left as is. 2207 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end 2208 */ 2209 static void sock_copy(struct sock *nsk, const struct sock *osk) 2210 { 2211 const struct proto *prot = READ_ONCE(osk->sk_prot); 2212 #ifdef CONFIG_SECURITY_NETWORK 2213 void *sptr = nsk->sk_security; 2214 #endif 2215 2216 /* If we move sk_tx_queue_mapping out of the private section, 2217 * we must check if sk_tx_queue_clear() is called after 2218 * sock_copy() in sk_clone_lock(). 2219 */ 2220 BUILD_BUG_ON(offsetof(struct sock, sk_tx_queue_mapping) < 2221 offsetof(struct sock, sk_dontcopy_begin) || 2222 offsetof(struct sock, sk_tx_queue_mapping) >= 2223 offsetof(struct sock, sk_dontcopy_end)); 2224 2225 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin)); 2226 2227 unsafe_memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end, 2228 prot->obj_size - offsetof(struct sock, sk_dontcopy_end), 2229 /* alloc is larger than struct, see sk_prot_alloc() */); 2230 2231 #ifdef CONFIG_SECURITY_NETWORK 2232 nsk->sk_security = sptr; 2233 security_sk_clone(osk, nsk); 2234 #endif 2235 } 2236 2237 static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority, 2238 int family) 2239 { 2240 struct sock *sk; 2241 struct kmem_cache *slab; 2242 2243 slab = prot->slab; 2244 if (slab != NULL) { 2245 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO); 2246 if (!sk) 2247 return sk; 2248 if (want_init_on_alloc(priority)) 2249 sk_prot_clear_nulls(sk, prot->obj_size); 2250 } else 2251 sk = kmalloc(prot->obj_size, priority); 2252 2253 if (sk != NULL) { 2254 if (security_sk_alloc(sk, family, priority)) 2255 goto out_free; 2256 2257 if (!try_module_get(prot->owner)) 2258 goto out_free_sec; 2259 } 2260 2261 return sk; 2262 2263 out_free_sec: 2264 security_sk_free(sk); 2265 out_free: 2266 if (slab != NULL) 2267 kmem_cache_free(slab, sk); 2268 else 2269 kfree(sk); 2270 return NULL; 2271 } 2272 2273 static void sk_prot_free(struct proto *prot, struct sock *sk) 2274 { 2275 struct kmem_cache *slab; 2276 struct module *owner; 2277 2278 owner = prot->owner; 2279 slab = prot->slab; 2280 2281 cgroup_sk_free(&sk->sk_cgrp_data); 2282 mem_cgroup_sk_free(sk); 2283 security_sk_free(sk); 2284 2285 sk_owner_put(sk); 2286 2287 if (slab != NULL) 2288 kmem_cache_free(slab, sk); 2289 else 2290 kfree(sk); 2291 module_put(owner); 2292 } 2293 2294 /** 2295 * sk_alloc - All socket objects are allocated here 2296 * @net: the applicable net namespace 2297 * @family: protocol family 2298 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 2299 * @prot: struct proto associated with this new sock instance 2300 * @kern: is this to be a kernel socket? 2301 */ 2302 struct sock *sk_alloc(struct net *net, int family, gfp_t priority, 2303 struct proto *prot, int kern) 2304 { 2305 struct sock *sk; 2306 2307 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family); 2308 if (sk) { 2309 sk->sk_family = family; 2310 /* 2311 * See comment in struct sock definition to understand 2312 * why we need sk_prot_creator -acme 2313 */ 2314 sk->sk_prot = sk->sk_prot_creator = prot; 2315 2316 if (READ_ONCE(net->core.sysctl_bypass_prot_mem)) 2317 sk->sk_bypass_prot_mem = 1; 2318 2319 sk->sk_kern_sock = kern; 2320 sock_lock_init(sk); 2321 2322 sk->sk_net_refcnt = kern ? 0 : 1; 2323 if (likely(sk->sk_net_refcnt)) { 2324 get_net_track(net, &sk->ns_tracker, priority); 2325 sock_inuse_add(net, 1); 2326 } else { 2327 net_passive_inc(net); 2328 __netns_tracker_alloc(net, &sk->ns_tracker, 2329 false, priority); 2330 } 2331 2332 sock_net_set(sk, net); 2333 refcount_set(&sk->sk_wmem_alloc, SK_WMEM_ALLOC_BIAS); 2334 2335 mem_cgroup_sk_alloc(sk); 2336 cgroup_sk_alloc(&sk->sk_cgrp_data); 2337 sock_update_classid(&sk->sk_cgrp_data); 2338 sock_update_netprioidx(&sk->sk_cgrp_data); 2339 sk_tx_queue_clear(sk); 2340 } 2341 2342 return sk; 2343 } 2344 EXPORT_SYMBOL(sk_alloc); 2345 2346 /* Sockets having SOCK_RCU_FREE will call this function after one RCU 2347 * grace period. This is the case for UDP sockets and TCP listeners. 2348 */ 2349 static void __sk_destruct(struct rcu_head *head) 2350 { 2351 struct sock *sk = container_of(head, struct sock, sk_rcu); 2352 struct net *net = sock_net(sk); 2353 struct sk_filter *filter; 2354 2355 if (sk->sk_destruct) 2356 sk->sk_destruct(sk); 2357 2358 filter = rcu_dereference_check(sk->sk_filter, 2359 refcount_read(&sk->sk_wmem_alloc) == 0); 2360 if (filter) { 2361 sk_filter_uncharge(sk, filter); 2362 RCU_INIT_POINTER(sk->sk_filter, NULL); 2363 } 2364 2365 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP); 2366 2367 #ifdef CONFIG_BPF_SYSCALL 2368 bpf_sk_storage_free(sk); 2369 #endif 2370 2371 if (atomic_read(&sk->sk_omem_alloc)) 2372 pr_debug("%s: optmem leakage (%d bytes) detected\n", 2373 __func__, atomic_read(&sk->sk_omem_alloc)); 2374 2375 if (sk->sk_frag.page) { 2376 put_page(sk->sk_frag.page); 2377 sk->sk_frag.page = NULL; 2378 } 2379 2380 /* We do not need to acquire sk->sk_peer_lock, we are the last user. */ 2381 put_cred(sk->sk_peer_cred); 2382 put_pid(sk->sk_peer_pid); 2383 2384 if (likely(sk->sk_net_refcnt)) { 2385 put_net_track(net, &sk->ns_tracker); 2386 } else { 2387 __netns_tracker_free(net, &sk->ns_tracker, false); 2388 net_passive_dec(net); 2389 } 2390 sk_prot_free(sk->sk_prot_creator, sk); 2391 } 2392 2393 void sk_net_refcnt_upgrade(struct sock *sk) 2394 { 2395 struct net *net = sock_net(sk); 2396 2397 WARN_ON_ONCE(sk->sk_net_refcnt); 2398 __netns_tracker_free(net, &sk->ns_tracker, false); 2399 net_passive_dec(net); 2400 sk->sk_net_refcnt = 1; 2401 get_net_track(net, &sk->ns_tracker, GFP_KERNEL); 2402 sock_inuse_add(net, 1); 2403 } 2404 EXPORT_SYMBOL_GPL(sk_net_refcnt_upgrade); 2405 2406 void sk_destruct(struct sock *sk) 2407 { 2408 bool use_call_rcu = sock_flag(sk, SOCK_RCU_FREE); 2409 2410 if (rcu_access_pointer(sk->sk_reuseport_cb)) { 2411 reuseport_detach_sock(sk); 2412 use_call_rcu = true; 2413 } 2414 2415 if (use_call_rcu) 2416 call_rcu(&sk->sk_rcu, __sk_destruct); 2417 else 2418 __sk_destruct(&sk->sk_rcu); 2419 } 2420 2421 static void __sk_free(struct sock *sk) 2422 { 2423 if (likely(sk->sk_net_refcnt)) 2424 sock_inuse_add(sock_net(sk), -1); 2425 2426 if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk))) 2427 sock_diag_broadcast_destroy(sk); 2428 else 2429 sk_destruct(sk); 2430 } 2431 2432 void sk_free(struct sock *sk) 2433 { 2434 /* 2435 * We subtract one from sk_wmem_alloc and can know if 2436 * some packets are still in some tx queue. 2437 * If not null, sock_wfree() will call __sk_free(sk) later 2438 */ 2439 if (refcount_dec_and_test(&sk->sk_wmem_alloc)) 2440 __sk_free(sk); 2441 } 2442 EXPORT_SYMBOL(sk_free); 2443 2444 static void sk_init_common(struct sock *sk) 2445 { 2446 skb_queue_head_init(&sk->sk_receive_queue); 2447 skb_queue_head_init(&sk->sk_write_queue); 2448 skb_queue_head_init(&sk->sk_error_queue); 2449 2450 rwlock_init(&sk->sk_callback_lock); 2451 lockdep_set_class_and_name(&sk->sk_receive_queue.lock, 2452 af_rlock_keys + sk->sk_family, 2453 af_family_rlock_key_strings[sk->sk_family]); 2454 lockdep_set_class_and_name(&sk->sk_write_queue.lock, 2455 af_wlock_keys + sk->sk_family, 2456 af_family_wlock_key_strings[sk->sk_family]); 2457 lockdep_set_class_and_name(&sk->sk_error_queue.lock, 2458 af_elock_keys + sk->sk_family, 2459 af_family_elock_key_strings[sk->sk_family]); 2460 if (sk->sk_kern_sock) 2461 lockdep_set_class_and_name(&sk->sk_callback_lock, 2462 af_kern_callback_keys + sk->sk_family, 2463 af_family_kern_clock_key_strings[sk->sk_family]); 2464 else 2465 lockdep_set_class_and_name(&sk->sk_callback_lock, 2466 af_callback_keys + sk->sk_family, 2467 af_family_clock_key_strings[sk->sk_family]); 2468 } 2469 2470 /** 2471 * sk_clone - clone a socket 2472 * @sk: the socket to clone 2473 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 2474 * @lock: if true, lock the cloned sk 2475 * 2476 * If @lock is true, the clone is locked by bh_lock_sock(), and 2477 * caller must unlock socket even in error path by bh_unlock_sock(). 2478 */ 2479 struct sock *sk_clone(const struct sock *sk, const gfp_t priority, 2480 bool lock) 2481 { 2482 struct proto *prot = READ_ONCE(sk->sk_prot); 2483 struct sk_filter *filter; 2484 bool is_charged = true; 2485 struct sock *newsk; 2486 2487 newsk = sk_prot_alloc(prot, priority, sk->sk_family); 2488 if (!newsk) 2489 goto out; 2490 2491 sock_copy(newsk, sk); 2492 2493 newsk->sk_prot_creator = prot; 2494 #ifdef CONFIG_BPF_SYSCALL 2495 RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL); 2496 #endif 2497 2498 /* SANITY */ 2499 if (likely(newsk->sk_net_refcnt)) { 2500 get_net_track(sock_net(newsk), &newsk->ns_tracker, priority); 2501 sock_inuse_add(sock_net(newsk), 1); 2502 } else { 2503 /* Kernel sockets are not elevating the struct net refcount. 2504 * Instead, use a tracker to more easily detect if a layer 2505 * is not properly dismantling its kernel sockets at netns 2506 * destroy time. 2507 */ 2508 net_passive_inc(sock_net(newsk)); 2509 __netns_tracker_alloc(sock_net(newsk), &newsk->ns_tracker, 2510 false, priority); 2511 } 2512 2513 sk_node_init(&newsk->sk_node); 2514 sock_lock_init(newsk); 2515 2516 if (lock) 2517 bh_lock_sock(newsk); 2518 2519 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL; 2520 newsk->sk_backlog.len = 0; 2521 2522 atomic_set(&newsk->sk_rmem_alloc, 0); 2523 2524 refcount_set(&newsk->sk_wmem_alloc, SK_WMEM_ALLOC_BIAS); 2525 2526 atomic_set(&newsk->sk_omem_alloc, 0); 2527 sk_init_common(newsk); 2528 2529 newsk->sk_dst_cache = NULL; 2530 newsk->sk_dst_pending_confirm = 0; 2531 newsk->sk_wmem_queued = 0; 2532 newsk->sk_forward_alloc = 0; 2533 newsk->sk_reserved_mem = 0; 2534 DEBUG_NET_WARN_ON_ONCE(newsk->sk_drop_counters); 2535 sk_drops_reset(newsk); 2536 newsk->sk_send_head = NULL; 2537 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK; 2538 atomic_set(&newsk->sk_zckey, 0); 2539 2540 sock_reset_flag(newsk, SOCK_DONE); 2541 2542 #ifdef CONFIG_MEMCG 2543 /* sk->sk_memcg will be populated at accept() time */ 2544 newsk->sk_memcg = NULL; 2545 #endif 2546 2547 cgroup_sk_clone(&newsk->sk_cgrp_data); 2548 2549 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL); 2550 2551 if (sock_needs_netstamp(sk) && newsk->sk_flags & SK_FLAGS_TIMESTAMP) 2552 net_enable_timestamp(); 2553 2554 rcu_read_lock(); 2555 filter = rcu_dereference(sk->sk_filter); 2556 if (filter != NULL) 2557 /* though it's an empty new sock, the charging may fail 2558 * if sysctl_optmem_max was changed between creation of 2559 * original socket and cloning 2560 */ 2561 is_charged = sk_filter_charge(newsk, filter); 2562 RCU_INIT_POINTER(newsk->sk_filter, filter); 2563 rcu_read_unlock(); 2564 2565 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) { 2566 /* We need to make sure that we don't uncharge the new 2567 * socket if we couldn't charge it in the first place 2568 * as otherwise we uncharge the parent's filter. 2569 */ 2570 if (!is_charged) 2571 RCU_INIT_POINTER(newsk->sk_filter, NULL); 2572 2573 goto free; 2574 } 2575 2576 if (bpf_sk_storage_clone(sk, newsk)) 2577 goto free; 2578 2579 /* Clear sk_user_data if parent had the pointer tagged 2580 * as not suitable for copying when cloning. 2581 */ 2582 if (sk_user_data_is_nocopy(newsk)) 2583 newsk->sk_user_data = NULL; 2584 2585 newsk->sk_err = 0; 2586 newsk->sk_err_soft = 0; 2587 newsk->sk_priority = 0; 2588 newsk->sk_incoming_cpu = raw_smp_processor_id(); 2589 2590 /* Before updating sk_refcnt, we must commit prior changes to memory 2591 * (Documentation/RCU/rculist_nulls.rst for details) 2592 */ 2593 smp_wmb(); 2594 refcount_set(&newsk->sk_refcnt, 2); 2595 2596 sk_set_socket(newsk, NULL); 2597 sk_tx_queue_clear(newsk); 2598 sk_rx_queue_clear(newsk); 2599 RCU_INIT_POINTER(newsk->sk_wq, NULL); 2600 2601 if (newsk->sk_prot->sockets_allocated) 2602 sk_sockets_allocated_inc(newsk); 2603 out: 2604 return newsk; 2605 free: 2606 /* It is still raw copy of parent, so invalidate 2607 * destructor and make plain sk_free() 2608 */ 2609 newsk->sk_destruct = NULL; 2610 if (lock) 2611 bh_unlock_sock(newsk); 2612 sk_free(newsk); 2613 newsk = NULL; 2614 goto out; 2615 } 2616 EXPORT_SYMBOL_GPL(sk_clone); 2617 2618 static u32 sk_dst_gso_max_size(struct sock *sk, const struct net_device *dev) 2619 { 2620 bool is_ipv6 = false; 2621 u32 max_size; 2622 2623 #if IS_ENABLED(CONFIG_IPV6) 2624 is_ipv6 = (sk->sk_family == AF_INET6 && 2625 !ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)); 2626 #endif 2627 /* pairs with the WRITE_ONCE() in netif_set_gso(_ipv4)_max_size() */ 2628 max_size = is_ipv6 ? READ_ONCE(dev->gso_max_size) : 2629 READ_ONCE(dev->gso_ipv4_max_size); 2630 if (max_size > GSO_LEGACY_MAX_SIZE && !sk_is_tcp(sk)) 2631 max_size = GSO_LEGACY_MAX_SIZE; 2632 2633 return max_size - (MAX_TCP_HEADER + 1); 2634 } 2635 2636 void sk_setup_caps(struct sock *sk, struct dst_entry *dst) 2637 { 2638 const struct net_device *dev; 2639 u32 max_segs = 1; 2640 2641 rcu_read_lock(); 2642 dev = dst_dev_rcu(dst); 2643 sk->sk_route_caps = dev->features; 2644 if (sk_is_tcp(sk)) { 2645 struct inet_connection_sock *icsk = inet_csk(sk); 2646 2647 sk->sk_route_caps |= NETIF_F_GSO; 2648 icsk->icsk_ack.dst_quick_ack = dst_metric(dst, RTAX_QUICKACK); 2649 } 2650 if (sk->sk_route_caps & NETIF_F_GSO) 2651 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE; 2652 if (unlikely(sk->sk_gso_disabled)) 2653 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 2654 if (sk_can_gso(sk)) { 2655 if (dst->header_len && !xfrm_dst_offload_ok(dst)) { 2656 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 2657 } else { 2658 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM; 2659 sk->sk_gso_max_size = sk_dst_gso_max_size(sk, dev); 2660 /* pairs with the WRITE_ONCE() in netif_set_gso_max_segs() */ 2661 max_segs = max_t(u32, READ_ONCE(dev->gso_max_segs), 1); 2662 } 2663 } 2664 sk->sk_gso_max_segs = max_segs; 2665 sk_dst_set(sk, dst); 2666 rcu_read_unlock(); 2667 } 2668 EXPORT_SYMBOL_GPL(sk_setup_caps); 2669 2670 /* 2671 * Simple resource managers for sockets. 2672 */ 2673 2674 2675 /* 2676 * Write buffer destructor automatically called from kfree_skb. 2677 */ 2678 void sock_wfree(struct sk_buff *skb) 2679 { 2680 unsigned int len = skb->truesize; 2681 struct sock *sk = skb->sk; 2682 bool free; 2683 int old; 2684 2685 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) { 2686 void (*sk_write_space)(struct sock *sk); 2687 2688 sk_write_space = READ_ONCE(sk->sk_write_space); 2689 2690 if (sock_flag(sk, SOCK_RCU_FREE) && 2691 sk_write_space == sock_def_write_space) { 2692 rcu_read_lock(); 2693 free = __refcount_sub_and_test(len, &sk->sk_wmem_alloc, 2694 &old); 2695 sock_def_write_space_wfree(sk, old - len); 2696 rcu_read_unlock(); 2697 if (unlikely(free)) 2698 __sk_free(sk); 2699 return; 2700 } 2701 2702 /* 2703 * Keep a reference on sk_wmem_alloc, this will be released 2704 * after sk_write_space() call 2705 */ 2706 WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc)); 2707 sk_write_space(sk); 2708 len = 1; 2709 } 2710 /* 2711 * if sk_wmem_alloc reaches 0, we must finish what sk_free() 2712 * could not do because of in-flight packets 2713 */ 2714 if (refcount_sub_and_test(len, &sk->sk_wmem_alloc)) 2715 __sk_free(sk); 2716 } 2717 EXPORT_SYMBOL(sock_wfree); 2718 2719 /* This variant of sock_wfree() is used by TCP, 2720 * since it sets SOCK_USE_WRITE_QUEUE. 2721 */ 2722 #ifdef CONFIG_INET 2723 void __sock_wfree(struct sk_buff *skb) 2724 { 2725 struct sock *sk = skb->sk; 2726 2727 if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)) 2728 __sk_free(sk); 2729 } 2730 EXPORT_SYMBOL_GPL(__sock_wfree); 2731 #endif 2732 2733 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk) 2734 { 2735 int old_wmem; 2736 2737 skb_orphan(skb); 2738 #ifdef CONFIG_INET 2739 if (unlikely(!sk_fullsock(sk))) 2740 return skb_set_owner_edemux(skb, sk); 2741 #endif 2742 skb->sk = sk; 2743 skb->destructor = sock_wfree; 2744 skb_set_hash_from_sk(skb, sk); 2745 /* 2746 * We used to take a refcount on sk, but following operation 2747 * is enough to guarantee sk_free() won't free this sock until 2748 * all in-flight packets are completed 2749 */ 2750 __refcount_add(skb->truesize, &sk->sk_wmem_alloc, &old_wmem); 2751 2752 /* (old_wmem == SK_WMEM_ALLOC_BIAS) if no other TX packet for this socket 2753 * is in a host queue (qdisc, NIC queue). 2754 * Set skb->ooo_okay so that netdev_pick_tx() can choose a TX queue 2755 * based on XPS for better performance. 2756 * Otherwise clear ooo_okay to not risk Out Of Order delivery. 2757 */ 2758 skb->ooo_okay = (old_wmem == SK_WMEM_ALLOC_BIAS); 2759 } 2760 EXPORT_SYMBOL(skb_set_owner_w); 2761 2762 static bool can_skb_orphan_partial(const struct sk_buff *skb) 2763 { 2764 /* Drivers depend on in-order delivery for crypto offload, 2765 * partial orphan breaks out-of-order-OK logic. 2766 */ 2767 if (skb_is_decrypted(skb)) 2768 return false; 2769 2770 return (skb->destructor == sock_wfree || 2771 (IS_ENABLED(CONFIG_INET) && skb->destructor == tcp_wfree)); 2772 } 2773 2774 /* This helper is used by netem, as it can hold packets in its 2775 * delay queue. We want to allow the owner socket to send more 2776 * packets, as if they were already TX completed by a typical driver. 2777 * But we also want to keep skb->sk set because some packet schedulers 2778 * rely on it (sch_fq for example). 2779 */ 2780 void skb_orphan_partial(struct sk_buff *skb) 2781 { 2782 if (skb_is_tcp_pure_ack(skb)) 2783 return; 2784 2785 if (can_skb_orphan_partial(skb) && skb_set_owner_sk_safe(skb, skb->sk)) 2786 return; 2787 2788 skb_orphan(skb); 2789 } 2790 EXPORT_SYMBOL(skb_orphan_partial); 2791 2792 /* 2793 * Read buffer destructor automatically called from kfree_skb. 2794 */ 2795 void sock_rfree(struct sk_buff *skb) 2796 { 2797 struct sock *sk = skb->sk; 2798 unsigned int len = skb->truesize; 2799 2800 atomic_sub(len, &sk->sk_rmem_alloc); 2801 sk_mem_uncharge(sk, len); 2802 } 2803 EXPORT_SYMBOL(sock_rfree); 2804 2805 /* 2806 * Buffer destructor for skbs that are not used directly in read or write 2807 * path, e.g. for error handler skbs. Automatically called from kfree_skb. 2808 */ 2809 void sock_efree(struct sk_buff *skb) 2810 { 2811 sock_put(skb->sk); 2812 } 2813 EXPORT_SYMBOL(sock_efree); 2814 2815 /* Buffer destructor for prefetch/receive path where reference count may 2816 * not be held, e.g. for listen sockets. 2817 */ 2818 #ifdef CONFIG_INET 2819 void sock_pfree(struct sk_buff *skb) 2820 { 2821 struct sock *sk = skb->sk; 2822 2823 if (!sk_is_refcounted(sk)) 2824 return; 2825 2826 if (sk->sk_state == TCP_NEW_SYN_RECV && inet_reqsk(sk)->syncookie) { 2827 inet_reqsk(sk)->rsk_listener = NULL; 2828 reqsk_free(inet_reqsk(sk)); 2829 return; 2830 } 2831 2832 sock_gen_put(sk); 2833 } 2834 EXPORT_SYMBOL(sock_pfree); 2835 #endif /* CONFIG_INET */ 2836 2837 /* 2838 * Allocate a skb from the socket's send buffer. 2839 */ 2840 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force, 2841 gfp_t priority) 2842 { 2843 if (force || 2844 refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) { 2845 struct sk_buff *skb = alloc_skb(size, priority); 2846 2847 if (skb) { 2848 skb_set_owner_w(skb, sk); 2849 return skb; 2850 } 2851 } 2852 return NULL; 2853 } 2854 EXPORT_SYMBOL(sock_wmalloc); 2855 2856 static void sock_ofree(struct sk_buff *skb) 2857 { 2858 struct sock *sk = skb->sk; 2859 2860 atomic_sub(skb->truesize, &sk->sk_omem_alloc); 2861 } 2862 2863 struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size, 2864 gfp_t priority) 2865 { 2866 struct sk_buff *skb; 2867 2868 /* small safe race: SKB_TRUESIZE may differ from final skb->truesize */ 2869 if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) > 2870 READ_ONCE(sock_net(sk)->core.sysctl_optmem_max)) 2871 return NULL; 2872 2873 skb = alloc_skb(size, priority); 2874 if (!skb) 2875 return NULL; 2876 2877 atomic_add(skb->truesize, &sk->sk_omem_alloc); 2878 skb->sk = sk; 2879 skb->destructor = sock_ofree; 2880 return skb; 2881 } 2882 2883 /* 2884 * Allocate a memory block from the socket's option memory buffer. 2885 */ 2886 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority) 2887 { 2888 int optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max); 2889 2890 if ((unsigned int)size <= optmem_max && 2891 atomic_read(&sk->sk_omem_alloc) + size < optmem_max) { 2892 void *mem; 2893 /* First do the add, to avoid the race if kmalloc 2894 * might sleep. 2895 */ 2896 atomic_add(size, &sk->sk_omem_alloc); 2897 mem = kmalloc(size, priority); 2898 if (mem) 2899 return mem; 2900 atomic_sub(size, &sk->sk_omem_alloc); 2901 } 2902 return NULL; 2903 } 2904 EXPORT_SYMBOL(sock_kmalloc); 2905 2906 /* 2907 * Duplicate the input "src" memory block using the socket's 2908 * option memory buffer. 2909 */ 2910 void *sock_kmemdup(struct sock *sk, const void *src, 2911 int size, gfp_t priority) 2912 { 2913 void *mem; 2914 2915 mem = sock_kmalloc(sk, size, priority); 2916 if (mem) 2917 memcpy(mem, src, size); 2918 return mem; 2919 } 2920 EXPORT_SYMBOL(sock_kmemdup); 2921 2922 /* Free an option memory block. Note, we actually want the inline 2923 * here as this allows gcc to detect the nullify and fold away the 2924 * condition entirely. 2925 */ 2926 static inline void __sock_kfree_s(struct sock *sk, void *mem, int size, 2927 const bool nullify) 2928 { 2929 if (WARN_ON_ONCE(!mem)) 2930 return; 2931 if (nullify) 2932 kfree_sensitive(mem); 2933 else 2934 kfree(mem); 2935 atomic_sub(size, &sk->sk_omem_alloc); 2936 } 2937 2938 void sock_kfree_s(struct sock *sk, void *mem, int size) 2939 { 2940 __sock_kfree_s(sk, mem, size, false); 2941 } 2942 EXPORT_SYMBOL(sock_kfree_s); 2943 2944 void sock_kzfree_s(struct sock *sk, void *mem, int size) 2945 { 2946 __sock_kfree_s(sk, mem, size, true); 2947 } 2948 EXPORT_SYMBOL(sock_kzfree_s); 2949 2950 /* It is almost wait_for_tcp_memory minus release_sock/lock_sock. 2951 I think, these locks should be removed for datagram sockets. 2952 */ 2953 static long sock_wait_for_wmem(struct sock *sk, long timeo) 2954 { 2955 DEFINE_WAIT(wait); 2956 2957 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 2958 for (;;) { 2959 if (!timeo) 2960 break; 2961 if (signal_pending(current)) 2962 break; 2963 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2964 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); 2965 if (refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) 2966 break; 2967 if (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN) 2968 break; 2969 if (READ_ONCE(sk->sk_err)) 2970 break; 2971 timeo = schedule_timeout(timeo); 2972 } 2973 finish_wait(sk_sleep(sk), &wait); 2974 return timeo; 2975 } 2976 2977 2978 /* 2979 * Generic send/receive buffer handlers 2980 */ 2981 2982 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len, 2983 unsigned long data_len, int noblock, 2984 int *errcode, int max_page_order) 2985 { 2986 struct sk_buff *skb; 2987 long timeo; 2988 int err; 2989 2990 timeo = sock_sndtimeo(sk, noblock); 2991 for (;;) { 2992 err = sock_error(sk); 2993 if (err != 0) 2994 goto failure; 2995 2996 err = -EPIPE; 2997 if (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN) 2998 goto failure; 2999 3000 if (sk_wmem_alloc_get(sk) < READ_ONCE(sk->sk_sndbuf)) 3001 break; 3002 3003 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 3004 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 3005 err = -EAGAIN; 3006 if (!timeo) 3007 goto failure; 3008 if (signal_pending(current)) 3009 goto interrupted; 3010 timeo = sock_wait_for_wmem(sk, timeo); 3011 } 3012 skb = alloc_skb_with_frags(header_len, data_len, max_page_order, 3013 errcode, sk->sk_allocation); 3014 if (skb) 3015 skb_set_owner_w(skb, sk); 3016 return skb; 3017 3018 interrupted: 3019 err = sock_intr_errno(timeo); 3020 failure: 3021 *errcode = err; 3022 return NULL; 3023 } 3024 EXPORT_SYMBOL(sock_alloc_send_pskb); 3025 3026 int __sock_cmsg_send(struct sock *sk, struct cmsghdr *cmsg, 3027 struct sockcm_cookie *sockc) 3028 { 3029 u32 tsflags; 3030 3031 BUILD_BUG_ON(SOF_TIMESTAMPING_LAST == (1 << 31)); 3032 3033 switch (cmsg->cmsg_type) { 3034 case SO_MARK: 3035 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && 3036 !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 3037 return -EPERM; 3038 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3039 return -EINVAL; 3040 sockc->mark = *(u32 *)CMSG_DATA(cmsg); 3041 break; 3042 case SO_TIMESTAMPING_OLD: 3043 case SO_TIMESTAMPING_NEW: 3044 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3045 return -EINVAL; 3046 3047 tsflags = *(u32 *)CMSG_DATA(cmsg); 3048 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK) 3049 return -EINVAL; 3050 3051 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK; 3052 sockc->tsflags |= tsflags; 3053 break; 3054 case SCM_TXTIME: 3055 { 3056 ktime_t tmin; 3057 u64 txtime; 3058 3059 if (!sock_flag(sk, SOCK_TXTIME)) 3060 return -EINVAL; 3061 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64))) 3062 return -EINVAL; 3063 3064 txtime = get_unaligned((u64 *)CMSG_DATA(cmsg)); 3065 3066 /* Allow sending without a delivery time: zero special case */ 3067 if (!txtime) { 3068 sockc->transmit_time = 0; 3069 break; 3070 } 3071 3072 switch (sk->sk_clockid) { 3073 case CLOCK_MONOTONIC: 3074 tmin = 1; 3075 break; 3076 case CLOCK_REALTIME: 3077 tmin = max(ktime_mono_to_real(0), 1); 3078 break; 3079 case CLOCK_TAI: 3080 tmin = max(ktime_mono_to_any(0, TK_OFFS_TAI), 1); 3081 break; 3082 default: 3083 tmin = 1; 3084 WARN_ON_ONCE(1); 3085 break; 3086 } 3087 3088 sockc->transmit_time = max_t(ktime_t, txtime, tmin); 3089 break; 3090 } 3091 case SCM_TS_OPT_ID: 3092 if (sk_is_tcp(sk)) 3093 return -EINVAL; 3094 tsflags = READ_ONCE(sk->sk_tsflags); 3095 if (!(tsflags & SOF_TIMESTAMPING_OPT_ID)) 3096 return -EINVAL; 3097 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3098 return -EINVAL; 3099 sockc->ts_opt_id = *(u32 *)CMSG_DATA(cmsg); 3100 sockc->tsflags |= SOCKCM_FLAG_TS_OPT_ID; 3101 break; 3102 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */ 3103 case SCM_RIGHTS: 3104 case SCM_CREDENTIALS: 3105 break; 3106 case SO_PRIORITY: 3107 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3108 return -EINVAL; 3109 if (!sk_set_prio_allowed(sk, *(u32 *)CMSG_DATA(cmsg))) 3110 return -EPERM; 3111 sockc->priority = *(u32 *)CMSG_DATA(cmsg); 3112 break; 3113 case SCM_DEVMEM_DMABUF: 3114 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3115 return -EINVAL; 3116 sockc->dmabuf_id = *(u32 *)CMSG_DATA(cmsg); 3117 break; 3118 default: 3119 return -EINVAL; 3120 } 3121 return 0; 3122 } 3123 EXPORT_SYMBOL(__sock_cmsg_send); 3124 3125 int sock_cmsg_send(struct sock *sk, struct msghdr *msg, 3126 struct sockcm_cookie *sockc) 3127 { 3128 struct cmsghdr *cmsg; 3129 int ret; 3130 3131 for_each_cmsghdr(cmsg, msg) { 3132 if (!CMSG_OK(msg, cmsg)) 3133 return -EINVAL; 3134 if (cmsg->cmsg_level != SOL_SOCKET) 3135 continue; 3136 ret = __sock_cmsg_send(sk, cmsg, sockc); 3137 if (ret) 3138 return ret; 3139 } 3140 return 0; 3141 } 3142 EXPORT_SYMBOL(sock_cmsg_send); 3143 3144 static void sk_enter_memory_pressure(struct sock *sk) 3145 { 3146 if (!sk->sk_prot->enter_memory_pressure) 3147 return; 3148 3149 sk->sk_prot->enter_memory_pressure(sk); 3150 } 3151 3152 static void sk_leave_memory_pressure(struct sock *sk) 3153 { 3154 if (sk->sk_prot->leave_memory_pressure) { 3155 INDIRECT_CALL_INET_1(sk->sk_prot->leave_memory_pressure, 3156 tcp_leave_memory_pressure, sk); 3157 } else { 3158 unsigned long *memory_pressure = sk->sk_prot->memory_pressure; 3159 3160 if (memory_pressure && READ_ONCE(*memory_pressure)) 3161 WRITE_ONCE(*memory_pressure, 0); 3162 } 3163 } 3164 3165 DEFINE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key); 3166 3167 /** 3168 * skb_page_frag_refill - check that a page_frag contains enough room 3169 * @sz: minimum size of the fragment we want to get 3170 * @pfrag: pointer to page_frag 3171 * @gfp: priority for memory allocation 3172 * 3173 * Note: While this allocator tries to use high order pages, there is 3174 * no guarantee that allocations succeed. Therefore, @sz MUST be 3175 * less or equal than PAGE_SIZE. 3176 */ 3177 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp) 3178 { 3179 if (pfrag->page) { 3180 if (page_ref_count(pfrag->page) == 1) { 3181 pfrag->offset = 0; 3182 return true; 3183 } 3184 if (pfrag->offset + sz <= pfrag->size) 3185 return true; 3186 put_page(pfrag->page); 3187 } 3188 3189 pfrag->offset = 0; 3190 if (SKB_FRAG_PAGE_ORDER && 3191 !static_branch_unlikely(&net_high_order_alloc_disable_key)) { 3192 /* Avoid direct reclaim but allow kswapd to wake */ 3193 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) | 3194 __GFP_COMP | __GFP_NOWARN | 3195 __GFP_NORETRY, 3196 SKB_FRAG_PAGE_ORDER); 3197 if (likely(pfrag->page)) { 3198 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER; 3199 return true; 3200 } 3201 } 3202 pfrag->page = alloc_page(gfp); 3203 if (likely(pfrag->page)) { 3204 pfrag->size = PAGE_SIZE; 3205 return true; 3206 } 3207 return false; 3208 } 3209 EXPORT_SYMBOL(skb_page_frag_refill); 3210 3211 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 3212 { 3213 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation))) 3214 return true; 3215 3216 if (!sk->sk_bypass_prot_mem) 3217 sk_enter_memory_pressure(sk); 3218 3219 sk_stream_moderate_sndbuf(sk); 3220 3221 return false; 3222 } 3223 EXPORT_SYMBOL(sk_page_frag_refill); 3224 3225 static void __lock_sock(struct sock *sk) 3226 __releases(&sk->sk_lock.slock) 3227 __acquires(&sk->sk_lock.slock) 3228 { 3229 DEFINE_WAIT(wait); 3230 3231 for (;;) { 3232 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait, 3233 TASK_UNINTERRUPTIBLE); 3234 spin_unlock_bh(&sk->sk_lock.slock); 3235 schedule(); 3236 spin_lock_bh(&sk->sk_lock.slock); 3237 if (!sock_owned_by_user(sk)) 3238 break; 3239 } 3240 finish_wait(&sk->sk_lock.wq, &wait); 3241 } 3242 3243 void __release_sock(struct sock *sk) 3244 __releases(&sk->sk_lock.slock) 3245 __acquires(&sk->sk_lock.slock) 3246 { 3247 struct sk_buff *skb, *next; 3248 int nb = 0; 3249 3250 while ((skb = sk->sk_backlog.head) != NULL) { 3251 sk->sk_backlog.head = sk->sk_backlog.tail = NULL; 3252 3253 spin_unlock_bh(&sk->sk_lock.slock); 3254 3255 while (1) { 3256 next = skb->next; 3257 prefetch(next); 3258 DEBUG_NET_WARN_ON_ONCE(skb_dst_is_noref(skb)); 3259 skb_mark_not_on_list(skb); 3260 sk_backlog_rcv(sk, skb); 3261 3262 skb = next; 3263 if (!skb) 3264 break; 3265 3266 if (!(++nb & 15)) 3267 cond_resched(); 3268 } 3269 3270 spin_lock_bh(&sk->sk_lock.slock); 3271 } 3272 3273 /* 3274 * Doing the zeroing here guarantee we can not loop forever 3275 * while a wild producer attempts to flood us. 3276 */ 3277 sk->sk_backlog.len = 0; 3278 } 3279 3280 void __sk_flush_backlog(struct sock *sk) 3281 { 3282 spin_lock_bh(&sk->sk_lock.slock); 3283 __release_sock(sk); 3284 3285 if (sk->sk_prot->release_cb) 3286 INDIRECT_CALL_INET_1(sk->sk_prot->release_cb, 3287 tcp_release_cb, sk); 3288 3289 spin_unlock_bh(&sk->sk_lock.slock); 3290 } 3291 EXPORT_SYMBOL_GPL(__sk_flush_backlog); 3292 3293 /** 3294 * sk_wait_data - wait for data to arrive at sk_receive_queue 3295 * @sk: sock to wait on 3296 * @timeo: for how long 3297 * @skb: last skb seen on sk_receive_queue 3298 * 3299 * Now socket state including sk->sk_err is changed only under lock, 3300 * hence we may omit checks after joining wait queue. 3301 * We check receive queue before schedule() only as optimization; 3302 * it is very likely that release_sock() added new data. 3303 */ 3304 int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb) 3305 { 3306 DEFINE_WAIT_FUNC(wait, woken_wake_function); 3307 int rc; 3308 3309 add_wait_queue(sk_sleep(sk), &wait); 3310 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 3311 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait); 3312 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 3313 remove_wait_queue(sk_sleep(sk), &wait); 3314 return rc; 3315 } 3316 EXPORT_SYMBOL(sk_wait_data); 3317 3318 /** 3319 * __sk_mem_raise_allocated - increase memory_allocated 3320 * @sk: socket 3321 * @size: memory size to allocate 3322 * @amt: pages to allocate 3323 * @kind: allocation type 3324 * 3325 * Similar to __sk_mem_schedule(), but does not update sk_forward_alloc. 3326 * 3327 * Unlike the globally shared limits among the sockets under same protocol, 3328 * consuming the budget of a memcg won't have direct effect on other ones. 3329 * So be optimistic about memcg's tolerance, and leave the callers to decide 3330 * whether or not to raise allocated through sk_under_memory_pressure() or 3331 * its variants. 3332 */ 3333 int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind) 3334 { 3335 bool memcg_enabled = false, charged = false; 3336 struct proto *prot = sk->sk_prot; 3337 long allocated = 0; 3338 3339 if (!sk->sk_bypass_prot_mem) { 3340 sk_memory_allocated_add(sk, amt); 3341 allocated = sk_memory_allocated(sk); 3342 } 3343 3344 if (mem_cgroup_sk_enabled(sk)) { 3345 memcg_enabled = true; 3346 charged = mem_cgroup_sk_charge(sk, amt, gfp_memcg_charge()); 3347 if (!charged) 3348 goto suppress_allocation; 3349 } 3350 3351 if (!allocated) 3352 return 1; 3353 3354 /* Under limit. */ 3355 if (allocated <= sk_prot_mem_limits(sk, 0)) { 3356 sk_leave_memory_pressure(sk); 3357 return 1; 3358 } 3359 3360 /* Under pressure. */ 3361 if (allocated > sk_prot_mem_limits(sk, 1)) 3362 sk_enter_memory_pressure(sk); 3363 3364 /* Over hard limit. */ 3365 if (allocated > sk_prot_mem_limits(sk, 2)) 3366 goto suppress_allocation; 3367 3368 /* Guarantee minimum buffer size under pressure (either global 3369 * or memcg) to make sure features described in RFC 7323 (TCP 3370 * Extensions for High Performance) work properly. 3371 * 3372 * This rule does NOT stand when exceeds global or memcg's hard 3373 * limit, or else a DoS attack can be taken place by spawning 3374 * lots of sockets whose usage are under minimum buffer size. 3375 */ 3376 if (kind == SK_MEM_RECV) { 3377 if (atomic_read(&sk->sk_rmem_alloc) < sk_get_rmem0(sk, prot)) 3378 return 1; 3379 3380 } else { /* SK_MEM_SEND */ 3381 int wmem0 = sk_get_wmem0(sk, prot); 3382 3383 if (sk->sk_type == SOCK_STREAM) { 3384 if (sk->sk_wmem_queued < wmem0) 3385 return 1; 3386 } else if (refcount_read(&sk->sk_wmem_alloc) < wmem0) { 3387 return 1; 3388 } 3389 } 3390 3391 if (sk_has_memory_pressure(sk)) { 3392 u64 alloc; 3393 3394 /* The following 'average' heuristic is within the 3395 * scope of global accounting, so it only makes 3396 * sense for global memory pressure. 3397 */ 3398 if (!sk_under_global_memory_pressure(sk)) 3399 return 1; 3400 3401 /* Try to be fair among all the sockets under global 3402 * pressure by allowing the ones that below average 3403 * usage to raise. 3404 */ 3405 alloc = sk_sockets_allocated_read_positive(sk); 3406 if (sk_prot_mem_limits(sk, 2) > alloc * 3407 sk_mem_pages(sk->sk_wmem_queued + 3408 atomic_read(&sk->sk_rmem_alloc) + 3409 sk->sk_forward_alloc)) 3410 return 1; 3411 } 3412 3413 suppress_allocation: 3414 3415 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) { 3416 sk_stream_moderate_sndbuf(sk); 3417 3418 /* Fail only if socket is _under_ its sndbuf. 3419 * In this case we cannot block, so that we have to fail. 3420 */ 3421 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf) { 3422 /* Force charge with __GFP_NOFAIL */ 3423 if (memcg_enabled && !charged) 3424 mem_cgroup_sk_charge(sk, amt, 3425 gfp_memcg_charge() | __GFP_NOFAIL); 3426 return 1; 3427 } 3428 } 3429 3430 trace_sock_exceed_buf_limit(sk, prot, allocated, kind); 3431 3432 if (allocated) 3433 sk_memory_allocated_sub(sk, amt); 3434 3435 if (charged) 3436 mem_cgroup_sk_uncharge(sk, amt); 3437 3438 return 0; 3439 } 3440 3441 /** 3442 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated 3443 * @sk: socket 3444 * @size: memory size to allocate 3445 * @kind: allocation type 3446 * 3447 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means 3448 * rmem allocation. This function assumes that protocols which have 3449 * memory_pressure use sk_wmem_queued as write buffer accounting. 3450 */ 3451 int __sk_mem_schedule(struct sock *sk, int size, int kind) 3452 { 3453 int ret, amt = sk_mem_pages(size); 3454 3455 sk_forward_alloc_add(sk, amt << PAGE_SHIFT); 3456 ret = __sk_mem_raise_allocated(sk, size, amt, kind); 3457 if (!ret) 3458 sk_forward_alloc_add(sk, -(amt << PAGE_SHIFT)); 3459 return ret; 3460 } 3461 EXPORT_SYMBOL(__sk_mem_schedule); 3462 3463 /** 3464 * __sk_mem_reduce_allocated - reclaim memory_allocated 3465 * @sk: socket 3466 * @amount: number of quanta 3467 * 3468 * Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc 3469 */ 3470 void __sk_mem_reduce_allocated(struct sock *sk, int amount) 3471 { 3472 if (mem_cgroup_sk_enabled(sk)) 3473 mem_cgroup_sk_uncharge(sk, amount); 3474 3475 if (sk->sk_bypass_prot_mem) 3476 return; 3477 3478 sk_memory_allocated_sub(sk, amount); 3479 3480 if (sk_under_global_memory_pressure(sk) && 3481 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0))) 3482 sk_leave_memory_pressure(sk); 3483 } 3484 3485 /** 3486 * __sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated 3487 * @sk: socket 3488 * @amount: number of bytes (rounded down to a PAGE_SIZE multiple) 3489 */ 3490 void __sk_mem_reclaim(struct sock *sk, int amount) 3491 { 3492 amount >>= PAGE_SHIFT; 3493 sk_forward_alloc_add(sk, -(amount << PAGE_SHIFT)); 3494 __sk_mem_reduce_allocated(sk, amount); 3495 } 3496 EXPORT_SYMBOL(__sk_mem_reclaim); 3497 3498 void __sk_charge(struct sock *sk, gfp_t gfp) 3499 { 3500 int amt; 3501 3502 gfp |= __GFP_NOFAIL; 3503 if (mem_cgroup_from_sk(sk)) { 3504 /* The socket has not been accepted yet, no need 3505 * to look at newsk->sk_wmem_queued. 3506 */ 3507 amt = sk_mem_pages(sk->sk_forward_alloc + 3508 atomic_read(&sk->sk_rmem_alloc)); 3509 if (amt) 3510 mem_cgroup_sk_charge(sk, amt, gfp); 3511 } 3512 3513 kmem_cache_charge(sk, gfp); 3514 } 3515 3516 int sk_set_peek_off(struct sock *sk, int val) 3517 { 3518 WRITE_ONCE(sk->sk_peek_off, val); 3519 return 0; 3520 } 3521 EXPORT_SYMBOL_GPL(sk_set_peek_off); 3522 3523 /* 3524 * Set of default routines for initialising struct proto_ops when 3525 * the protocol does not support a particular function. In certain 3526 * cases where it makes no sense for a protocol to have a "do nothing" 3527 * function, some default processing is provided. 3528 */ 3529 3530 int sock_no_bind(struct socket *sock, struct sockaddr_unsized *saddr, int len) 3531 { 3532 return -EOPNOTSUPP; 3533 } 3534 EXPORT_SYMBOL(sock_no_bind); 3535 3536 int sock_no_connect(struct socket *sock, struct sockaddr_unsized *saddr, 3537 int len, int flags) 3538 { 3539 return -EOPNOTSUPP; 3540 } 3541 EXPORT_SYMBOL(sock_no_connect); 3542 3543 int sock_no_socketpair(struct socket *sock1, struct socket *sock2) 3544 { 3545 return -EOPNOTSUPP; 3546 } 3547 EXPORT_SYMBOL(sock_no_socketpair); 3548 3549 int sock_no_accept(struct socket *sock, struct socket *newsock, 3550 struct proto_accept_arg *arg) 3551 { 3552 return -EOPNOTSUPP; 3553 } 3554 EXPORT_SYMBOL(sock_no_accept); 3555 3556 int sock_no_getname(struct socket *sock, struct sockaddr *saddr, 3557 int peer) 3558 { 3559 return -EOPNOTSUPP; 3560 } 3561 EXPORT_SYMBOL(sock_no_getname); 3562 3563 int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) 3564 { 3565 return -EOPNOTSUPP; 3566 } 3567 EXPORT_SYMBOL(sock_no_ioctl); 3568 3569 int sock_no_listen(struct socket *sock, int backlog) 3570 { 3571 return -EOPNOTSUPP; 3572 } 3573 EXPORT_SYMBOL(sock_no_listen); 3574 3575 int sock_no_shutdown(struct socket *sock, int how) 3576 { 3577 return -EOPNOTSUPP; 3578 } 3579 EXPORT_SYMBOL(sock_no_shutdown); 3580 3581 int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len) 3582 { 3583 return -EOPNOTSUPP; 3584 } 3585 EXPORT_SYMBOL(sock_no_sendmsg); 3586 3587 int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len) 3588 { 3589 return -EOPNOTSUPP; 3590 } 3591 EXPORT_SYMBOL(sock_no_sendmsg_locked); 3592 3593 int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len, 3594 int flags) 3595 { 3596 return -EOPNOTSUPP; 3597 } 3598 EXPORT_SYMBOL(sock_no_recvmsg); 3599 3600 int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma) 3601 { 3602 /* Mirror missing mmap method error code */ 3603 return -ENODEV; 3604 } 3605 EXPORT_SYMBOL(sock_no_mmap); 3606 3607 /* 3608 * When a file is received (via SCM_RIGHTS, etc), we must bump the 3609 * various sock-based usage counts. 3610 */ 3611 void __receive_sock(struct file *file) 3612 { 3613 struct socket *sock; 3614 3615 sock = sock_from_file(file); 3616 if (sock) { 3617 sock_update_netprioidx(&sock->sk->sk_cgrp_data); 3618 sock_update_classid(&sock->sk->sk_cgrp_data); 3619 } 3620 } 3621 3622 /* 3623 * Default Socket Callbacks 3624 */ 3625 3626 static void sock_def_wakeup(struct sock *sk) 3627 { 3628 struct socket_wq *wq; 3629 3630 rcu_read_lock(); 3631 wq = rcu_dereference(sk->sk_wq); 3632 if (skwq_has_sleeper(wq)) 3633 wake_up_interruptible_all(&wq->wait); 3634 rcu_read_unlock(); 3635 } 3636 3637 static void sock_def_error_report(struct sock *sk) 3638 { 3639 struct socket_wq *wq; 3640 3641 rcu_read_lock(); 3642 wq = rcu_dereference(sk->sk_wq); 3643 if (skwq_has_sleeper(wq)) 3644 wake_up_interruptible_poll(&wq->wait, EPOLLERR); 3645 sk_wake_async_rcu(sk, SOCK_WAKE_IO, POLL_ERR); 3646 rcu_read_unlock(); 3647 } 3648 3649 void sock_def_readable(struct sock *sk) 3650 { 3651 struct socket_wq *wq; 3652 3653 trace_sk_data_ready(sk); 3654 3655 rcu_read_lock(); 3656 wq = rcu_dereference(sk->sk_wq); 3657 if (skwq_has_sleeper(wq)) 3658 wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI | 3659 EPOLLRDNORM | EPOLLRDBAND); 3660 sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_IN); 3661 rcu_read_unlock(); 3662 } 3663 3664 static void sock_def_write_space(struct sock *sk) 3665 { 3666 struct socket_wq *wq; 3667 3668 rcu_read_lock(); 3669 3670 /* Do not wake up a writer until he can make "significant" 3671 * progress. --DaveM 3672 */ 3673 if (sock_writeable(sk)) { 3674 wq = rcu_dereference(sk->sk_wq); 3675 if (skwq_has_sleeper(wq)) 3676 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 3677 EPOLLWRNORM | EPOLLWRBAND); 3678 3679 /* Should agree with poll, otherwise some programs break */ 3680 sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT); 3681 } 3682 3683 rcu_read_unlock(); 3684 } 3685 3686 /* An optimised version of sock_def_write_space(), should only be called 3687 * for SOCK_RCU_FREE sockets under RCU read section and after putting 3688 * ->sk_wmem_alloc. 3689 */ 3690 static void sock_def_write_space_wfree(struct sock *sk, int wmem_alloc) 3691 { 3692 /* Do not wake up a writer until he can make "significant" 3693 * progress. --DaveM 3694 */ 3695 if (__sock_writeable(sk, wmem_alloc)) { 3696 struct socket_wq *wq = rcu_dereference(sk->sk_wq); 3697 3698 /* rely on refcount_sub from sock_wfree() */ 3699 smp_mb__after_atomic(); 3700 if (wq && waitqueue_active(&wq->wait)) 3701 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 3702 EPOLLWRNORM | EPOLLWRBAND); 3703 3704 /* Should agree with poll, otherwise some programs break */ 3705 sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT); 3706 } 3707 } 3708 3709 static void sock_def_destruct(struct sock *sk) 3710 { 3711 } 3712 3713 void sk_send_sigurg(struct sock *sk) 3714 { 3715 if (sk->sk_socket && sk->sk_socket->file) 3716 if (send_sigurg(sk->sk_socket->file)) 3717 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI); 3718 } 3719 EXPORT_SYMBOL(sk_send_sigurg); 3720 3721 void sk_reset_timer(struct sock *sk, struct timer_list* timer, 3722 unsigned long expires) 3723 { 3724 if (!mod_timer(timer, expires)) 3725 sock_hold(sk); 3726 } 3727 EXPORT_SYMBOL(sk_reset_timer); 3728 3729 void sk_stop_timer(struct sock *sk, struct timer_list* timer) 3730 { 3731 if (timer_delete(timer)) 3732 __sock_put(sk); 3733 } 3734 EXPORT_SYMBOL(sk_stop_timer); 3735 3736 void sk_stop_timer_sync(struct sock *sk, struct timer_list *timer) 3737 { 3738 if (timer_delete_sync(timer)) 3739 __sock_put(sk); 3740 } 3741 EXPORT_SYMBOL(sk_stop_timer_sync); 3742 3743 void sock_init_data_uid(struct socket *sock, struct sock *sk, kuid_t uid) 3744 { 3745 sk_init_common(sk); 3746 sk->sk_send_head = NULL; 3747 3748 timer_setup(&sk->sk_timer, NULL, 0); 3749 3750 sk->sk_allocation = GFP_KERNEL; 3751 sk->sk_rcvbuf = READ_ONCE(sysctl_rmem_default); 3752 sk->sk_sndbuf = READ_ONCE(sysctl_wmem_default); 3753 sk->sk_state = TCP_CLOSE; 3754 sk->sk_use_task_frag = true; 3755 sk_set_socket(sk, sock); 3756 3757 sock_set_flag(sk, SOCK_ZAPPED); 3758 3759 if (sock) { 3760 sk->sk_type = sock->type; 3761 RCU_INIT_POINTER(sk->sk_wq, &sock->wq); 3762 sock->sk = sk; 3763 } else { 3764 RCU_INIT_POINTER(sk->sk_wq, NULL); 3765 } 3766 sk->sk_uid = uid; 3767 3768 sk->sk_state_change = sock_def_wakeup; 3769 sk->sk_data_ready = sock_def_readable; 3770 sk->sk_write_space = sock_def_write_space; 3771 sk->sk_error_report = sock_def_error_report; 3772 sk->sk_destruct = sock_def_destruct; 3773 3774 sk->sk_frag.page = NULL; 3775 sk->sk_frag.offset = 0; 3776 sk->sk_peek_off = -1; 3777 3778 sk->sk_peer_pid = NULL; 3779 sk->sk_peer_cred = NULL; 3780 spin_lock_init(&sk->sk_peer_lock); 3781 3782 sk->sk_write_pending = 0; 3783 sk->sk_rcvlowat = 1; 3784 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT; 3785 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT; 3786 3787 sk->sk_stamp = SK_DEFAULT_STAMP; 3788 #if BITS_PER_LONG==32 3789 seqlock_init(&sk->sk_stamp_seq); 3790 #endif 3791 atomic_set(&sk->sk_zckey, 0); 3792 3793 #ifdef CONFIG_NET_RX_BUSY_POLL 3794 sk->sk_napi_id = 0; 3795 sk->sk_ll_usec = READ_ONCE(sysctl_net_busy_read); 3796 #endif 3797 3798 sk->sk_max_pacing_rate = ~0UL; 3799 sk->sk_pacing_rate = ~0UL; 3800 WRITE_ONCE(sk->sk_pacing_shift, 10); 3801 sk->sk_incoming_cpu = -1; 3802 3803 sk_rx_queue_clear(sk); 3804 /* 3805 * Before updating sk_refcnt, we must commit prior changes to memory 3806 * (Documentation/RCU/rculist_nulls.rst for details) 3807 */ 3808 smp_wmb(); 3809 refcount_set(&sk->sk_refcnt, 1); 3810 sk_drops_reset(sk); 3811 } 3812 EXPORT_SYMBOL(sock_init_data_uid); 3813 3814 void sock_init_data(struct socket *sock, struct sock *sk) 3815 { 3816 kuid_t uid = sock ? 3817 SOCK_INODE(sock)->i_uid : 3818 make_kuid(sock_net(sk)->user_ns, 0); 3819 3820 sock_init_data_uid(sock, sk, uid); 3821 } 3822 EXPORT_SYMBOL(sock_init_data); 3823 3824 void noinline lock_sock_nested(struct sock *sk, int subclass) 3825 { 3826 /* The sk_lock has mutex_lock() semantics here. */ 3827 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_); 3828 3829 might_sleep(); 3830 #ifdef CONFIG_64BIT 3831 if (sizeof(struct slock_owned) == sizeof(long)) { 3832 socket_lock_t tmp = { 3833 .slock = __SPIN_LOCK_UNLOCKED(tmp.slock), 3834 .owned = 1, 3835 }; 3836 socket_lock_t old = { 3837 .slock = __SPIN_LOCK_UNLOCKED(old.slock), 3838 .owned = 0, 3839 }; 3840 3841 if (likely(try_cmpxchg(&sk->sk_lock.combined, 3842 &old.combined, tmp.combined))) 3843 return; 3844 } 3845 #endif 3846 spin_lock_bh(&sk->sk_lock.slock); 3847 if (unlikely(sock_owned_by_user_nocheck(sk))) 3848 __lock_sock(sk); 3849 sk->sk_lock.owned = 1; 3850 spin_unlock_bh(&sk->sk_lock.slock); 3851 } 3852 EXPORT_SYMBOL(lock_sock_nested); 3853 3854 void release_sock(struct sock *sk) 3855 { 3856 spin_lock_bh(&sk->sk_lock.slock); 3857 3858 if (unlikely(sk->sk_backlog.tail)) 3859 __release_sock(sk); 3860 3861 if (sk->sk_prot->release_cb) { 3862 if (!tcp_release_cb_cond(sk)) 3863 sk->sk_prot->release_cb(sk); 3864 } 3865 sock_release_ownership(sk); 3866 if (unlikely(waitqueue_active(&sk->sk_lock.wq))) 3867 wake_up(&sk->sk_lock.wq); 3868 3869 spin_unlock_bh(&sk->sk_lock.slock); 3870 } 3871 EXPORT_SYMBOL(release_sock); 3872 3873 bool __lock_sock_fast(struct sock *sk) __acquires(&sk->sk_lock.slock) 3874 { 3875 might_sleep(); 3876 spin_lock_bh(&sk->sk_lock.slock); 3877 3878 if (likely(!sock_owned_by_user_nocheck(sk))) { 3879 /* 3880 * Fast path return with bottom halves disabled and 3881 * sock::sk_lock.slock held. 3882 * 3883 * The 'mutex' is not contended and holding 3884 * sock::sk_lock.slock prevents all other lockers to 3885 * proceed so the corresponding unlock_sock_fast() can 3886 * avoid the slow path of release_sock() completely and 3887 * just release slock. 3888 * 3889 * From a semantical POV this is equivalent to 'acquiring' 3890 * the 'mutex', hence the corresponding lockdep 3891 * mutex_release() has to happen in the fast path of 3892 * unlock_sock_fast(). 3893 */ 3894 return false; 3895 } 3896 3897 __lock_sock(sk); 3898 sk->sk_lock.owned = 1; 3899 __acquire(&sk->sk_lock.slock); 3900 spin_unlock_bh(&sk->sk_lock.slock); 3901 return true; 3902 } 3903 EXPORT_SYMBOL(__lock_sock_fast); 3904 3905 int sock_gettstamp(struct socket *sock, void __user *userstamp, 3906 bool timeval, bool time32) 3907 { 3908 struct sock *sk = sock->sk; 3909 struct timespec64 ts; 3910 3911 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 3912 ts = ktime_to_timespec64(sock_read_timestamp(sk)); 3913 if (ts.tv_sec == -1) 3914 return -ENOENT; 3915 if (ts.tv_sec == 0) { 3916 ktime_t kt = ktime_get_real(); 3917 sock_write_timestamp(sk, kt); 3918 ts = ktime_to_timespec64(kt); 3919 } 3920 3921 if (timeval) 3922 ts.tv_nsec /= 1000; 3923 3924 #ifdef CONFIG_COMPAT_32BIT_TIME 3925 if (time32) 3926 return put_old_timespec32(&ts, userstamp); 3927 #endif 3928 #ifdef CONFIG_SPARC64 3929 /* beware of padding in sparc64 timeval */ 3930 if (timeval && !in_compat_syscall()) { 3931 struct __kernel_old_timeval __user tv = { 3932 .tv_sec = ts.tv_sec, 3933 .tv_usec = ts.tv_nsec, 3934 }; 3935 if (copy_to_user(userstamp, &tv, sizeof(tv))) 3936 return -EFAULT; 3937 return 0; 3938 } 3939 #endif 3940 return put_timespec64(&ts, userstamp); 3941 } 3942 EXPORT_SYMBOL(sock_gettstamp); 3943 3944 void sock_enable_timestamp(struct sock *sk, enum sock_flags flag) 3945 { 3946 if (!sock_flag(sk, flag)) { 3947 unsigned long previous_flags = sk->sk_flags; 3948 3949 sock_set_flag(sk, flag); 3950 /* 3951 * we just set one of the two flags which require net 3952 * time stamping, but time stamping might have been on 3953 * already because of the other one 3954 */ 3955 if (sock_needs_netstamp(sk) && 3956 !(previous_flags & SK_FLAGS_TIMESTAMP)) 3957 net_enable_timestamp(); 3958 } 3959 } 3960 3961 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, 3962 int level, int type) 3963 { 3964 struct sock_extended_err ee; 3965 struct sk_buff *skb; 3966 int copied, err; 3967 3968 err = -EAGAIN; 3969 skb = sock_dequeue_err_skb(sk); 3970 if (skb == NULL) 3971 goto out; 3972 3973 copied = skb->len; 3974 if (copied > len) { 3975 msg->msg_flags |= MSG_TRUNC; 3976 copied = len; 3977 } 3978 err = skb_copy_datagram_msg(skb, 0, msg, copied); 3979 if (err) 3980 goto out_free_skb; 3981 3982 sock_recv_timestamp(msg, sk, skb); 3983 3984 /* We must use a bounce buffer for CONFIG_HARDENED_USERCOPY=y */ 3985 ee = SKB_EXT_ERR(skb)->ee; 3986 put_cmsg(msg, level, type, sizeof(ee), &ee); 3987 3988 msg->msg_flags |= MSG_ERRQUEUE; 3989 err = copied; 3990 3991 out_free_skb: 3992 kfree_skb(skb); 3993 out: 3994 return err; 3995 } 3996 EXPORT_SYMBOL(sock_recv_errqueue); 3997 3998 /* 3999 * Get a socket option on an socket. 4000 * 4001 * FIX: POSIX 1003.1g is very ambiguous here. It states that 4002 * asynchronous errors should be reported by getsockopt. We assume 4003 * this means if you specify SO_ERROR (otherwise what is the point of it). 4004 */ 4005 int sock_common_getsockopt(struct socket *sock, int level, int optname, 4006 char __user *optval, int __user *optlen) 4007 { 4008 struct sock *sk = sock->sk; 4009 4010 /* IPV6_ADDRFORM can change sk->sk_prot under us. */ 4011 return READ_ONCE(sk->sk_prot)->getsockopt(sk, level, optname, optval, optlen); 4012 } 4013 EXPORT_SYMBOL(sock_common_getsockopt); 4014 4015 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, 4016 int flags) 4017 { 4018 struct sock *sk = sock->sk; 4019 4020 return sk->sk_prot->recvmsg(sk, msg, size, flags); 4021 } 4022 EXPORT_SYMBOL(sock_common_recvmsg); 4023 4024 /* 4025 * Set socket options on an inet socket. 4026 */ 4027 int sock_common_setsockopt(struct socket *sock, int level, int optname, 4028 sockptr_t optval, unsigned int optlen) 4029 { 4030 struct sock *sk = sock->sk; 4031 4032 /* IPV6_ADDRFORM can change sk->sk_prot under us. */ 4033 return READ_ONCE(sk->sk_prot)->setsockopt(sk, level, optname, optval, optlen); 4034 } 4035 EXPORT_SYMBOL(sock_common_setsockopt); 4036 4037 void sk_common_release(struct sock *sk) 4038 { 4039 if (sk->sk_prot->destroy) 4040 sk->sk_prot->destroy(sk); 4041 4042 /* 4043 * Observation: when sk_common_release is called, processes have 4044 * no access to socket. But net still has. 4045 * Step one, detach it from networking: 4046 * 4047 * A. Remove from hash tables. 4048 */ 4049 4050 sk->sk_prot->unhash(sk); 4051 4052 /* 4053 * In this point socket cannot receive new packets, but it is possible 4054 * that some packets are in flight because some CPU runs receiver and 4055 * did hash table lookup before we unhashed socket. They will achieve 4056 * receive queue and will be purged by socket destructor. 4057 * 4058 * Also we still have packets pending on receive queue and probably, 4059 * our own packets waiting in device queues. sock_destroy will drain 4060 * receive queue, but transmitted packets will delay socket destruction 4061 * until the last reference will be released. 4062 */ 4063 4064 sock_orphan(sk); 4065 4066 xfrm_sk_free_policy(sk); 4067 4068 sock_put(sk); 4069 } 4070 EXPORT_SYMBOL(sk_common_release); 4071 4072 void sk_get_meminfo(const struct sock *sk, u32 *mem) 4073 { 4074 memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS); 4075 4076 mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk); 4077 mem[SK_MEMINFO_RCVBUF] = READ_ONCE(sk->sk_rcvbuf); 4078 mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk); 4079 mem[SK_MEMINFO_SNDBUF] = READ_ONCE(sk->sk_sndbuf); 4080 mem[SK_MEMINFO_FWD_ALLOC] = READ_ONCE(sk->sk_forward_alloc); 4081 mem[SK_MEMINFO_WMEM_QUEUED] = READ_ONCE(sk->sk_wmem_queued); 4082 mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc); 4083 mem[SK_MEMINFO_BACKLOG] = READ_ONCE(sk->sk_backlog.len); 4084 mem[SK_MEMINFO_DROPS] = sk_drops_read(sk); 4085 } 4086 4087 #ifdef CONFIG_PROC_FS 4088 static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR); 4089 4090 int sock_prot_inuse_get(struct net *net, struct proto *prot) 4091 { 4092 int cpu, idx = prot->inuse_idx; 4093 int res = 0; 4094 4095 for_each_possible_cpu(cpu) 4096 res += per_cpu_ptr(net->core.prot_inuse, cpu)->val[idx]; 4097 4098 return res >= 0 ? res : 0; 4099 } 4100 EXPORT_SYMBOL_GPL(sock_prot_inuse_get); 4101 4102 int sock_inuse_get(struct net *net) 4103 { 4104 int cpu, res = 0; 4105 4106 for_each_possible_cpu(cpu) 4107 res += per_cpu_ptr(net->core.prot_inuse, cpu)->all; 4108 4109 return res; 4110 } 4111 4112 EXPORT_SYMBOL_GPL(sock_inuse_get); 4113 4114 static int __net_init sock_inuse_init_net(struct net *net) 4115 { 4116 net->core.prot_inuse = alloc_percpu(struct prot_inuse); 4117 if (net->core.prot_inuse == NULL) 4118 return -ENOMEM; 4119 return 0; 4120 } 4121 4122 static void __net_exit sock_inuse_exit_net(struct net *net) 4123 { 4124 free_percpu(net->core.prot_inuse); 4125 } 4126 4127 static struct pernet_operations net_inuse_ops = { 4128 .init = sock_inuse_init_net, 4129 .exit = sock_inuse_exit_net, 4130 }; 4131 4132 static __init int net_inuse_init(void) 4133 { 4134 if (register_pernet_subsys(&net_inuse_ops)) 4135 panic("Cannot initialize net inuse counters"); 4136 4137 return 0; 4138 } 4139 4140 core_initcall(net_inuse_init); 4141 4142 static int assign_proto_idx(struct proto *prot) 4143 { 4144 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR); 4145 4146 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR)) { 4147 pr_err("PROTO_INUSE_NR exhausted\n"); 4148 return -ENOSPC; 4149 } 4150 4151 set_bit(prot->inuse_idx, proto_inuse_idx); 4152 return 0; 4153 } 4154 4155 static void release_proto_idx(struct proto *prot) 4156 { 4157 if (prot->inuse_idx != PROTO_INUSE_NR) 4158 clear_bit(prot->inuse_idx, proto_inuse_idx); 4159 } 4160 #else 4161 static inline int assign_proto_idx(struct proto *prot) 4162 { 4163 return 0; 4164 } 4165 4166 static inline void release_proto_idx(struct proto *prot) 4167 { 4168 } 4169 4170 #endif 4171 4172 static void tw_prot_cleanup(struct timewait_sock_ops *twsk_prot) 4173 { 4174 if (!twsk_prot) 4175 return; 4176 kfree(twsk_prot->twsk_slab_name); 4177 twsk_prot->twsk_slab_name = NULL; 4178 kmem_cache_destroy(twsk_prot->twsk_slab); 4179 twsk_prot->twsk_slab = NULL; 4180 } 4181 4182 static int tw_prot_init(const struct proto *prot) 4183 { 4184 struct timewait_sock_ops *twsk_prot = prot->twsk_prot; 4185 4186 if (!twsk_prot) 4187 return 0; 4188 4189 twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", 4190 prot->name); 4191 if (!twsk_prot->twsk_slab_name) 4192 return -ENOMEM; 4193 4194 twsk_prot->twsk_slab = 4195 kmem_cache_create(twsk_prot->twsk_slab_name, 4196 twsk_prot->twsk_obj_size, 0, 4197 SLAB_ACCOUNT | prot->slab_flags, 4198 NULL); 4199 if (!twsk_prot->twsk_slab) { 4200 pr_crit("%s: Can't create timewait sock SLAB cache!\n", 4201 prot->name); 4202 return -ENOMEM; 4203 } 4204 4205 return 0; 4206 } 4207 4208 static void req_prot_cleanup(struct request_sock_ops *rsk_prot) 4209 { 4210 if (!rsk_prot) 4211 return; 4212 kfree(rsk_prot->slab_name); 4213 rsk_prot->slab_name = NULL; 4214 kmem_cache_destroy(rsk_prot->slab); 4215 rsk_prot->slab = NULL; 4216 } 4217 4218 static int req_prot_init(const struct proto *prot) 4219 { 4220 struct request_sock_ops *rsk_prot = prot->rsk_prot; 4221 4222 if (!rsk_prot) 4223 return 0; 4224 4225 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s", 4226 prot->name); 4227 if (!rsk_prot->slab_name) 4228 return -ENOMEM; 4229 4230 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name, 4231 rsk_prot->obj_size, 0, 4232 SLAB_ACCOUNT | prot->slab_flags, 4233 NULL); 4234 4235 if (!rsk_prot->slab) { 4236 pr_crit("%s: Can't create request sock SLAB cache!\n", 4237 prot->name); 4238 return -ENOMEM; 4239 } 4240 return 0; 4241 } 4242 4243 int proto_register(struct proto *prot, int alloc_slab) 4244 { 4245 int ret = -ENOBUFS; 4246 4247 if (prot->memory_allocated && !prot->sysctl_mem) { 4248 pr_err("%s: missing sysctl_mem\n", prot->name); 4249 return -EINVAL; 4250 } 4251 if (prot->memory_allocated && !prot->per_cpu_fw_alloc) { 4252 pr_err("%s: missing per_cpu_fw_alloc\n", prot->name); 4253 return -EINVAL; 4254 } 4255 if (alloc_slab) { 4256 struct kmem_cache_args args = { 4257 .useroffset = prot->useroffset, 4258 .usersize = prot->usersize, 4259 .freeptr_offset = prot->freeptr_offset, 4260 .use_freeptr_offset = !!prot->freeptr_offset, 4261 }; 4262 4263 prot->slab = kmem_cache_create(prot->name, prot->obj_size, 4264 &args, 4265 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT | 4266 prot->slab_flags); 4267 if (prot->slab == NULL) { 4268 pr_crit("%s: Can't create sock SLAB cache!\n", 4269 prot->name); 4270 goto out; 4271 } 4272 4273 if (req_prot_init(prot)) 4274 goto out_free_request_sock_slab; 4275 4276 if (tw_prot_init(prot)) 4277 goto out_free_timewait_sock_slab; 4278 } 4279 4280 mutex_lock(&proto_list_mutex); 4281 ret = assign_proto_idx(prot); 4282 if (ret) { 4283 mutex_unlock(&proto_list_mutex); 4284 goto out_free_timewait_sock_slab; 4285 } 4286 list_add(&prot->node, &proto_list); 4287 mutex_unlock(&proto_list_mutex); 4288 return ret; 4289 4290 out_free_timewait_sock_slab: 4291 if (alloc_slab) 4292 tw_prot_cleanup(prot->twsk_prot); 4293 out_free_request_sock_slab: 4294 if (alloc_slab) { 4295 req_prot_cleanup(prot->rsk_prot); 4296 4297 kmem_cache_destroy(prot->slab); 4298 prot->slab = NULL; 4299 } 4300 out: 4301 return ret; 4302 } 4303 EXPORT_SYMBOL(proto_register); 4304 4305 void proto_unregister(struct proto *prot) 4306 { 4307 mutex_lock(&proto_list_mutex); 4308 release_proto_idx(prot); 4309 list_del(&prot->node); 4310 mutex_unlock(&proto_list_mutex); 4311 4312 kmem_cache_destroy(prot->slab); 4313 prot->slab = NULL; 4314 4315 req_prot_cleanup(prot->rsk_prot); 4316 tw_prot_cleanup(prot->twsk_prot); 4317 } 4318 EXPORT_SYMBOL(proto_unregister); 4319 4320 int sock_load_diag_module(int family, int protocol) 4321 { 4322 if (!protocol) { 4323 if (!sock_is_registered(family)) 4324 return -ENOENT; 4325 4326 return request_module("net-pf-%d-proto-%d-type-%d", PF_NETLINK, 4327 NETLINK_SOCK_DIAG, family); 4328 } 4329 4330 #ifdef CONFIG_INET 4331 if (family == AF_INET && 4332 protocol != IPPROTO_RAW && 4333 protocol < MAX_INET_PROTOS && 4334 !rcu_access_pointer(inet_protos[protocol])) 4335 return -ENOENT; 4336 #endif 4337 4338 return request_module("net-pf-%d-proto-%d-type-%d-%d", PF_NETLINK, 4339 NETLINK_SOCK_DIAG, family, protocol); 4340 } 4341 EXPORT_SYMBOL(sock_load_diag_module); 4342 4343 #ifdef CONFIG_PROC_FS 4344 static void *proto_seq_start(struct seq_file *seq, loff_t *pos) 4345 __acquires(proto_list_mutex) 4346 { 4347 mutex_lock(&proto_list_mutex); 4348 return seq_list_start_head(&proto_list, *pos); 4349 } 4350 4351 static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos) 4352 { 4353 return seq_list_next(v, &proto_list, pos); 4354 } 4355 4356 static void proto_seq_stop(struct seq_file *seq, void *v) 4357 __releases(proto_list_mutex) 4358 { 4359 mutex_unlock(&proto_list_mutex); 4360 } 4361 4362 static char proto_method_implemented(const void *method) 4363 { 4364 return method == NULL ? 'n' : 'y'; 4365 } 4366 static long sock_prot_memory_allocated(struct proto *proto) 4367 { 4368 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L; 4369 } 4370 4371 static const char *sock_prot_memory_pressure(struct proto *proto) 4372 { 4373 return proto->memory_pressure != NULL ? 4374 proto_memory_pressure(proto) ? "yes" : "no" : "NI"; 4375 } 4376 4377 static void proto_seq_printf(struct seq_file *seq, struct proto *proto) 4378 { 4379 4380 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s " 4381 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n", 4382 proto->name, 4383 proto->obj_size, 4384 sock_prot_inuse_get(seq_file_net(seq), proto), 4385 sock_prot_memory_allocated(proto), 4386 sock_prot_memory_pressure(proto), 4387 proto->max_header, 4388 proto->slab == NULL ? "no" : "yes", 4389 module_name(proto->owner), 4390 proto_method_implemented(proto->close), 4391 proto_method_implemented(proto->connect), 4392 proto_method_implemented(proto->disconnect), 4393 proto_method_implemented(proto->accept), 4394 proto_method_implemented(proto->ioctl), 4395 proto_method_implemented(proto->init), 4396 proto_method_implemented(proto->destroy), 4397 proto_method_implemented(proto->shutdown), 4398 proto_method_implemented(proto->setsockopt), 4399 proto_method_implemented(proto->getsockopt), 4400 proto_method_implemented(proto->sendmsg), 4401 proto_method_implemented(proto->recvmsg), 4402 proto_method_implemented(proto->bind), 4403 proto_method_implemented(proto->backlog_rcv), 4404 proto_method_implemented(proto->hash), 4405 proto_method_implemented(proto->unhash), 4406 proto_method_implemented(proto->get_port), 4407 proto_method_implemented(proto->enter_memory_pressure)); 4408 } 4409 4410 static int proto_seq_show(struct seq_file *seq, void *v) 4411 { 4412 if (v == &proto_list) 4413 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s", 4414 "protocol", 4415 "size", 4416 "sockets", 4417 "memory", 4418 "press", 4419 "maxhdr", 4420 "slab", 4421 "module", 4422 "cl co di ac io in de sh ss gs se re bi br ha uh gp em\n"); 4423 else 4424 proto_seq_printf(seq, list_entry(v, struct proto, node)); 4425 return 0; 4426 } 4427 4428 static const struct seq_operations proto_seq_ops = { 4429 .start = proto_seq_start, 4430 .next = proto_seq_next, 4431 .stop = proto_seq_stop, 4432 .show = proto_seq_show, 4433 }; 4434 4435 static __net_init int proto_init_net(struct net *net) 4436 { 4437 if (!proc_create_net("protocols", 0444, net->proc_net, &proto_seq_ops, 4438 sizeof(struct seq_net_private))) 4439 return -ENOMEM; 4440 4441 return 0; 4442 } 4443 4444 static __net_exit void proto_exit_net(struct net *net) 4445 { 4446 remove_proc_entry("protocols", net->proc_net); 4447 } 4448 4449 4450 static __net_initdata struct pernet_operations proto_net_ops = { 4451 .init = proto_init_net, 4452 .exit = proto_exit_net, 4453 }; 4454 4455 static int __init proto_init(void) 4456 { 4457 return register_pernet_subsys(&proto_net_ops); 4458 } 4459 4460 subsys_initcall(proto_init); 4461 4462 #endif /* PROC_FS */ 4463 4464 #ifdef CONFIG_NET_RX_BUSY_POLL 4465 bool sk_busy_loop_end(void *p, unsigned long start_time) 4466 { 4467 struct sock *sk = p; 4468 4469 if (!skb_queue_empty_lockless(&sk->sk_receive_queue)) 4470 return true; 4471 4472 if (sk_is_udp(sk) && 4473 !skb_queue_empty_lockless(&udp_sk(sk)->reader_queue)) 4474 return true; 4475 4476 return sk_busy_loop_timeout(sk, start_time); 4477 } 4478 EXPORT_SYMBOL(sk_busy_loop_end); 4479 #endif /* CONFIG_NET_RX_BUSY_POLL */ 4480 4481 int sock_bind_add(struct sock *sk, struct sockaddr_unsized *addr, int addr_len) 4482 { 4483 if (!sk->sk_prot->bind_add) 4484 return -EOPNOTSUPP; 4485 return sk->sk_prot->bind_add(sk, addr, addr_len); 4486 } 4487 EXPORT_SYMBOL(sock_bind_add); 4488 4489 /* Copy 'size' bytes from userspace and return `size` back to userspace */ 4490 int sock_ioctl_inout(struct sock *sk, unsigned int cmd, 4491 void __user *arg, void *karg, size_t size) 4492 { 4493 int ret; 4494 4495 if (copy_from_user(karg, arg, size)) 4496 return -EFAULT; 4497 4498 ret = READ_ONCE(sk->sk_prot)->ioctl(sk, cmd, karg); 4499 if (ret) 4500 return ret; 4501 4502 if (copy_to_user(arg, karg, size)) 4503 return -EFAULT; 4504 4505 return 0; 4506 } 4507 EXPORT_SYMBOL(sock_ioctl_inout); 4508 4509 /* This is the most common ioctl prep function, where the result (4 bytes) is 4510 * copied back to userspace if the ioctl() returns successfully. No input is 4511 * copied from userspace as input argument. 4512 */ 4513 static int sock_ioctl_out(struct sock *sk, unsigned int cmd, void __user *arg) 4514 { 4515 int ret, karg = 0; 4516 4517 ret = READ_ONCE(sk->sk_prot)->ioctl(sk, cmd, &karg); 4518 if (ret) 4519 return ret; 4520 4521 return put_user(karg, (int __user *)arg); 4522 } 4523 4524 /* A wrapper around sock ioctls, which copies the data from userspace 4525 * (depending on the protocol/ioctl), and copies back the result to userspace. 4526 * The main motivation for this function is to pass kernel memory to the 4527 * protocol ioctl callbacks, instead of userspace memory. 4528 */ 4529 int sk_ioctl(struct sock *sk, unsigned int cmd, void __user *arg) 4530 { 4531 int rc = 1; 4532 4533 if (sk->sk_type == SOCK_RAW && sk->sk_family == AF_INET) 4534 rc = ipmr_sk_ioctl(sk, cmd, arg); 4535 else if (sk->sk_type == SOCK_RAW && sk->sk_family == AF_INET6) 4536 rc = ip6mr_sk_ioctl(sk, cmd, arg); 4537 else if (sk_is_phonet(sk)) 4538 rc = phonet_sk_ioctl(sk, cmd, arg); 4539 4540 /* If ioctl was processed, returns its value */ 4541 if (rc <= 0) 4542 return rc; 4543 4544 /* Otherwise call the default handler */ 4545 return sock_ioctl_out(sk, cmd, arg); 4546 } 4547 EXPORT_SYMBOL(sk_ioctl); 4548 4549 static int __init sock_struct_check(void) 4550 { 4551 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_drops); 4552 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_peek_off); 4553 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_error_queue); 4554 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_receive_queue); 4555 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_backlog); 4556 4557 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rx_dst); 4558 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rx_dst_ifindex); 4559 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rx_dst_cookie); 4560 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rcvbuf); 4561 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_filter); 4562 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_wq); 4563 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_data_ready); 4564 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rcvtimeo); 4565 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rcvlowat); 4566 4567 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rxtx, sk_err); 4568 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rxtx, sk_socket); 4569 #ifdef CONFIG_MEMCG 4570 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rxtx, sk_memcg); 4571 #endif 4572 4573 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_lock); 4574 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_reserved_mem); 4575 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_forward_alloc); 4576 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_tsflags); 4577 4578 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_omem_alloc); 4579 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_omem_alloc); 4580 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_err_soft); 4581 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_wmem_queued); 4582 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_wmem_alloc); 4583 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_tsq_flags); 4584 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_send_head); 4585 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_write_queue); 4586 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_write_pending); 4587 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_frag); 4588 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_timer); 4589 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_pacing_rate); 4590 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_zckey); 4591 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_tskey); 4592 4593 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_dst_pending_confirm); 4594 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_pacing_status); 4595 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_max_pacing_rate); 4596 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_sndtimeo); 4597 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_priority); 4598 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_mark); 4599 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_uid); 4600 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_protocol); 4601 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_dst_cache); 4602 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_route_caps); 4603 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_gso_type); 4604 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_gso_max_size); 4605 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_allocation); 4606 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_txhash); 4607 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_sndbuf); 4608 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_gso_max_segs); 4609 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_pacing_shift); 4610 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_use_task_frag); 4611 return 0; 4612 } 4613 4614 core_initcall(sock_struct_check); 4615