1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * Generic socket support routines. Memory allocators, socket lock/release 8 * handler for protocols to use and generic option handler. 9 * 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Florian La Roche, <flla@stud.uni-sb.de> 13 * Alan Cox, <A.Cox@swansea.ac.uk> 14 * 15 * Fixes: 16 * Alan Cox : Numerous verify_area() problems 17 * Alan Cox : Connecting on a connecting socket 18 * now returns an error for tcp. 19 * Alan Cox : sock->protocol is set correctly. 20 * and is not sometimes left as 0. 21 * Alan Cox : connect handles icmp errors on a 22 * connect properly. Unfortunately there 23 * is a restart syscall nasty there. I 24 * can't match BSD without hacking the C 25 * library. Ideas urgently sought! 26 * Alan Cox : Disallow bind() to addresses that are 27 * not ours - especially broadcast ones!! 28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost) 29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets, 30 * instead they leave that for the DESTROY timer. 31 * Alan Cox : Clean up error flag in accept 32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer 33 * was buggy. Put a remove_sock() in the handler 34 * for memory when we hit 0. Also altered the timer 35 * code. The ACK stuff can wait and needs major 36 * TCP layer surgery. 37 * Alan Cox : Fixed TCP ack bug, removed remove sock 38 * and fixed timer/inet_bh race. 39 * Alan Cox : Added zapped flag for TCP 40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code 41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb 42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources 43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing. 44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so... 45 * Rick Sladkey : Relaxed UDP rules for matching packets. 46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support 47 * Pauline Middelink : identd support 48 * Alan Cox : Fixed connect() taking signals I think. 49 * Alan Cox : SO_LINGER supported 50 * Alan Cox : Error reporting fixes 51 * Anonymous : inet_create tidied up (sk->reuse setting) 52 * Alan Cox : inet sockets don't set sk->type! 53 * Alan Cox : Split socket option code 54 * Alan Cox : Callbacks 55 * Alan Cox : Nagle flag for Charles & Johannes stuff 56 * Alex : Removed restriction on inet fioctl 57 * Alan Cox : Splitting INET from NET core 58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt() 59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code 60 * Alan Cox : Split IP from generic code 61 * Alan Cox : New kfree_skbmem() 62 * Alan Cox : Make SO_DEBUG superuser only. 63 * Alan Cox : Allow anyone to clear SO_DEBUG 64 * (compatibility fix) 65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput. 66 * Alan Cox : Allocator for a socket is settable. 67 * Alan Cox : SO_ERROR includes soft errors. 68 * Alan Cox : Allow NULL arguments on some SO_ opts 69 * Alan Cox : Generic socket allocation to make hooks 70 * easier (suggested by Craig Metz). 71 * Michael Pall : SO_ERROR returns positive errno again 72 * Steve Whitehouse: Added default destructor to free 73 * protocol private data. 74 * Steve Whitehouse: Added various other default routines 75 * common to several socket families. 76 * Chris Evans : Call suser() check last on F_SETOWN 77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER. 78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s() 79 * Andi Kleen : Fix write_space callback 80 * Chris Evans : Security fixes - signedness again 81 * Arnaldo C. Melo : cleanups, use skb_queue_purge 82 * 83 * To Fix: 84 */ 85 86 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 87 88 #include <linux/unaligned.h> 89 #include <linux/capability.h> 90 #include <linux/errno.h> 91 #include <linux/errqueue.h> 92 #include <linux/types.h> 93 #include <linux/socket.h> 94 #include <linux/in.h> 95 #include <linux/kernel.h> 96 #include <linux/module.h> 97 #include <linux/proc_fs.h> 98 #include <linux/seq_file.h> 99 #include <linux/sched.h> 100 #include <linux/sched/mm.h> 101 #include <linux/timer.h> 102 #include <linux/string.h> 103 #include <linux/sockios.h> 104 #include <linux/net.h> 105 #include <linux/mm.h> 106 #include <linux/slab.h> 107 #include <linux/interrupt.h> 108 #include <linux/poll.h> 109 #include <linux/tcp.h> 110 #include <linux/udp.h> 111 #include <linux/init.h> 112 #include <linux/highmem.h> 113 #include <linux/user_namespace.h> 114 #include <linux/static_key.h> 115 #include <linux/memcontrol.h> 116 #include <linux/prefetch.h> 117 #include <linux/compat.h> 118 #include <linux/mroute.h> 119 #include <linux/mroute6.h> 120 #include <linux/icmpv6.h> 121 122 #include <linux/uaccess.h> 123 124 #include <linux/netdevice.h> 125 #include <net/protocol.h> 126 #include <linux/skbuff.h> 127 #include <linux/skbuff_ref.h> 128 #include <net/net_namespace.h> 129 #include <net/request_sock.h> 130 #include <net/sock.h> 131 #include <net/proto_memory.h> 132 #include <linux/net_tstamp.h> 133 #include <net/xfrm.h> 134 #include <linux/ipsec.h> 135 #include <net/cls_cgroup.h> 136 #include <net/netprio_cgroup.h> 137 #include <linux/sock_diag.h> 138 139 #include <linux/filter.h> 140 #include <net/sock_reuseport.h> 141 #include <net/bpf_sk_storage.h> 142 143 #include <trace/events/sock.h> 144 145 #include <net/tcp.h> 146 #include <net/busy_poll.h> 147 #include <net/phonet/phonet.h> 148 149 #include <linux/ethtool.h> 150 151 #include <uapi/linux/pidfd.h> 152 153 #include "dev.h" 154 155 static DEFINE_MUTEX(proto_list_mutex); 156 static LIST_HEAD(proto_list); 157 158 static void sock_def_write_space_wfree(struct sock *sk, int wmem_alloc); 159 static void sock_def_write_space(struct sock *sk); 160 161 /** 162 * sk_ns_capable - General socket capability test 163 * @sk: Socket to use a capability on or through 164 * @user_ns: The user namespace of the capability to use 165 * @cap: The capability to use 166 * 167 * Test to see if the opener of the socket had when the socket was 168 * created and the current process has the capability @cap in the user 169 * namespace @user_ns. 170 */ 171 bool sk_ns_capable(const struct sock *sk, 172 struct user_namespace *user_ns, int cap) 173 { 174 return file_ns_capable(sk->sk_socket->file, user_ns, cap) && 175 ns_capable(user_ns, cap); 176 } 177 EXPORT_SYMBOL(sk_ns_capable); 178 179 /** 180 * sk_capable - Socket global capability test 181 * @sk: Socket to use a capability on or through 182 * @cap: The global capability to use 183 * 184 * Test to see if the opener of the socket had when the socket was 185 * created and the current process has the capability @cap in all user 186 * namespaces. 187 */ 188 bool sk_capable(const struct sock *sk, int cap) 189 { 190 return sk_ns_capable(sk, &init_user_ns, cap); 191 } 192 EXPORT_SYMBOL(sk_capable); 193 194 /** 195 * sk_net_capable - Network namespace socket capability test 196 * @sk: Socket to use a capability on or through 197 * @cap: The capability to use 198 * 199 * Test to see if the opener of the socket had when the socket was created 200 * and the current process has the capability @cap over the network namespace 201 * the socket is a member of. 202 */ 203 bool sk_net_capable(const struct sock *sk, int cap) 204 { 205 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap); 206 } 207 EXPORT_SYMBOL(sk_net_capable); 208 209 /* 210 * Each address family might have different locking rules, so we have 211 * one slock key per address family and separate keys for internal and 212 * userspace sockets. 213 */ 214 static struct lock_class_key af_family_keys[AF_MAX]; 215 static struct lock_class_key af_family_kern_keys[AF_MAX]; 216 static struct lock_class_key af_family_slock_keys[AF_MAX]; 217 static struct lock_class_key af_family_kern_slock_keys[AF_MAX]; 218 219 /* 220 * Make lock validator output more readable. (we pre-construct these 221 * strings build-time, so that runtime initialization of socket 222 * locks is fast): 223 */ 224 225 #define _sock_locks(x) \ 226 x "AF_UNSPEC", x "AF_UNIX" , x "AF_INET" , \ 227 x "AF_AX25" , x "AF_IPX" , x "AF_APPLETALK", \ 228 x "AF_NETROM", x "AF_BRIDGE" , x "AF_ATMPVC" , \ 229 x "AF_X25" , x "AF_INET6" , x "AF_ROSE" , \ 230 x "AF_DECnet", x "AF_NETBEUI" , x "AF_SECURITY" , \ 231 x "AF_KEY" , x "AF_NETLINK" , x "AF_PACKET" , \ 232 x "AF_ASH" , x "AF_ECONET" , x "AF_ATMSVC" , \ 233 x "AF_RDS" , x "AF_SNA" , x "AF_IRDA" , \ 234 x "AF_PPPOX" , x "AF_WANPIPE" , x "AF_LLC" , \ 235 x "27" , x "28" , x "AF_CAN" , \ 236 x "AF_TIPC" , x "AF_BLUETOOTH", x "IUCV" , \ 237 x "AF_RXRPC" , x "AF_ISDN" , x "AF_PHONET" , \ 238 x "AF_IEEE802154", x "AF_CAIF" , x "AF_ALG" , \ 239 x "AF_NFC" , x "AF_VSOCK" , x "AF_KCM" , \ 240 x "AF_QIPCRTR", x "AF_SMC" , x "AF_XDP" , \ 241 x "AF_MCTP" , \ 242 x "AF_MAX" 243 244 static const char *const af_family_key_strings[AF_MAX+1] = { 245 _sock_locks("sk_lock-") 246 }; 247 static const char *const af_family_slock_key_strings[AF_MAX+1] = { 248 _sock_locks("slock-") 249 }; 250 static const char *const af_family_clock_key_strings[AF_MAX+1] = { 251 _sock_locks("clock-") 252 }; 253 254 static const char *const af_family_kern_key_strings[AF_MAX+1] = { 255 _sock_locks("k-sk_lock-") 256 }; 257 static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = { 258 _sock_locks("k-slock-") 259 }; 260 static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = { 261 _sock_locks("k-clock-") 262 }; 263 static const char *const af_family_rlock_key_strings[AF_MAX+1] = { 264 _sock_locks("rlock-") 265 }; 266 static const char *const af_family_wlock_key_strings[AF_MAX+1] = { 267 _sock_locks("wlock-") 268 }; 269 static const char *const af_family_elock_key_strings[AF_MAX+1] = { 270 _sock_locks("elock-") 271 }; 272 273 /* 274 * sk_callback_lock and sk queues locking rules are per-address-family, 275 * so split the lock classes by using a per-AF key: 276 */ 277 static struct lock_class_key af_callback_keys[AF_MAX]; 278 static struct lock_class_key af_rlock_keys[AF_MAX]; 279 static struct lock_class_key af_wlock_keys[AF_MAX]; 280 static struct lock_class_key af_elock_keys[AF_MAX]; 281 static struct lock_class_key af_kern_callback_keys[AF_MAX]; 282 283 /* Run time adjustable parameters. */ 284 __u32 sysctl_wmem_max __read_mostly = 4 << 20; 285 EXPORT_SYMBOL(sysctl_wmem_max); 286 __u32 sysctl_rmem_max __read_mostly = 4 << 20; 287 EXPORT_SYMBOL(sysctl_rmem_max); 288 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_DEFAULT; 289 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_DEFAULT; 290 291 DEFINE_STATIC_KEY_FALSE(memalloc_socks_key); 292 EXPORT_SYMBOL_GPL(memalloc_socks_key); 293 294 /** 295 * sk_set_memalloc - sets %SOCK_MEMALLOC 296 * @sk: socket to set it on 297 * 298 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves. 299 * It's the responsibility of the admin to adjust min_free_kbytes 300 * to meet the requirements 301 */ 302 void sk_set_memalloc(struct sock *sk) 303 { 304 sock_set_flag(sk, SOCK_MEMALLOC); 305 sk->sk_allocation |= __GFP_MEMALLOC; 306 static_branch_inc(&memalloc_socks_key); 307 } 308 EXPORT_SYMBOL_GPL(sk_set_memalloc); 309 310 void sk_clear_memalloc(struct sock *sk) 311 { 312 sock_reset_flag(sk, SOCK_MEMALLOC); 313 sk->sk_allocation &= ~__GFP_MEMALLOC; 314 static_branch_dec(&memalloc_socks_key); 315 316 /* 317 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward 318 * progress of swapping. SOCK_MEMALLOC may be cleared while 319 * it has rmem allocations due to the last swapfile being deactivated 320 * but there is a risk that the socket is unusable due to exceeding 321 * the rmem limits. Reclaim the reserves and obey rmem limits again. 322 */ 323 sk_mem_reclaim(sk); 324 } 325 EXPORT_SYMBOL_GPL(sk_clear_memalloc); 326 327 int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb) 328 { 329 int ret; 330 unsigned int noreclaim_flag; 331 332 /* these should have been dropped before queueing */ 333 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC)); 334 335 noreclaim_flag = memalloc_noreclaim_save(); 336 ret = INDIRECT_CALL_INET(sk->sk_backlog_rcv, 337 tcp_v6_do_rcv, 338 tcp_v4_do_rcv, 339 sk, skb); 340 memalloc_noreclaim_restore(noreclaim_flag); 341 342 return ret; 343 } 344 EXPORT_SYMBOL(__sk_backlog_rcv); 345 346 void sk_error_report(struct sock *sk) 347 { 348 sk->sk_error_report(sk); 349 350 switch (sk->sk_family) { 351 case AF_INET: 352 fallthrough; 353 case AF_INET6: 354 trace_inet_sk_error_report(sk); 355 break; 356 default: 357 break; 358 } 359 } 360 EXPORT_SYMBOL(sk_error_report); 361 362 int sock_get_timeout(long timeo, void *optval, bool old_timeval) 363 { 364 struct __kernel_sock_timeval tv; 365 366 if (timeo == MAX_SCHEDULE_TIMEOUT) { 367 tv.tv_sec = 0; 368 tv.tv_usec = 0; 369 } else { 370 tv.tv_sec = timeo / HZ; 371 tv.tv_usec = ((timeo % HZ) * USEC_PER_SEC) / HZ; 372 } 373 374 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) { 375 struct old_timeval32 tv32 = { tv.tv_sec, tv.tv_usec }; 376 *(struct old_timeval32 *)optval = tv32; 377 return sizeof(tv32); 378 } 379 380 if (old_timeval) { 381 struct __kernel_old_timeval old_tv; 382 old_tv.tv_sec = tv.tv_sec; 383 old_tv.tv_usec = tv.tv_usec; 384 *(struct __kernel_old_timeval *)optval = old_tv; 385 return sizeof(old_tv); 386 } 387 388 *(struct __kernel_sock_timeval *)optval = tv; 389 return sizeof(tv); 390 } 391 EXPORT_SYMBOL(sock_get_timeout); 392 393 int sock_copy_user_timeval(struct __kernel_sock_timeval *tv, 394 sockptr_t optval, int optlen, bool old_timeval) 395 { 396 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) { 397 struct old_timeval32 tv32; 398 399 if (optlen < sizeof(tv32)) 400 return -EINVAL; 401 402 if (copy_from_sockptr(&tv32, optval, sizeof(tv32))) 403 return -EFAULT; 404 tv->tv_sec = tv32.tv_sec; 405 tv->tv_usec = tv32.tv_usec; 406 } else if (old_timeval) { 407 struct __kernel_old_timeval old_tv; 408 409 if (optlen < sizeof(old_tv)) 410 return -EINVAL; 411 if (copy_from_sockptr(&old_tv, optval, sizeof(old_tv))) 412 return -EFAULT; 413 tv->tv_sec = old_tv.tv_sec; 414 tv->tv_usec = old_tv.tv_usec; 415 } else { 416 if (optlen < sizeof(*tv)) 417 return -EINVAL; 418 if (copy_from_sockptr(tv, optval, sizeof(*tv))) 419 return -EFAULT; 420 } 421 422 return 0; 423 } 424 EXPORT_SYMBOL(sock_copy_user_timeval); 425 426 static int sock_set_timeout(long *timeo_p, sockptr_t optval, int optlen, 427 bool old_timeval) 428 { 429 struct __kernel_sock_timeval tv; 430 int err = sock_copy_user_timeval(&tv, optval, optlen, old_timeval); 431 long val; 432 433 if (err) 434 return err; 435 436 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC) 437 return -EDOM; 438 439 if (tv.tv_sec < 0) { 440 static int warned __read_mostly; 441 442 WRITE_ONCE(*timeo_p, 0); 443 if (warned < 10 && net_ratelimit()) { 444 warned++; 445 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n", 446 __func__, current->comm, task_pid_nr(current)); 447 } 448 return 0; 449 } 450 val = MAX_SCHEDULE_TIMEOUT; 451 if ((tv.tv_sec || tv.tv_usec) && 452 (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT / HZ - 1))) 453 val = tv.tv_sec * HZ + DIV_ROUND_UP((unsigned long)tv.tv_usec, 454 USEC_PER_SEC / HZ); 455 WRITE_ONCE(*timeo_p, val); 456 return 0; 457 } 458 459 static bool sk_set_prio_allowed(const struct sock *sk, int val) 460 { 461 return ((val >= TC_PRIO_BESTEFFORT && val <= TC_PRIO_INTERACTIVE) || 462 sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) || 463 sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)); 464 } 465 466 static bool sock_needs_netstamp(const struct sock *sk) 467 { 468 switch (sk->sk_family) { 469 case AF_UNSPEC: 470 case AF_UNIX: 471 return false; 472 default: 473 return true; 474 } 475 } 476 477 static void sock_disable_timestamp(struct sock *sk, unsigned long flags) 478 { 479 if (sk->sk_flags & flags) { 480 sk->sk_flags &= ~flags; 481 if (sock_needs_netstamp(sk) && 482 !(sk->sk_flags & SK_FLAGS_TIMESTAMP)) 483 net_disable_timestamp(); 484 } 485 } 486 487 488 int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 489 { 490 unsigned long flags; 491 struct sk_buff_head *list = &sk->sk_receive_queue; 492 493 if (atomic_read(&sk->sk_rmem_alloc) >= READ_ONCE(sk->sk_rcvbuf)) { 494 sk_drops_inc(sk); 495 trace_sock_rcvqueue_full(sk, skb); 496 return -ENOMEM; 497 } 498 499 if (!sk_rmem_schedule(sk, skb, skb->truesize)) { 500 sk_drops_inc(sk); 501 return -ENOBUFS; 502 } 503 504 skb->dev = NULL; 505 skb_set_owner_r(skb, sk); 506 507 /* we escape from rcu protected region, make sure we dont leak 508 * a norefcounted dst 509 */ 510 skb_dst_force(skb); 511 512 spin_lock_irqsave(&list->lock, flags); 513 sock_skb_set_dropcount(sk, skb); 514 __skb_queue_tail(list, skb); 515 spin_unlock_irqrestore(&list->lock, flags); 516 517 if (!sock_flag(sk, SOCK_DEAD)) 518 sk->sk_data_ready(sk); 519 return 0; 520 } 521 EXPORT_SYMBOL(__sock_queue_rcv_skb); 522 523 enum skb_drop_reason 524 sock_queue_rcv_skb_reason(struct sock *sk, struct sk_buff *skb) 525 { 526 enum skb_drop_reason drop_reason; 527 int err; 528 529 drop_reason = sk_filter_reason(sk, skb); 530 if (drop_reason) 531 return drop_reason; 532 533 err = __sock_queue_rcv_skb(sk, skb); 534 switch (err) { 535 case -ENOMEM: 536 return SKB_DROP_REASON_SOCKET_RCVBUFF; 537 case -ENOBUFS: 538 return SKB_DROP_REASON_PROTO_MEM; 539 } 540 return SKB_NOT_DROPPED_YET; 541 } 542 EXPORT_SYMBOL(sock_queue_rcv_skb_reason); 543 544 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, 545 const int nested, unsigned int trim_cap, bool refcounted) 546 { 547 enum skb_drop_reason reason; 548 int rc = NET_RX_SUCCESS; 549 int err; 550 551 reason = sk_filter_trim_cap(sk, skb, trim_cap); 552 if (reason) 553 goto discard_and_relse; 554 555 skb->dev = NULL; 556 557 if (sk_rcvqueues_full(sk, READ_ONCE(sk->sk_rcvbuf))) { 558 sk_drops_inc(sk); 559 reason = SKB_DROP_REASON_SOCKET_RCVBUFF; 560 goto discard_and_relse; 561 } 562 if (nested) 563 bh_lock_sock_nested(sk); 564 else 565 bh_lock_sock(sk); 566 if (!sock_owned_by_user(sk)) { 567 /* 568 * trylock + unlock semantics: 569 */ 570 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_); 571 572 rc = sk_backlog_rcv(sk, skb); 573 574 mutex_release(&sk->sk_lock.dep_map, _RET_IP_); 575 } else if ((err = sk_add_backlog(sk, skb, READ_ONCE(sk->sk_rcvbuf)))) { 576 bh_unlock_sock(sk); 577 if (err == -ENOMEM) 578 reason = SKB_DROP_REASON_PFMEMALLOC; 579 if (err == -ENOBUFS) 580 reason = SKB_DROP_REASON_SOCKET_BACKLOG; 581 sk_drops_inc(sk); 582 goto discard_and_relse; 583 } 584 585 bh_unlock_sock(sk); 586 out: 587 if (refcounted) 588 sock_put(sk); 589 return rc; 590 discard_and_relse: 591 sk_skb_reason_drop(sk, skb, reason); 592 goto out; 593 } 594 EXPORT_SYMBOL(__sk_receive_skb); 595 596 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ip6_dst_check(struct dst_entry *, 597 u32)); 598 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *, 599 u32)); 600 struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie) 601 { 602 struct dst_entry *dst = __sk_dst_get(sk); 603 604 if (dst && READ_ONCE(dst->obsolete) && 605 INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check, 606 dst, cookie) == NULL) { 607 sk_tx_queue_clear(sk); 608 WRITE_ONCE(sk->sk_dst_pending_confirm, 0); 609 RCU_INIT_POINTER(sk->sk_dst_cache, NULL); 610 dst_release(dst); 611 return NULL; 612 } 613 614 return dst; 615 } 616 EXPORT_SYMBOL(__sk_dst_check); 617 618 struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie) 619 { 620 struct dst_entry *dst = sk_dst_get(sk); 621 622 if (dst && READ_ONCE(dst->obsolete) && 623 INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check, 624 dst, cookie) == NULL) { 625 sk_dst_reset(sk); 626 dst_release(dst); 627 return NULL; 628 } 629 630 return dst; 631 } 632 EXPORT_SYMBOL(sk_dst_check); 633 634 static int sock_bindtoindex_locked(struct sock *sk, int ifindex) 635 { 636 int ret = -ENOPROTOOPT; 637 #ifdef CONFIG_NETDEVICES 638 struct net *net = sock_net(sk); 639 640 /* Sorry... */ 641 ret = -EPERM; 642 if (sk->sk_bound_dev_if && !ns_capable(net->user_ns, CAP_NET_RAW)) 643 goto out; 644 645 ret = -EINVAL; 646 if (ifindex < 0) 647 goto out; 648 649 /* Paired with all READ_ONCE() done locklessly. */ 650 WRITE_ONCE(sk->sk_bound_dev_if, ifindex); 651 652 if (sk->sk_prot->rehash) 653 sk->sk_prot->rehash(sk); 654 sk_dst_reset(sk); 655 656 ret = 0; 657 658 out: 659 #endif 660 661 return ret; 662 } 663 664 int sock_bindtoindex(struct sock *sk, int ifindex, bool lock_sk) 665 { 666 int ret; 667 668 if (lock_sk) 669 lock_sock(sk); 670 ret = sock_bindtoindex_locked(sk, ifindex); 671 if (lock_sk) 672 release_sock(sk); 673 674 return ret; 675 } 676 EXPORT_SYMBOL(sock_bindtoindex); 677 678 static int sock_setbindtodevice(struct sock *sk, sockptr_t optval, int optlen) 679 { 680 int ret = -ENOPROTOOPT; 681 #ifdef CONFIG_NETDEVICES 682 struct net *net = sock_net(sk); 683 char devname[IFNAMSIZ]; 684 int index; 685 686 ret = -EINVAL; 687 if (optlen < 0) 688 goto out; 689 690 /* Bind this socket to a particular device like "eth0", 691 * as specified in the passed interface name. If the 692 * name is "" or the option length is zero the socket 693 * is not bound. 694 */ 695 if (optlen > IFNAMSIZ - 1) 696 optlen = IFNAMSIZ - 1; 697 memset(devname, 0, sizeof(devname)); 698 699 ret = -EFAULT; 700 if (copy_from_sockptr(devname, optval, optlen)) 701 goto out; 702 703 index = 0; 704 if (devname[0] != '\0') { 705 struct net_device *dev; 706 707 rcu_read_lock(); 708 dev = dev_get_by_name_rcu(net, devname); 709 if (dev) 710 index = dev->ifindex; 711 rcu_read_unlock(); 712 ret = -ENODEV; 713 if (!dev) 714 goto out; 715 } 716 717 sockopt_lock_sock(sk); 718 ret = sock_bindtoindex_locked(sk, index); 719 sockopt_release_sock(sk); 720 out: 721 #endif 722 723 return ret; 724 } 725 726 static int sock_getbindtodevice(struct sock *sk, sockptr_t optval, 727 sockptr_t optlen, int len) 728 { 729 int ret = -ENOPROTOOPT; 730 #ifdef CONFIG_NETDEVICES 731 int bound_dev_if = READ_ONCE(sk->sk_bound_dev_if); 732 struct net *net = sock_net(sk); 733 char devname[IFNAMSIZ]; 734 735 if (bound_dev_if == 0) { 736 len = 0; 737 goto zero; 738 } 739 740 ret = -EINVAL; 741 if (len < IFNAMSIZ) 742 goto out; 743 744 ret = netdev_get_name(net, devname, bound_dev_if); 745 if (ret) 746 goto out; 747 748 len = strlen(devname) + 1; 749 750 ret = -EFAULT; 751 if (copy_to_sockptr(optval, devname, len)) 752 goto out; 753 754 zero: 755 ret = -EFAULT; 756 if (copy_to_sockptr(optlen, &len, sizeof(int))) 757 goto out; 758 759 ret = 0; 760 761 out: 762 #endif 763 764 return ret; 765 } 766 767 bool sk_mc_loop(const struct sock *sk) 768 { 769 if (dev_recursion_level()) 770 return false; 771 if (!sk) 772 return true; 773 /* IPV6_ADDRFORM can change sk->sk_family under us. */ 774 switch (READ_ONCE(sk->sk_family)) { 775 case AF_INET: 776 return inet_test_bit(MC_LOOP, sk); 777 #if IS_ENABLED(CONFIG_IPV6) 778 case AF_INET6: 779 return inet6_test_bit(MC6_LOOP, sk); 780 #endif 781 } 782 WARN_ON_ONCE(1); 783 return true; 784 } 785 EXPORT_SYMBOL(sk_mc_loop); 786 787 void sock_set_reuseaddr(struct sock *sk) 788 { 789 lock_sock(sk); 790 sk->sk_reuse = SK_CAN_REUSE; 791 release_sock(sk); 792 } 793 EXPORT_SYMBOL(sock_set_reuseaddr); 794 795 void sock_set_reuseport(struct sock *sk) 796 { 797 lock_sock(sk); 798 sk->sk_reuseport = true; 799 release_sock(sk); 800 } 801 EXPORT_SYMBOL(sock_set_reuseport); 802 803 void sock_no_linger(struct sock *sk) 804 { 805 lock_sock(sk); 806 WRITE_ONCE(sk->sk_lingertime, 0); 807 sock_set_flag(sk, SOCK_LINGER); 808 release_sock(sk); 809 } 810 EXPORT_SYMBOL(sock_no_linger); 811 812 void sock_set_priority(struct sock *sk, u32 priority) 813 { 814 WRITE_ONCE(sk->sk_priority, priority); 815 } 816 EXPORT_SYMBOL(sock_set_priority); 817 818 void sock_set_sndtimeo(struct sock *sk, s64 secs) 819 { 820 if (secs && secs < MAX_SCHEDULE_TIMEOUT / HZ - 1) 821 WRITE_ONCE(sk->sk_sndtimeo, secs * HZ); 822 else 823 WRITE_ONCE(sk->sk_sndtimeo, MAX_SCHEDULE_TIMEOUT); 824 } 825 EXPORT_SYMBOL(sock_set_sndtimeo); 826 827 static void __sock_set_timestamps(struct sock *sk, bool val, bool new, bool ns) 828 { 829 sock_valbool_flag(sk, SOCK_RCVTSTAMP, val); 830 sock_valbool_flag(sk, SOCK_RCVTSTAMPNS, val && ns); 831 if (val) { 832 sock_valbool_flag(sk, SOCK_TSTAMP_NEW, new); 833 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 834 } 835 } 836 837 void sock_set_timestamp(struct sock *sk, int optname, bool valbool) 838 { 839 switch (optname) { 840 case SO_TIMESTAMP_OLD: 841 __sock_set_timestamps(sk, valbool, false, false); 842 break; 843 case SO_TIMESTAMP_NEW: 844 __sock_set_timestamps(sk, valbool, true, false); 845 break; 846 case SO_TIMESTAMPNS_OLD: 847 __sock_set_timestamps(sk, valbool, false, true); 848 break; 849 case SO_TIMESTAMPNS_NEW: 850 __sock_set_timestamps(sk, valbool, true, true); 851 break; 852 } 853 } 854 855 static int sock_timestamping_bind_phc(struct sock *sk, int phc_index) 856 { 857 struct net *net = sock_net(sk); 858 struct net_device *dev = NULL; 859 bool match = false; 860 int *vclock_index; 861 int i, num; 862 863 if (sk->sk_bound_dev_if) 864 dev = dev_get_by_index(net, sk->sk_bound_dev_if); 865 866 if (!dev) { 867 pr_err("%s: sock not bind to device\n", __func__); 868 return -EOPNOTSUPP; 869 } 870 871 num = ethtool_get_phc_vclocks(dev, &vclock_index); 872 dev_put(dev); 873 874 for (i = 0; i < num; i++) { 875 if (*(vclock_index + i) == phc_index) { 876 match = true; 877 break; 878 } 879 } 880 881 if (num > 0) 882 kfree(vclock_index); 883 884 if (!match) 885 return -EINVAL; 886 887 WRITE_ONCE(sk->sk_bind_phc, phc_index); 888 889 return 0; 890 } 891 892 int sock_set_timestamping(struct sock *sk, int optname, 893 struct so_timestamping timestamping) 894 { 895 int val = timestamping.flags; 896 int ret; 897 898 if (val & ~SOF_TIMESTAMPING_MASK) 899 return -EINVAL; 900 901 if (val & SOF_TIMESTAMPING_OPT_ID_TCP && 902 !(val & SOF_TIMESTAMPING_OPT_ID)) 903 return -EINVAL; 904 905 if (val & SOF_TIMESTAMPING_OPT_ID && 906 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) { 907 if (sk_is_tcp(sk)) { 908 if ((1 << sk->sk_state) & 909 (TCPF_CLOSE | TCPF_LISTEN)) 910 return -EINVAL; 911 if (val & SOF_TIMESTAMPING_OPT_ID_TCP) 912 atomic_set(&sk->sk_tskey, tcp_sk(sk)->write_seq); 913 else 914 atomic_set(&sk->sk_tskey, tcp_sk(sk)->snd_una); 915 } else { 916 atomic_set(&sk->sk_tskey, 0); 917 } 918 } 919 920 if (val & SOF_TIMESTAMPING_OPT_STATS && 921 !(val & SOF_TIMESTAMPING_OPT_TSONLY)) 922 return -EINVAL; 923 924 if (val & SOF_TIMESTAMPING_BIND_PHC) { 925 ret = sock_timestamping_bind_phc(sk, timestamping.bind_phc); 926 if (ret) 927 return ret; 928 } 929 930 WRITE_ONCE(sk->sk_tsflags, val); 931 sock_valbool_flag(sk, SOCK_TSTAMP_NEW, optname == SO_TIMESTAMPING_NEW); 932 sock_valbool_flag(sk, SOCK_TIMESTAMPING_ANY, !!(val & TSFLAGS_ANY)); 933 934 if (val & SOF_TIMESTAMPING_RX_SOFTWARE) 935 sock_enable_timestamp(sk, 936 SOCK_TIMESTAMPING_RX_SOFTWARE); 937 else 938 sock_disable_timestamp(sk, 939 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE)); 940 return 0; 941 } 942 943 #if defined(CONFIG_CGROUP_BPF) 944 void bpf_skops_tx_timestamping(struct sock *sk, struct sk_buff *skb, int op) 945 { 946 struct bpf_sock_ops_kern sock_ops; 947 948 memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); 949 sock_ops.op = op; 950 sock_ops.is_fullsock = 1; 951 sock_ops.sk = sk; 952 bpf_skops_init_skb(&sock_ops, skb, 0); 953 __cgroup_bpf_run_filter_sock_ops(sk, &sock_ops, CGROUP_SOCK_OPS); 954 } 955 #endif 956 957 void sock_set_keepalive(struct sock *sk) 958 { 959 lock_sock(sk); 960 if (sk->sk_prot->keepalive) 961 sk->sk_prot->keepalive(sk, true); 962 sock_valbool_flag(sk, SOCK_KEEPOPEN, true); 963 release_sock(sk); 964 } 965 EXPORT_SYMBOL(sock_set_keepalive); 966 967 static void __sock_set_rcvbuf(struct sock *sk, int val) 968 { 969 struct socket *sock = sk->sk_socket; 970 971 /* Ensure val * 2 fits into an int, to prevent max_t() from treating it 972 * as a negative value. 973 */ 974 val = min_t(int, val, INT_MAX / 2); 975 sk->sk_userlocks |= SOCK_RCVBUF_LOCK; 976 977 /* We double it on the way in to account for "struct sk_buff" etc. 978 * overhead. Applications assume that the SO_RCVBUF setting they make 979 * will allow that much actual data to be received on that socket. 980 * 981 * Applications are unaware that "struct sk_buff" and other overheads 982 * allocate from the receive buffer during socket buffer allocation. 983 * 984 * And after considering the possible alternatives, returning the value 985 * we actually used in getsockopt is the most desirable behavior. 986 */ 987 WRITE_ONCE(sk->sk_rcvbuf, max_t(int, val * 2, SOCK_MIN_RCVBUF)); 988 989 if (sock) { 990 const struct proto_ops *ops = READ_ONCE(sock->ops); 991 992 if (ops->set_rcvbuf) 993 ops->set_rcvbuf(sk, sk->sk_rcvbuf); 994 } 995 } 996 997 void sock_set_rcvbuf(struct sock *sk, int val) 998 { 999 lock_sock(sk); 1000 __sock_set_rcvbuf(sk, val); 1001 release_sock(sk); 1002 } 1003 EXPORT_SYMBOL(sock_set_rcvbuf); 1004 1005 static void __sock_set_mark(struct sock *sk, u32 val) 1006 { 1007 if (val != sk->sk_mark) { 1008 WRITE_ONCE(sk->sk_mark, val); 1009 sk_dst_reset(sk); 1010 } 1011 } 1012 1013 void sock_set_mark(struct sock *sk, u32 val) 1014 { 1015 lock_sock(sk); 1016 __sock_set_mark(sk, val); 1017 release_sock(sk); 1018 } 1019 EXPORT_SYMBOL(sock_set_mark); 1020 1021 static void sock_release_reserved_memory(struct sock *sk, int bytes) 1022 { 1023 /* Round down bytes to multiple of pages */ 1024 bytes = round_down(bytes, PAGE_SIZE); 1025 1026 WARN_ON(bytes > sk->sk_reserved_mem); 1027 WRITE_ONCE(sk->sk_reserved_mem, sk->sk_reserved_mem - bytes); 1028 sk_mem_reclaim(sk); 1029 } 1030 1031 static int sock_reserve_memory(struct sock *sk, int bytes) 1032 { 1033 long allocated; 1034 bool charged; 1035 int pages; 1036 1037 if (!mem_cgroup_sk_enabled(sk) || !sk_has_account(sk)) 1038 return -EOPNOTSUPP; 1039 1040 if (!bytes) 1041 return 0; 1042 1043 pages = sk_mem_pages(bytes); 1044 1045 /* pre-charge to memcg */ 1046 charged = mem_cgroup_sk_charge(sk, pages, 1047 GFP_KERNEL | __GFP_RETRY_MAYFAIL); 1048 if (!charged) 1049 return -ENOMEM; 1050 1051 if (sk->sk_bypass_prot_mem) 1052 goto success; 1053 1054 /* pre-charge to forward_alloc */ 1055 sk_memory_allocated_add(sk, pages); 1056 allocated = sk_memory_allocated(sk); 1057 1058 /* If the system goes into memory pressure with this 1059 * precharge, give up and return error. 1060 */ 1061 if (allocated > sk_prot_mem_limits(sk, 1)) { 1062 sk_memory_allocated_sub(sk, pages); 1063 mem_cgroup_sk_uncharge(sk, pages); 1064 return -ENOMEM; 1065 } 1066 1067 success: 1068 sk_forward_alloc_add(sk, pages << PAGE_SHIFT); 1069 1070 WRITE_ONCE(sk->sk_reserved_mem, 1071 sk->sk_reserved_mem + (pages << PAGE_SHIFT)); 1072 1073 return 0; 1074 } 1075 1076 #ifdef CONFIG_PAGE_POOL 1077 1078 /* This is the number of tokens and frags that the user can SO_DEVMEM_DONTNEED 1079 * in 1 syscall. The limit exists to limit the amount of memory the kernel 1080 * allocates to copy these tokens, and to prevent looping over the frags for 1081 * too long. 1082 */ 1083 #define MAX_DONTNEED_TOKENS 128 1084 #define MAX_DONTNEED_FRAGS 1024 1085 1086 static noinline_for_stack int 1087 sock_devmem_dontneed(struct sock *sk, sockptr_t optval, unsigned int optlen) 1088 { 1089 unsigned int num_tokens, i, j, k, netmem_num = 0; 1090 struct dmabuf_token *tokens; 1091 int ret = 0, num_frags = 0; 1092 netmem_ref netmems[16]; 1093 1094 if (!sk_is_tcp(sk)) 1095 return -EBADF; 1096 1097 if (optlen % sizeof(*tokens) || 1098 optlen > sizeof(*tokens) * MAX_DONTNEED_TOKENS) 1099 return -EINVAL; 1100 1101 num_tokens = optlen / sizeof(*tokens); 1102 tokens = kvmalloc_objs(*tokens, num_tokens); 1103 if (!tokens) 1104 return -ENOMEM; 1105 1106 if (copy_from_sockptr(tokens, optval, optlen)) { 1107 kvfree(tokens); 1108 return -EFAULT; 1109 } 1110 1111 xa_lock_bh(&sk->sk_user_frags); 1112 for (i = 0; i < num_tokens; i++) { 1113 for (j = 0; j < tokens[i].token_count; j++) { 1114 if (++num_frags > MAX_DONTNEED_FRAGS) 1115 goto frag_limit_reached; 1116 1117 netmem_ref netmem = (__force netmem_ref)__xa_erase( 1118 &sk->sk_user_frags, tokens[i].token_start + j); 1119 1120 if (!netmem || WARN_ON_ONCE(!netmem_is_net_iov(netmem))) 1121 continue; 1122 1123 netmems[netmem_num++] = netmem; 1124 if (netmem_num == ARRAY_SIZE(netmems)) { 1125 xa_unlock_bh(&sk->sk_user_frags); 1126 for (k = 0; k < netmem_num; k++) 1127 WARN_ON_ONCE(!napi_pp_put_page(netmems[k])); 1128 netmem_num = 0; 1129 xa_lock_bh(&sk->sk_user_frags); 1130 } 1131 ret++; 1132 } 1133 } 1134 1135 frag_limit_reached: 1136 xa_unlock_bh(&sk->sk_user_frags); 1137 for (k = 0; k < netmem_num; k++) 1138 WARN_ON_ONCE(!napi_pp_put_page(netmems[k])); 1139 1140 kvfree(tokens); 1141 return ret; 1142 } 1143 #endif 1144 1145 void sockopt_lock_sock(struct sock *sk) 1146 { 1147 /* When current->bpf_ctx is set, the setsockopt is called from 1148 * a bpf prog. bpf has ensured the sk lock has been 1149 * acquired before calling setsockopt(). 1150 */ 1151 if (has_current_bpf_ctx()) 1152 return; 1153 1154 lock_sock(sk); 1155 } 1156 EXPORT_SYMBOL(sockopt_lock_sock); 1157 1158 void sockopt_release_sock(struct sock *sk) 1159 { 1160 if (has_current_bpf_ctx()) 1161 return; 1162 1163 release_sock(sk); 1164 } 1165 EXPORT_SYMBOL(sockopt_release_sock); 1166 1167 bool sockopt_ns_capable(struct user_namespace *ns, int cap) 1168 { 1169 return has_current_bpf_ctx() || ns_capable(ns, cap); 1170 } 1171 EXPORT_SYMBOL(sockopt_ns_capable); 1172 1173 bool sockopt_capable(int cap) 1174 { 1175 return has_current_bpf_ctx() || capable(cap); 1176 } 1177 EXPORT_SYMBOL(sockopt_capable); 1178 1179 static int sockopt_validate_clockid(__kernel_clockid_t value) 1180 { 1181 switch (value) { 1182 case CLOCK_REALTIME: 1183 case CLOCK_MONOTONIC: 1184 case CLOCK_TAI: 1185 return 0; 1186 } 1187 return -EINVAL; 1188 } 1189 1190 /* 1191 * This is meant for all protocols to use and covers goings on 1192 * at the socket level. Everything here is generic. 1193 */ 1194 1195 int sk_setsockopt(struct sock *sk, int level, int optname, 1196 sockptr_t optval, unsigned int optlen) 1197 { 1198 struct so_timestamping timestamping; 1199 struct socket *sock = sk->sk_socket; 1200 struct sock_txtime sk_txtime; 1201 int val; 1202 int valbool; 1203 struct linger ling; 1204 int ret = 0; 1205 1206 /* 1207 * Options without arguments 1208 */ 1209 1210 if (optname == SO_BINDTODEVICE) 1211 return sock_setbindtodevice(sk, optval, optlen); 1212 1213 if (optlen < sizeof(int)) 1214 return -EINVAL; 1215 1216 if (copy_from_sockptr(&val, optval, sizeof(val))) 1217 return -EFAULT; 1218 1219 valbool = val ? 1 : 0; 1220 1221 /* handle options which do not require locking the socket. */ 1222 switch (optname) { 1223 case SO_PRIORITY: 1224 if (sk_set_prio_allowed(sk, val)) { 1225 sock_set_priority(sk, val); 1226 return 0; 1227 } 1228 return -EPERM; 1229 case SO_TYPE: 1230 case SO_PROTOCOL: 1231 case SO_DOMAIN: 1232 case SO_ERROR: 1233 return -ENOPROTOOPT; 1234 #ifdef CONFIG_NET_RX_BUSY_POLL 1235 case SO_BUSY_POLL: 1236 if (val < 0) 1237 return -EINVAL; 1238 WRITE_ONCE(sk->sk_ll_usec, val); 1239 return 0; 1240 case SO_PREFER_BUSY_POLL: 1241 if (valbool && !sockopt_capable(CAP_NET_ADMIN)) 1242 return -EPERM; 1243 WRITE_ONCE(sk->sk_prefer_busy_poll, valbool); 1244 return 0; 1245 case SO_BUSY_POLL_BUDGET: 1246 if (val > READ_ONCE(sk->sk_busy_poll_budget) && 1247 !sockopt_capable(CAP_NET_ADMIN)) 1248 return -EPERM; 1249 if (val < 0 || val > U16_MAX) 1250 return -EINVAL; 1251 WRITE_ONCE(sk->sk_busy_poll_budget, val); 1252 return 0; 1253 #endif 1254 case SO_MAX_PACING_RATE: 1255 { 1256 unsigned long ulval = (val == ~0U) ? ~0UL : (unsigned int)val; 1257 unsigned long pacing_rate; 1258 1259 if (sizeof(ulval) != sizeof(val) && 1260 optlen >= sizeof(ulval) && 1261 copy_from_sockptr(&ulval, optval, sizeof(ulval))) { 1262 return -EFAULT; 1263 } 1264 if (ulval != ~0UL) 1265 cmpxchg(&sk->sk_pacing_status, 1266 SK_PACING_NONE, 1267 SK_PACING_NEEDED); 1268 /* Pairs with READ_ONCE() from sk_getsockopt() */ 1269 WRITE_ONCE(sk->sk_max_pacing_rate, ulval); 1270 pacing_rate = READ_ONCE(sk->sk_pacing_rate); 1271 if (ulval < pacing_rate) 1272 WRITE_ONCE(sk->sk_pacing_rate, ulval); 1273 return 0; 1274 } 1275 case SO_TXREHASH: 1276 if (!sk_is_tcp(sk)) 1277 return -EOPNOTSUPP; 1278 if (val < -1 || val > 1) 1279 return -EINVAL; 1280 if ((u8)val == SOCK_TXREHASH_DEFAULT) 1281 val = READ_ONCE(sock_net(sk)->core.sysctl_txrehash); 1282 /* Paired with READ_ONCE() in tcp_rtx_synack() 1283 * and sk_getsockopt(). 1284 */ 1285 WRITE_ONCE(sk->sk_txrehash, (u8)val); 1286 return 0; 1287 case SO_PEEK_OFF: 1288 { 1289 int (*set_peek_off)(struct sock *sk, int val); 1290 1291 set_peek_off = READ_ONCE(sock->ops)->set_peek_off; 1292 if (set_peek_off) 1293 ret = set_peek_off(sk, val); 1294 else 1295 ret = -EOPNOTSUPP; 1296 return ret; 1297 } 1298 #ifdef CONFIG_PAGE_POOL 1299 case SO_DEVMEM_DONTNEED: 1300 return sock_devmem_dontneed(sk, optval, optlen); 1301 #endif 1302 case SO_SNDTIMEO_OLD: 1303 case SO_SNDTIMEO_NEW: 1304 return sock_set_timeout(&sk->sk_sndtimeo, optval, 1305 optlen, optname == SO_SNDTIMEO_OLD); 1306 case SO_RCVTIMEO_OLD: 1307 case SO_RCVTIMEO_NEW: 1308 return sock_set_timeout(&sk->sk_rcvtimeo, optval, 1309 optlen, optname == SO_RCVTIMEO_OLD); 1310 } 1311 1312 sockopt_lock_sock(sk); 1313 1314 switch (optname) { 1315 case SO_DEBUG: 1316 if (val && !sockopt_capable(CAP_NET_ADMIN)) 1317 ret = -EACCES; 1318 else 1319 sock_valbool_flag(sk, SOCK_DBG, valbool); 1320 break; 1321 case SO_REUSEADDR: 1322 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE); 1323 break; 1324 case SO_REUSEPORT: 1325 if (valbool && !sk_is_inet(sk)) 1326 ret = -EOPNOTSUPP; 1327 else 1328 sk->sk_reuseport = valbool; 1329 break; 1330 case SO_DONTROUTE: 1331 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool); 1332 sk_dst_reset(sk); 1333 break; 1334 case SO_BROADCAST: 1335 sock_valbool_flag(sk, SOCK_BROADCAST, valbool); 1336 break; 1337 case SO_SNDBUF: 1338 /* Don't error on this BSD doesn't and if you think 1339 * about it this is right. Otherwise apps have to 1340 * play 'guess the biggest size' games. RCVBUF/SNDBUF 1341 * are treated in BSD as hints 1342 */ 1343 val = min_t(u32, val, READ_ONCE(sysctl_wmem_max)); 1344 set_sndbuf: 1345 /* Ensure val * 2 fits into an int, to prevent max_t() 1346 * from treating it as a negative value. 1347 */ 1348 val = min_t(int, val, INT_MAX / 2); 1349 sk->sk_userlocks |= SOCK_SNDBUF_LOCK; 1350 WRITE_ONCE(sk->sk_sndbuf, 1351 max_t(int, val * 2, SOCK_MIN_SNDBUF)); 1352 /* Wake up sending tasks if we upped the value. */ 1353 sk->sk_write_space(sk); 1354 break; 1355 1356 case SO_SNDBUFFORCE: 1357 if (!sockopt_capable(CAP_NET_ADMIN)) { 1358 ret = -EPERM; 1359 break; 1360 } 1361 1362 /* No negative values (to prevent underflow, as val will be 1363 * multiplied by 2). 1364 */ 1365 if (val < 0) 1366 val = 0; 1367 goto set_sndbuf; 1368 1369 case SO_RCVBUF: 1370 /* Don't error on this BSD doesn't and if you think 1371 * about it this is right. Otherwise apps have to 1372 * play 'guess the biggest size' games. RCVBUF/SNDBUF 1373 * are treated in BSD as hints 1374 */ 1375 __sock_set_rcvbuf(sk, min_t(u32, val, READ_ONCE(sysctl_rmem_max))); 1376 break; 1377 1378 case SO_RCVBUFFORCE: 1379 if (!sockopt_capable(CAP_NET_ADMIN)) { 1380 ret = -EPERM; 1381 break; 1382 } 1383 1384 /* No negative values (to prevent underflow, as val will be 1385 * multiplied by 2). 1386 */ 1387 __sock_set_rcvbuf(sk, max(val, 0)); 1388 break; 1389 1390 case SO_KEEPALIVE: 1391 if (sk->sk_prot->keepalive) 1392 sk->sk_prot->keepalive(sk, valbool); 1393 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool); 1394 break; 1395 1396 case SO_OOBINLINE: 1397 sock_valbool_flag(sk, SOCK_URGINLINE, valbool); 1398 break; 1399 1400 case SO_NO_CHECK: 1401 sk->sk_no_check_tx = valbool; 1402 break; 1403 1404 case SO_LINGER: 1405 if (optlen < sizeof(ling)) { 1406 ret = -EINVAL; /* 1003.1g */ 1407 break; 1408 } 1409 if (copy_from_sockptr(&ling, optval, sizeof(ling))) { 1410 ret = -EFAULT; 1411 break; 1412 } 1413 if (!ling.l_onoff) { 1414 sock_reset_flag(sk, SOCK_LINGER); 1415 } else { 1416 unsigned long t_sec = ling.l_linger; 1417 1418 if (t_sec >= MAX_SCHEDULE_TIMEOUT / HZ) 1419 WRITE_ONCE(sk->sk_lingertime, MAX_SCHEDULE_TIMEOUT); 1420 else 1421 WRITE_ONCE(sk->sk_lingertime, t_sec * HZ); 1422 sock_set_flag(sk, SOCK_LINGER); 1423 } 1424 break; 1425 1426 case SO_BSDCOMPAT: 1427 break; 1428 1429 case SO_TIMESTAMP_OLD: 1430 case SO_TIMESTAMP_NEW: 1431 case SO_TIMESTAMPNS_OLD: 1432 case SO_TIMESTAMPNS_NEW: 1433 sock_set_timestamp(sk, optname, valbool); 1434 break; 1435 1436 case SO_TIMESTAMPING_NEW: 1437 case SO_TIMESTAMPING_OLD: 1438 if (optlen == sizeof(timestamping)) { 1439 if (copy_from_sockptr(×tamping, optval, 1440 sizeof(timestamping))) { 1441 ret = -EFAULT; 1442 break; 1443 } 1444 } else { 1445 memset(×tamping, 0, sizeof(timestamping)); 1446 timestamping.flags = val; 1447 } 1448 ret = sock_set_timestamping(sk, optname, timestamping); 1449 break; 1450 1451 case SO_RCVLOWAT: 1452 { 1453 int (*set_rcvlowat)(struct sock *sk, int val) = NULL; 1454 1455 if (val < 0) 1456 val = INT_MAX; 1457 if (sock) 1458 set_rcvlowat = READ_ONCE(sock->ops)->set_rcvlowat; 1459 if (set_rcvlowat) 1460 ret = set_rcvlowat(sk, val); 1461 else 1462 WRITE_ONCE(sk->sk_rcvlowat, val ? : 1); 1463 break; 1464 } 1465 case SO_ATTACH_FILTER: { 1466 struct sock_fprog fprog; 1467 1468 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen); 1469 if (!ret) 1470 ret = sk_attach_filter(&fprog, sk); 1471 break; 1472 } 1473 case SO_ATTACH_BPF: 1474 ret = -EINVAL; 1475 if (optlen == sizeof(u32)) { 1476 u32 ufd; 1477 1478 ret = -EFAULT; 1479 if (copy_from_sockptr(&ufd, optval, sizeof(ufd))) 1480 break; 1481 1482 ret = sk_attach_bpf(ufd, sk); 1483 } 1484 break; 1485 1486 case SO_ATTACH_REUSEPORT_CBPF: { 1487 struct sock_fprog fprog; 1488 1489 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen); 1490 if (!ret) 1491 ret = sk_reuseport_attach_filter(&fprog, sk); 1492 break; 1493 } 1494 case SO_ATTACH_REUSEPORT_EBPF: 1495 ret = -EINVAL; 1496 if (optlen == sizeof(u32)) { 1497 u32 ufd; 1498 1499 ret = -EFAULT; 1500 if (copy_from_sockptr(&ufd, optval, sizeof(ufd))) 1501 break; 1502 1503 ret = sk_reuseport_attach_bpf(ufd, sk); 1504 } 1505 break; 1506 1507 case SO_DETACH_REUSEPORT_BPF: 1508 ret = reuseport_detach_prog(sk); 1509 break; 1510 1511 case SO_DETACH_FILTER: 1512 ret = sk_detach_filter(sk); 1513 break; 1514 1515 case SO_LOCK_FILTER: 1516 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool) 1517 ret = -EPERM; 1518 else 1519 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool); 1520 break; 1521 1522 case SO_MARK: 1523 if (!sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && 1524 !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1525 ret = -EPERM; 1526 break; 1527 } 1528 1529 __sock_set_mark(sk, val); 1530 break; 1531 case SO_RCVMARK: 1532 sock_valbool_flag(sk, SOCK_RCVMARK, valbool); 1533 break; 1534 1535 case SO_RCVPRIORITY: 1536 sock_valbool_flag(sk, SOCK_RCVPRIORITY, valbool); 1537 break; 1538 1539 case SO_RXQ_OVFL: 1540 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool); 1541 break; 1542 1543 case SO_WIFI_STATUS: 1544 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool); 1545 break; 1546 1547 case SO_NOFCS: 1548 sock_valbool_flag(sk, SOCK_NOFCS, valbool); 1549 break; 1550 1551 case SO_SELECT_ERR_QUEUE: 1552 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool); 1553 break; 1554 1555 case SO_PASSCRED: 1556 if (sk_may_scm_recv(sk)) 1557 sk->sk_scm_credentials = valbool; 1558 else 1559 ret = -EOPNOTSUPP; 1560 break; 1561 1562 case SO_PASSSEC: 1563 if (IS_ENABLED(CONFIG_SECURITY_NETWORK) && sk_may_scm_recv(sk)) 1564 sk->sk_scm_security = valbool; 1565 else 1566 ret = -EOPNOTSUPP; 1567 break; 1568 1569 case SO_PASSPIDFD: 1570 if (sk_is_unix(sk)) 1571 sk->sk_scm_pidfd = valbool; 1572 else 1573 ret = -EOPNOTSUPP; 1574 break; 1575 1576 case SO_PASSRIGHTS: 1577 if (sk_is_unix(sk)) 1578 sk->sk_scm_rights = valbool; 1579 else 1580 ret = -EOPNOTSUPP; 1581 break; 1582 1583 case SO_INCOMING_CPU: 1584 reuseport_update_incoming_cpu(sk, val); 1585 break; 1586 1587 case SO_CNX_ADVICE: 1588 if (val == 1) 1589 dst_negative_advice(sk); 1590 break; 1591 1592 case SO_ZEROCOPY: 1593 if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) { 1594 if (!(sk_is_tcp(sk) || 1595 (sk->sk_type == SOCK_DGRAM && 1596 sk->sk_protocol == IPPROTO_UDP))) 1597 ret = -EOPNOTSUPP; 1598 } else if (sk->sk_family != PF_RDS) { 1599 ret = -EOPNOTSUPP; 1600 } 1601 if (!ret) { 1602 if (val < 0 || val > 1) 1603 ret = -EINVAL; 1604 else 1605 sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool); 1606 } 1607 break; 1608 1609 case SO_TXTIME: 1610 if (optlen != sizeof(struct sock_txtime)) { 1611 ret = -EINVAL; 1612 break; 1613 } else if (copy_from_sockptr(&sk_txtime, optval, 1614 sizeof(struct sock_txtime))) { 1615 ret = -EFAULT; 1616 break; 1617 } else if (sk_txtime.flags & ~SOF_TXTIME_FLAGS_MASK) { 1618 ret = -EINVAL; 1619 break; 1620 } 1621 /* CLOCK_MONOTONIC is only used by sch_fq, and this packet 1622 * scheduler has enough safe guards. 1623 */ 1624 if (sk_txtime.clockid != CLOCK_MONOTONIC && 1625 !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1626 ret = -EPERM; 1627 break; 1628 } 1629 1630 ret = sockopt_validate_clockid(sk_txtime.clockid); 1631 if (ret) 1632 break; 1633 1634 sock_valbool_flag(sk, SOCK_TXTIME, true); 1635 sk->sk_clockid = sk_txtime.clockid; 1636 sk->sk_txtime_deadline_mode = 1637 !!(sk_txtime.flags & SOF_TXTIME_DEADLINE_MODE); 1638 sk->sk_txtime_report_errors = 1639 !!(sk_txtime.flags & SOF_TXTIME_REPORT_ERRORS); 1640 break; 1641 1642 case SO_BINDTOIFINDEX: 1643 ret = sock_bindtoindex_locked(sk, val); 1644 break; 1645 1646 case SO_BUF_LOCK: 1647 if (val & ~SOCK_BUF_LOCK_MASK) { 1648 ret = -EINVAL; 1649 break; 1650 } 1651 sk->sk_userlocks = val | (sk->sk_userlocks & 1652 ~SOCK_BUF_LOCK_MASK); 1653 break; 1654 1655 case SO_RESERVE_MEM: 1656 { 1657 int delta; 1658 1659 if (val < 0) { 1660 ret = -EINVAL; 1661 break; 1662 } 1663 1664 delta = val - sk->sk_reserved_mem; 1665 if (delta < 0) 1666 sock_release_reserved_memory(sk, -delta); 1667 else 1668 ret = sock_reserve_memory(sk, delta); 1669 break; 1670 } 1671 1672 default: 1673 ret = -ENOPROTOOPT; 1674 break; 1675 } 1676 sockopt_release_sock(sk); 1677 return ret; 1678 } 1679 1680 int sock_setsockopt(struct socket *sock, int level, int optname, 1681 sockptr_t optval, unsigned int optlen) 1682 { 1683 return sk_setsockopt(sock->sk, level, optname, 1684 optval, optlen); 1685 } 1686 EXPORT_SYMBOL(sock_setsockopt); 1687 1688 static const struct cred *sk_get_peer_cred(struct sock *sk) 1689 { 1690 const struct cred *cred; 1691 1692 spin_lock(&sk->sk_peer_lock); 1693 cred = get_cred(sk->sk_peer_cred); 1694 spin_unlock(&sk->sk_peer_lock); 1695 1696 return cred; 1697 } 1698 1699 static void cred_to_ucred(struct pid *pid, const struct cred *cred, 1700 struct ucred *ucred) 1701 { 1702 ucred->pid = pid_vnr(pid); 1703 ucred->uid = ucred->gid = -1; 1704 if (cred) { 1705 struct user_namespace *current_ns = current_user_ns(); 1706 1707 ucred->uid = from_kuid_munged(current_ns, cred->euid); 1708 ucred->gid = from_kgid_munged(current_ns, cred->egid); 1709 } 1710 } 1711 1712 static int groups_to_user(sockptr_t dst, const struct group_info *src) 1713 { 1714 struct user_namespace *user_ns = current_user_ns(); 1715 int i; 1716 1717 for (i = 0; i < src->ngroups; i++) { 1718 gid_t gid = from_kgid_munged(user_ns, src->gid[i]); 1719 1720 if (copy_to_sockptr_offset(dst, i * sizeof(gid), &gid, sizeof(gid))) 1721 return -EFAULT; 1722 } 1723 1724 return 0; 1725 } 1726 1727 int sk_getsockopt(struct sock *sk, int level, int optname, 1728 sockptr_t optval, sockptr_t optlen) 1729 { 1730 struct socket *sock = sk->sk_socket; 1731 1732 union { 1733 int val; 1734 u64 val64; 1735 unsigned long ulval; 1736 struct linger ling; 1737 struct old_timeval32 tm32; 1738 struct __kernel_old_timeval tm; 1739 struct __kernel_sock_timeval stm; 1740 struct sock_txtime txtime; 1741 struct so_timestamping timestamping; 1742 } v; 1743 1744 int lv = sizeof(int); 1745 int len; 1746 1747 if (copy_from_sockptr(&len, optlen, sizeof(int))) 1748 return -EFAULT; 1749 if (len < 0) 1750 return -EINVAL; 1751 1752 memset(&v, 0, sizeof(v)); 1753 1754 switch (optname) { 1755 case SO_DEBUG: 1756 v.val = sock_flag(sk, SOCK_DBG); 1757 break; 1758 1759 case SO_DONTROUTE: 1760 v.val = sock_flag(sk, SOCK_LOCALROUTE); 1761 break; 1762 1763 case SO_BROADCAST: 1764 v.val = sock_flag(sk, SOCK_BROADCAST); 1765 break; 1766 1767 case SO_SNDBUF: 1768 v.val = READ_ONCE(sk->sk_sndbuf); 1769 break; 1770 1771 case SO_RCVBUF: 1772 v.val = READ_ONCE(sk->sk_rcvbuf); 1773 break; 1774 1775 case SO_REUSEADDR: 1776 v.val = sk->sk_reuse; 1777 break; 1778 1779 case SO_REUSEPORT: 1780 v.val = sk->sk_reuseport; 1781 break; 1782 1783 case SO_KEEPALIVE: 1784 v.val = sock_flag(sk, SOCK_KEEPOPEN); 1785 break; 1786 1787 case SO_TYPE: 1788 v.val = sk->sk_type; 1789 break; 1790 1791 case SO_PROTOCOL: 1792 v.val = sk->sk_protocol; 1793 break; 1794 1795 case SO_DOMAIN: 1796 v.val = sk->sk_family; 1797 break; 1798 1799 case SO_ERROR: 1800 v.val = -sock_error(sk); 1801 if (v.val == 0) 1802 v.val = xchg(&sk->sk_err_soft, 0); 1803 break; 1804 1805 case SO_OOBINLINE: 1806 v.val = sock_flag(sk, SOCK_URGINLINE); 1807 break; 1808 1809 case SO_NO_CHECK: 1810 v.val = sk->sk_no_check_tx; 1811 break; 1812 1813 case SO_PRIORITY: 1814 v.val = READ_ONCE(sk->sk_priority); 1815 break; 1816 1817 case SO_LINGER: 1818 lv = sizeof(v.ling); 1819 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER); 1820 v.ling.l_linger = READ_ONCE(sk->sk_lingertime) / HZ; 1821 break; 1822 1823 case SO_BSDCOMPAT: 1824 break; 1825 1826 case SO_TIMESTAMP_OLD: 1827 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && 1828 !sock_flag(sk, SOCK_TSTAMP_NEW) && 1829 !sock_flag(sk, SOCK_RCVTSTAMPNS); 1830 break; 1831 1832 case SO_TIMESTAMPNS_OLD: 1833 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && !sock_flag(sk, SOCK_TSTAMP_NEW); 1834 break; 1835 1836 case SO_TIMESTAMP_NEW: 1837 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && sock_flag(sk, SOCK_TSTAMP_NEW); 1838 break; 1839 1840 case SO_TIMESTAMPNS_NEW: 1841 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && sock_flag(sk, SOCK_TSTAMP_NEW); 1842 break; 1843 1844 case SO_TIMESTAMPING_OLD: 1845 case SO_TIMESTAMPING_NEW: 1846 lv = sizeof(v.timestamping); 1847 /* For the later-added case SO_TIMESTAMPING_NEW: Be strict about only 1848 * returning the flags when they were set through the same option. 1849 * Don't change the beviour for the old case SO_TIMESTAMPING_OLD. 1850 */ 1851 if (optname == SO_TIMESTAMPING_OLD || sock_flag(sk, SOCK_TSTAMP_NEW)) { 1852 v.timestamping.flags = READ_ONCE(sk->sk_tsflags); 1853 v.timestamping.bind_phc = READ_ONCE(sk->sk_bind_phc); 1854 } 1855 break; 1856 1857 case SO_RCVTIMEO_OLD: 1858 case SO_RCVTIMEO_NEW: 1859 lv = sock_get_timeout(READ_ONCE(sk->sk_rcvtimeo), &v, 1860 SO_RCVTIMEO_OLD == optname); 1861 break; 1862 1863 case SO_SNDTIMEO_OLD: 1864 case SO_SNDTIMEO_NEW: 1865 lv = sock_get_timeout(READ_ONCE(sk->sk_sndtimeo), &v, 1866 SO_SNDTIMEO_OLD == optname); 1867 break; 1868 1869 case SO_RCVLOWAT: 1870 v.val = READ_ONCE(sk->sk_rcvlowat); 1871 break; 1872 1873 case SO_SNDLOWAT: 1874 v.val = 1; 1875 break; 1876 1877 case SO_PASSCRED: 1878 if (!sk_may_scm_recv(sk)) 1879 return -EOPNOTSUPP; 1880 1881 v.val = sk->sk_scm_credentials; 1882 break; 1883 1884 case SO_PASSPIDFD: 1885 if (!sk_is_unix(sk)) 1886 return -EOPNOTSUPP; 1887 1888 v.val = sk->sk_scm_pidfd; 1889 break; 1890 1891 case SO_PASSRIGHTS: 1892 if (!sk_is_unix(sk)) 1893 return -EOPNOTSUPP; 1894 1895 v.val = sk->sk_scm_rights; 1896 break; 1897 1898 case SO_PEERCRED: 1899 { 1900 struct ucred peercred; 1901 if (len > sizeof(peercred)) 1902 len = sizeof(peercred); 1903 1904 spin_lock(&sk->sk_peer_lock); 1905 cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred); 1906 spin_unlock(&sk->sk_peer_lock); 1907 1908 if (copy_to_sockptr(optval, &peercred, len)) 1909 return -EFAULT; 1910 goto lenout; 1911 } 1912 1913 case SO_PEERPIDFD: 1914 { 1915 struct pid *peer_pid; 1916 struct file *pidfd_file = NULL; 1917 unsigned int flags = 0; 1918 int pidfd; 1919 1920 if (len > sizeof(pidfd)) 1921 len = sizeof(pidfd); 1922 1923 spin_lock(&sk->sk_peer_lock); 1924 peer_pid = get_pid(sk->sk_peer_pid); 1925 spin_unlock(&sk->sk_peer_lock); 1926 1927 if (!peer_pid) 1928 return -ENODATA; 1929 1930 /* The use of PIDFD_STALE requires stashing of struct pid 1931 * on pidfs with pidfs_register_pid() and only AF_UNIX 1932 * were prepared for this. 1933 */ 1934 if (sk->sk_family == AF_UNIX) 1935 flags = PIDFD_STALE; 1936 1937 pidfd = pidfd_prepare(peer_pid, flags, &pidfd_file); 1938 put_pid(peer_pid); 1939 if (pidfd < 0) 1940 return pidfd; 1941 1942 if (copy_to_sockptr(optval, &pidfd, len) || 1943 copy_to_sockptr(optlen, &len, sizeof(int))) { 1944 put_unused_fd(pidfd); 1945 fput(pidfd_file); 1946 1947 return -EFAULT; 1948 } 1949 1950 fd_install(pidfd, pidfd_file); 1951 return 0; 1952 } 1953 1954 case SO_PEERGROUPS: 1955 { 1956 const struct cred *cred; 1957 int ret, n; 1958 1959 cred = sk_get_peer_cred(sk); 1960 if (!cred) 1961 return -ENODATA; 1962 1963 n = cred->group_info->ngroups; 1964 if (len < n * sizeof(gid_t)) { 1965 len = n * sizeof(gid_t); 1966 put_cred(cred); 1967 return copy_to_sockptr(optlen, &len, sizeof(int)) ? -EFAULT : -ERANGE; 1968 } 1969 len = n * sizeof(gid_t); 1970 1971 ret = groups_to_user(optval, cred->group_info); 1972 put_cred(cred); 1973 if (ret) 1974 return ret; 1975 goto lenout; 1976 } 1977 1978 case SO_PEERNAME: 1979 { 1980 struct sockaddr_storage address; 1981 1982 lv = READ_ONCE(sock->ops)->getname(sock, (struct sockaddr *)&address, 2); 1983 if (lv < 0) 1984 return -ENOTCONN; 1985 if (lv < len) 1986 return -EINVAL; 1987 if (copy_to_sockptr(optval, &address, len)) 1988 return -EFAULT; 1989 goto lenout; 1990 } 1991 1992 /* Dubious BSD thing... Probably nobody even uses it, but 1993 * the UNIX standard wants it for whatever reason... -DaveM 1994 */ 1995 case SO_ACCEPTCONN: 1996 v.val = sk->sk_state == TCP_LISTEN; 1997 break; 1998 1999 case SO_PASSSEC: 2000 if (!IS_ENABLED(CONFIG_SECURITY_NETWORK) || !sk_may_scm_recv(sk)) 2001 return -EOPNOTSUPP; 2002 2003 v.val = sk->sk_scm_security; 2004 break; 2005 2006 case SO_PEERSEC: 2007 return security_socket_getpeersec_stream(sock, 2008 optval, optlen, len); 2009 2010 case SO_MARK: 2011 v.val = READ_ONCE(sk->sk_mark); 2012 break; 2013 2014 case SO_RCVMARK: 2015 v.val = sock_flag(sk, SOCK_RCVMARK); 2016 break; 2017 2018 case SO_RCVPRIORITY: 2019 v.val = sock_flag(sk, SOCK_RCVPRIORITY); 2020 break; 2021 2022 case SO_RXQ_OVFL: 2023 v.val = sock_flag(sk, SOCK_RXQ_OVFL); 2024 break; 2025 2026 case SO_WIFI_STATUS: 2027 v.val = sock_flag(sk, SOCK_WIFI_STATUS); 2028 break; 2029 2030 case SO_PEEK_OFF: 2031 if (!READ_ONCE(sock->ops)->set_peek_off) 2032 return -EOPNOTSUPP; 2033 2034 v.val = READ_ONCE(sk->sk_peek_off); 2035 break; 2036 case SO_NOFCS: 2037 v.val = sock_flag(sk, SOCK_NOFCS); 2038 break; 2039 2040 case SO_BINDTODEVICE: 2041 return sock_getbindtodevice(sk, optval, optlen, len); 2042 2043 case SO_GET_FILTER: 2044 len = sk_get_filter(sk, optval, len); 2045 if (len < 0) 2046 return len; 2047 2048 goto lenout; 2049 2050 case SO_LOCK_FILTER: 2051 v.val = sock_flag(sk, SOCK_FILTER_LOCKED); 2052 break; 2053 2054 case SO_BPF_EXTENSIONS: 2055 v.val = bpf_tell_extensions(); 2056 break; 2057 2058 case SO_SELECT_ERR_QUEUE: 2059 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE); 2060 break; 2061 2062 #ifdef CONFIG_NET_RX_BUSY_POLL 2063 case SO_BUSY_POLL: 2064 v.val = READ_ONCE(sk->sk_ll_usec); 2065 break; 2066 case SO_PREFER_BUSY_POLL: 2067 v.val = READ_ONCE(sk->sk_prefer_busy_poll); 2068 break; 2069 #endif 2070 2071 case SO_MAX_PACING_RATE: 2072 /* The READ_ONCE() pair with the WRITE_ONCE() in sk_setsockopt() */ 2073 if (sizeof(v.ulval) != sizeof(v.val) && len >= sizeof(v.ulval)) { 2074 lv = sizeof(v.ulval); 2075 v.ulval = READ_ONCE(sk->sk_max_pacing_rate); 2076 } else { 2077 /* 32bit version */ 2078 v.val = min_t(unsigned long, ~0U, 2079 READ_ONCE(sk->sk_max_pacing_rate)); 2080 } 2081 break; 2082 2083 case SO_INCOMING_CPU: 2084 v.val = READ_ONCE(sk->sk_incoming_cpu); 2085 break; 2086 2087 case SO_MEMINFO: 2088 { 2089 u32 meminfo[SK_MEMINFO_VARS]; 2090 2091 sk_get_meminfo(sk, meminfo); 2092 2093 len = min_t(unsigned int, len, sizeof(meminfo)); 2094 if (copy_to_sockptr(optval, &meminfo, len)) 2095 return -EFAULT; 2096 2097 goto lenout; 2098 } 2099 2100 #ifdef CONFIG_NET_RX_BUSY_POLL 2101 case SO_INCOMING_NAPI_ID: 2102 v.val = READ_ONCE(sk->sk_napi_id); 2103 2104 /* aggregate non-NAPI IDs down to 0 */ 2105 if (!napi_id_valid(v.val)) 2106 v.val = 0; 2107 2108 break; 2109 #endif 2110 2111 case SO_COOKIE: 2112 lv = sizeof(u64); 2113 if (len < lv) 2114 return -EINVAL; 2115 v.val64 = sock_gen_cookie(sk); 2116 break; 2117 2118 case SO_ZEROCOPY: 2119 v.val = sock_flag(sk, SOCK_ZEROCOPY); 2120 break; 2121 2122 case SO_TXTIME: 2123 lv = sizeof(v.txtime); 2124 v.txtime.clockid = sk->sk_clockid; 2125 v.txtime.flags |= sk->sk_txtime_deadline_mode ? 2126 SOF_TXTIME_DEADLINE_MODE : 0; 2127 v.txtime.flags |= sk->sk_txtime_report_errors ? 2128 SOF_TXTIME_REPORT_ERRORS : 0; 2129 break; 2130 2131 case SO_BINDTOIFINDEX: 2132 v.val = READ_ONCE(sk->sk_bound_dev_if); 2133 break; 2134 2135 case SO_NETNS_COOKIE: 2136 lv = sizeof(u64); 2137 if (len != lv) 2138 return -EINVAL; 2139 v.val64 = sock_net(sk)->net_cookie; 2140 break; 2141 2142 case SO_BUF_LOCK: 2143 v.val = sk->sk_userlocks & SOCK_BUF_LOCK_MASK; 2144 break; 2145 2146 case SO_RESERVE_MEM: 2147 v.val = READ_ONCE(sk->sk_reserved_mem); 2148 break; 2149 2150 case SO_TXREHASH: 2151 if (!sk_is_tcp(sk)) 2152 return -EOPNOTSUPP; 2153 2154 /* Paired with WRITE_ONCE() in sk_setsockopt() */ 2155 v.val = READ_ONCE(sk->sk_txrehash); 2156 break; 2157 2158 default: 2159 /* We implement the SO_SNDLOWAT etc to not be settable 2160 * (1003.1g 7). 2161 */ 2162 return -ENOPROTOOPT; 2163 } 2164 2165 if (len > lv) 2166 len = lv; 2167 if (copy_to_sockptr(optval, &v, len)) 2168 return -EFAULT; 2169 lenout: 2170 if (copy_to_sockptr(optlen, &len, sizeof(int))) 2171 return -EFAULT; 2172 return 0; 2173 } 2174 2175 /* 2176 * Initialize an sk_lock. 2177 * 2178 * (We also register the sk_lock with the lock validator.) 2179 */ 2180 static inline void sock_lock_init(struct sock *sk) 2181 { 2182 sk_owner_clear(sk); 2183 2184 if (sk->sk_kern_sock) 2185 sock_lock_init_class_and_name( 2186 sk, 2187 af_family_kern_slock_key_strings[sk->sk_family], 2188 af_family_kern_slock_keys + sk->sk_family, 2189 af_family_kern_key_strings[sk->sk_family], 2190 af_family_kern_keys + sk->sk_family); 2191 else 2192 sock_lock_init_class_and_name( 2193 sk, 2194 af_family_slock_key_strings[sk->sk_family], 2195 af_family_slock_keys + sk->sk_family, 2196 af_family_key_strings[sk->sk_family], 2197 af_family_keys + sk->sk_family); 2198 } 2199 2200 /* 2201 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet, 2202 * even temporarily, because of RCU lookups. sk_node should also be left as is. 2203 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end 2204 */ 2205 static void sock_copy(struct sock *nsk, const struct sock *osk) 2206 { 2207 const struct proto *prot = READ_ONCE(osk->sk_prot); 2208 #ifdef CONFIG_SECURITY_NETWORK 2209 void *sptr = nsk->sk_security; 2210 #endif 2211 2212 /* If we move sk_tx_queue_mapping out of the private section, 2213 * we must check if sk_tx_queue_clear() is called after 2214 * sock_copy() in sk_clone_lock(). 2215 */ 2216 BUILD_BUG_ON(offsetof(struct sock, sk_tx_queue_mapping) < 2217 offsetof(struct sock, sk_dontcopy_begin) || 2218 offsetof(struct sock, sk_tx_queue_mapping) >= 2219 offsetof(struct sock, sk_dontcopy_end)); 2220 2221 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin)); 2222 2223 unsafe_memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end, 2224 prot->obj_size - offsetof(struct sock, sk_dontcopy_end), 2225 /* alloc is larger than struct, see sk_prot_alloc() */); 2226 2227 #ifdef CONFIG_SECURITY_NETWORK 2228 nsk->sk_security = sptr; 2229 security_sk_clone(osk, nsk); 2230 #endif 2231 } 2232 2233 static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority, 2234 int family) 2235 { 2236 struct sock *sk; 2237 struct kmem_cache *slab; 2238 2239 slab = prot->slab; 2240 if (slab != NULL) { 2241 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO); 2242 if (!sk) 2243 return sk; 2244 if (want_init_on_alloc(priority)) 2245 sk_prot_clear_nulls(sk, prot->obj_size); 2246 } else 2247 sk = kmalloc(prot->obj_size, priority); 2248 2249 if (sk != NULL) { 2250 if (security_sk_alloc(sk, family, priority)) 2251 goto out_free; 2252 2253 if (!try_module_get(prot->owner)) 2254 goto out_free_sec; 2255 } 2256 2257 return sk; 2258 2259 out_free_sec: 2260 security_sk_free(sk); 2261 out_free: 2262 if (slab != NULL) 2263 kmem_cache_free(slab, sk); 2264 else 2265 kfree(sk); 2266 return NULL; 2267 } 2268 2269 static void sk_prot_free(struct proto *prot, struct sock *sk) 2270 { 2271 struct kmem_cache *slab; 2272 struct module *owner; 2273 2274 owner = prot->owner; 2275 slab = prot->slab; 2276 2277 cgroup_sk_free(&sk->sk_cgrp_data); 2278 mem_cgroup_sk_free(sk); 2279 security_sk_free(sk); 2280 2281 sk_owner_put(sk); 2282 2283 if (slab != NULL) 2284 kmem_cache_free(slab, sk); 2285 else 2286 kfree(sk); 2287 module_put(owner); 2288 } 2289 2290 /** 2291 * sk_alloc - All socket objects are allocated here 2292 * @net: the applicable net namespace 2293 * @family: protocol family 2294 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 2295 * @prot: struct proto associated with this new sock instance 2296 * @kern: is this to be a kernel socket? 2297 */ 2298 struct sock *sk_alloc(struct net *net, int family, gfp_t priority, 2299 struct proto *prot, int kern) 2300 { 2301 struct sock *sk; 2302 2303 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family); 2304 if (sk) { 2305 sk->sk_family = family; 2306 /* 2307 * See comment in struct sock definition to understand 2308 * why we need sk_prot_creator -acme 2309 */ 2310 sk->sk_prot = sk->sk_prot_creator = prot; 2311 2312 if (READ_ONCE(net->core.sysctl_bypass_prot_mem)) 2313 sk->sk_bypass_prot_mem = 1; 2314 2315 sk->sk_kern_sock = kern; 2316 sock_lock_init(sk); 2317 2318 sk->sk_net_refcnt = kern ? 0 : 1; 2319 if (likely(sk->sk_net_refcnt)) { 2320 get_net_track(net, &sk->ns_tracker, priority); 2321 sock_inuse_add(net, 1); 2322 } else { 2323 net_passive_inc(net); 2324 __netns_tracker_alloc(net, &sk->ns_tracker, 2325 false, priority); 2326 } 2327 2328 sock_net_set(sk, net); 2329 refcount_set(&sk->sk_wmem_alloc, SK_WMEM_ALLOC_BIAS); 2330 2331 mem_cgroup_sk_alloc(sk); 2332 cgroup_sk_alloc(&sk->sk_cgrp_data); 2333 sock_update_classid(&sk->sk_cgrp_data); 2334 sock_update_netprioidx(&sk->sk_cgrp_data); 2335 sk_tx_queue_clear(sk); 2336 } 2337 2338 return sk; 2339 } 2340 EXPORT_SYMBOL(sk_alloc); 2341 2342 /* Sockets having SOCK_RCU_FREE will call this function after one RCU 2343 * grace period. This is the case for UDP sockets and TCP listeners. 2344 */ 2345 static void __sk_destruct(struct rcu_head *head) 2346 { 2347 struct sock *sk = container_of(head, struct sock, sk_rcu); 2348 struct net *net = sock_net(sk); 2349 struct sk_filter *filter; 2350 2351 if (sk->sk_destruct) 2352 sk->sk_destruct(sk); 2353 2354 filter = rcu_dereference_check(sk->sk_filter, 2355 refcount_read(&sk->sk_wmem_alloc) == 0); 2356 if (filter) { 2357 sk_filter_uncharge(sk, filter); 2358 RCU_INIT_POINTER(sk->sk_filter, NULL); 2359 } 2360 2361 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP); 2362 2363 #ifdef CONFIG_BPF_SYSCALL 2364 bpf_sk_storage_free(sk); 2365 #endif 2366 2367 if (atomic_read(&sk->sk_omem_alloc)) 2368 pr_debug("%s: optmem leakage (%d bytes) detected\n", 2369 __func__, atomic_read(&sk->sk_omem_alloc)); 2370 2371 if (sk->sk_frag.page) { 2372 put_page(sk->sk_frag.page); 2373 sk->sk_frag.page = NULL; 2374 } 2375 2376 /* We do not need to acquire sk->sk_peer_lock, we are the last user. */ 2377 put_cred(sk->sk_peer_cred); 2378 put_pid(sk->sk_peer_pid); 2379 2380 if (likely(sk->sk_net_refcnt)) { 2381 put_net_track(net, &sk->ns_tracker); 2382 } else { 2383 __netns_tracker_free(net, &sk->ns_tracker, false); 2384 net_passive_dec(net); 2385 } 2386 sk_prot_free(sk->sk_prot_creator, sk); 2387 } 2388 2389 void sk_net_refcnt_upgrade(struct sock *sk) 2390 { 2391 struct net *net = sock_net(sk); 2392 2393 WARN_ON_ONCE(sk->sk_net_refcnt); 2394 __netns_tracker_free(net, &sk->ns_tracker, false); 2395 net_passive_dec(net); 2396 sk->sk_net_refcnt = 1; 2397 get_net_track(net, &sk->ns_tracker, GFP_KERNEL); 2398 sock_inuse_add(net, 1); 2399 } 2400 EXPORT_SYMBOL_GPL(sk_net_refcnt_upgrade); 2401 2402 void sk_destruct(struct sock *sk) 2403 { 2404 bool use_call_rcu = sock_flag(sk, SOCK_RCU_FREE); 2405 2406 if (rcu_access_pointer(sk->sk_reuseport_cb)) { 2407 reuseport_detach_sock(sk); 2408 use_call_rcu = true; 2409 } 2410 2411 if (use_call_rcu) 2412 call_rcu(&sk->sk_rcu, __sk_destruct); 2413 else 2414 __sk_destruct(&sk->sk_rcu); 2415 } 2416 2417 static void __sk_free(struct sock *sk) 2418 { 2419 if (likely(sk->sk_net_refcnt)) 2420 sock_inuse_add(sock_net(sk), -1); 2421 2422 if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk))) 2423 sock_diag_broadcast_destroy(sk); 2424 else 2425 sk_destruct(sk); 2426 } 2427 2428 void sk_free(struct sock *sk) 2429 { 2430 /* 2431 * We subtract one from sk_wmem_alloc and can know if 2432 * some packets are still in some tx queue. 2433 * If not null, sock_wfree() will call __sk_free(sk) later 2434 */ 2435 if (refcount_dec_and_test(&sk->sk_wmem_alloc)) 2436 __sk_free(sk); 2437 } 2438 EXPORT_SYMBOL(sk_free); 2439 2440 static void sk_init_common(struct sock *sk) 2441 { 2442 skb_queue_head_init(&sk->sk_receive_queue); 2443 skb_queue_head_init(&sk->sk_write_queue); 2444 skb_queue_head_init(&sk->sk_error_queue); 2445 2446 rwlock_init(&sk->sk_callback_lock); 2447 lockdep_set_class_and_name(&sk->sk_receive_queue.lock, 2448 af_rlock_keys + sk->sk_family, 2449 af_family_rlock_key_strings[sk->sk_family]); 2450 lockdep_set_class_and_name(&sk->sk_write_queue.lock, 2451 af_wlock_keys + sk->sk_family, 2452 af_family_wlock_key_strings[sk->sk_family]); 2453 lockdep_set_class_and_name(&sk->sk_error_queue.lock, 2454 af_elock_keys + sk->sk_family, 2455 af_family_elock_key_strings[sk->sk_family]); 2456 if (sk->sk_kern_sock) 2457 lockdep_set_class_and_name(&sk->sk_callback_lock, 2458 af_kern_callback_keys + sk->sk_family, 2459 af_family_kern_clock_key_strings[sk->sk_family]); 2460 else 2461 lockdep_set_class_and_name(&sk->sk_callback_lock, 2462 af_callback_keys + sk->sk_family, 2463 af_family_clock_key_strings[sk->sk_family]); 2464 } 2465 2466 /** 2467 * sk_clone - clone a socket 2468 * @sk: the socket to clone 2469 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 2470 * @lock: if true, lock the cloned sk 2471 * 2472 * If @lock is true, the clone is locked by bh_lock_sock(), and 2473 * caller must unlock socket even in error path by bh_unlock_sock(). 2474 */ 2475 struct sock *sk_clone(const struct sock *sk, const gfp_t priority, 2476 bool lock) 2477 { 2478 struct proto *prot = READ_ONCE(sk->sk_prot); 2479 struct sk_filter *filter; 2480 bool is_charged = true; 2481 struct sock *newsk; 2482 2483 newsk = sk_prot_alloc(prot, priority, sk->sk_family); 2484 if (!newsk) 2485 goto out; 2486 2487 sock_copy(newsk, sk); 2488 2489 newsk->sk_prot_creator = prot; 2490 2491 /* SANITY */ 2492 if (likely(newsk->sk_net_refcnt)) { 2493 get_net_track(sock_net(newsk), &newsk->ns_tracker, priority); 2494 sock_inuse_add(sock_net(newsk), 1); 2495 } else { 2496 /* Kernel sockets are not elevating the struct net refcount. 2497 * Instead, use a tracker to more easily detect if a layer 2498 * is not properly dismantling its kernel sockets at netns 2499 * destroy time. 2500 */ 2501 net_passive_inc(sock_net(newsk)); 2502 __netns_tracker_alloc(sock_net(newsk), &newsk->ns_tracker, 2503 false, priority); 2504 } 2505 2506 sk_node_init(&newsk->sk_node); 2507 sock_lock_init(newsk); 2508 2509 if (lock) 2510 bh_lock_sock(newsk); 2511 2512 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL; 2513 newsk->sk_backlog.len = 0; 2514 2515 atomic_set(&newsk->sk_rmem_alloc, 0); 2516 2517 refcount_set(&newsk->sk_wmem_alloc, SK_WMEM_ALLOC_BIAS); 2518 2519 atomic_set(&newsk->sk_omem_alloc, 0); 2520 sk_init_common(newsk); 2521 2522 newsk->sk_dst_cache = NULL; 2523 newsk->sk_dst_pending_confirm = 0; 2524 newsk->sk_wmem_queued = 0; 2525 newsk->sk_forward_alloc = 0; 2526 newsk->sk_reserved_mem = 0; 2527 DEBUG_NET_WARN_ON_ONCE(newsk->sk_drop_counters); 2528 sk_drops_reset(newsk); 2529 newsk->sk_send_head = NULL; 2530 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK; 2531 atomic_set(&newsk->sk_zckey, 0); 2532 2533 sock_reset_flag(newsk, SOCK_DONE); 2534 2535 #ifdef CONFIG_MEMCG 2536 /* sk->sk_memcg will be populated at accept() time */ 2537 newsk->sk_memcg = NULL; 2538 #endif 2539 2540 cgroup_sk_clone(&newsk->sk_cgrp_data); 2541 2542 rcu_read_lock(); 2543 filter = rcu_dereference(sk->sk_filter); 2544 if (filter != NULL) 2545 /* though it's an empty new sock, the charging may fail 2546 * if sysctl_optmem_max was changed between creation of 2547 * original socket and cloning 2548 */ 2549 is_charged = sk_filter_charge(newsk, filter); 2550 RCU_INIT_POINTER(newsk->sk_filter, filter); 2551 rcu_read_unlock(); 2552 2553 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) { 2554 /* We need to make sure that we don't uncharge the new 2555 * socket if we couldn't charge it in the first place 2556 * as otherwise we uncharge the parent's filter. 2557 */ 2558 if (!is_charged) 2559 RCU_INIT_POINTER(newsk->sk_filter, NULL); 2560 2561 goto free; 2562 } 2563 2564 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL); 2565 2566 if (bpf_sk_storage_clone(sk, newsk)) 2567 goto free; 2568 2569 /* Clear sk_user_data if parent had the pointer tagged 2570 * as not suitable for copying when cloning. 2571 */ 2572 if (sk_user_data_is_nocopy(newsk)) 2573 newsk->sk_user_data = NULL; 2574 2575 newsk->sk_err = 0; 2576 newsk->sk_err_soft = 0; 2577 newsk->sk_priority = 0; 2578 newsk->sk_incoming_cpu = raw_smp_processor_id(); 2579 2580 /* Before updating sk_refcnt, we must commit prior changes to memory 2581 * (Documentation/RCU/rculist_nulls.rst for details) 2582 */ 2583 smp_wmb(); 2584 refcount_set(&newsk->sk_refcnt, 2); 2585 2586 sk_set_socket(newsk, NULL); 2587 sk_tx_queue_clear(newsk); 2588 sk_rx_queue_clear(newsk); 2589 RCU_INIT_POINTER(newsk->sk_wq, NULL); 2590 2591 if (newsk->sk_prot->sockets_allocated) 2592 sk_sockets_allocated_inc(newsk); 2593 2594 if (sock_needs_netstamp(sk) && newsk->sk_flags & SK_FLAGS_TIMESTAMP) 2595 net_enable_timestamp(); 2596 out: 2597 return newsk; 2598 free: 2599 /* It is still raw copy of parent, so invalidate 2600 * destructor and make plain sk_free() 2601 */ 2602 newsk->sk_destruct = NULL; 2603 if (lock) 2604 bh_unlock_sock(newsk); 2605 sk_free(newsk); 2606 newsk = NULL; 2607 goto out; 2608 } 2609 EXPORT_SYMBOL_GPL(sk_clone); 2610 2611 static u32 sk_dst_gso_max_size(struct sock *sk, const struct net_device *dev) 2612 { 2613 bool is_ipv6 = false; 2614 u32 max_size; 2615 2616 #if IS_ENABLED(CONFIG_IPV6) 2617 is_ipv6 = (sk->sk_family == AF_INET6 && 2618 !ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)); 2619 #endif 2620 /* pairs with the WRITE_ONCE() in netif_set_gso(_ipv4)_max_size() */ 2621 max_size = is_ipv6 ? READ_ONCE(dev->gso_max_size) : 2622 READ_ONCE(dev->gso_ipv4_max_size); 2623 if (max_size > GSO_LEGACY_MAX_SIZE && !sk_is_tcp(sk)) 2624 max_size = GSO_LEGACY_MAX_SIZE; 2625 2626 return max_size - (MAX_TCP_HEADER + 1); 2627 } 2628 2629 void sk_setup_caps(struct sock *sk, struct dst_entry *dst) 2630 { 2631 const struct net_device *dev; 2632 u32 max_segs = 1; 2633 2634 rcu_read_lock(); 2635 dev = dst_dev_rcu(dst); 2636 sk->sk_route_caps = dev->features; 2637 if (sk_is_tcp(sk)) { 2638 struct inet_connection_sock *icsk = inet_csk(sk); 2639 2640 sk->sk_route_caps |= NETIF_F_GSO; 2641 icsk->icsk_ack.dst_quick_ack = dst_metric(dst, RTAX_QUICKACK); 2642 } 2643 if (sk->sk_route_caps & NETIF_F_GSO) 2644 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE; 2645 if (unlikely(sk->sk_gso_disabled)) 2646 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 2647 if (sk_can_gso(sk)) { 2648 if (dst->header_len && !xfrm_dst_offload_ok(dst)) { 2649 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 2650 } else { 2651 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM; 2652 sk->sk_gso_max_size = sk_dst_gso_max_size(sk, dev); 2653 /* pairs with the WRITE_ONCE() in netif_set_gso_max_segs() */ 2654 max_segs = max_t(u32, READ_ONCE(dev->gso_max_segs), 1); 2655 } 2656 } 2657 sk->sk_gso_max_segs = max_segs; 2658 sk_dst_set(sk, dst); 2659 rcu_read_unlock(); 2660 } 2661 EXPORT_SYMBOL_GPL(sk_setup_caps); 2662 2663 /* 2664 * Simple resource managers for sockets. 2665 */ 2666 2667 2668 /* 2669 * Write buffer destructor automatically called from kfree_skb. 2670 */ 2671 void sock_wfree(struct sk_buff *skb) 2672 { 2673 unsigned int len = skb->truesize; 2674 struct sock *sk = skb->sk; 2675 bool free; 2676 int old; 2677 2678 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) { 2679 void (*sk_write_space)(struct sock *sk); 2680 2681 sk_write_space = READ_ONCE(sk->sk_write_space); 2682 2683 if (sock_flag(sk, SOCK_RCU_FREE) && 2684 sk_write_space == sock_def_write_space) { 2685 rcu_read_lock(); 2686 free = __refcount_sub_and_test(len, &sk->sk_wmem_alloc, 2687 &old); 2688 sock_def_write_space_wfree(sk, old - len); 2689 rcu_read_unlock(); 2690 if (unlikely(free)) 2691 __sk_free(sk); 2692 return; 2693 } 2694 2695 /* 2696 * Keep a reference on sk_wmem_alloc, this will be released 2697 * after sk_write_space() call 2698 */ 2699 WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc)); 2700 sk_write_space(sk); 2701 len = 1; 2702 } 2703 /* 2704 * if sk_wmem_alloc reaches 0, we must finish what sk_free() 2705 * could not do because of in-flight packets 2706 */ 2707 if (refcount_sub_and_test(len, &sk->sk_wmem_alloc)) 2708 __sk_free(sk); 2709 } 2710 EXPORT_SYMBOL(sock_wfree); 2711 2712 /* This variant of sock_wfree() is used by TCP, 2713 * since it sets SOCK_USE_WRITE_QUEUE. 2714 */ 2715 #ifdef CONFIG_INET 2716 void __sock_wfree(struct sk_buff *skb) 2717 { 2718 struct sock *sk = skb->sk; 2719 2720 if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)) 2721 __sk_free(sk); 2722 } 2723 EXPORT_SYMBOL_GPL(__sock_wfree); 2724 #endif 2725 2726 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk) 2727 { 2728 int old_wmem; 2729 2730 skb_orphan(skb); 2731 #ifdef CONFIG_INET 2732 if (unlikely(!sk_fullsock(sk))) 2733 return skb_set_owner_edemux(skb, sk); 2734 #endif 2735 skb->sk = sk; 2736 skb->destructor = sock_wfree; 2737 skb_set_hash_from_sk(skb, sk); 2738 /* 2739 * We used to take a refcount on sk, but following operation 2740 * is enough to guarantee sk_free() won't free this sock until 2741 * all in-flight packets are completed 2742 */ 2743 __refcount_add(skb->truesize, &sk->sk_wmem_alloc, &old_wmem); 2744 2745 /* (old_wmem == SK_WMEM_ALLOC_BIAS) if no other TX packet for this socket 2746 * is in a host queue (qdisc, NIC queue). 2747 * Set skb->ooo_okay so that netdev_pick_tx() can choose a TX queue 2748 * based on XPS for better performance. 2749 * Otherwise clear ooo_okay to not risk Out Of Order delivery. 2750 */ 2751 skb->ooo_okay = (old_wmem == SK_WMEM_ALLOC_BIAS); 2752 } 2753 EXPORT_SYMBOL(skb_set_owner_w); 2754 2755 static bool can_skb_orphan_partial(const struct sk_buff *skb) 2756 { 2757 /* Drivers depend on in-order delivery for crypto offload, 2758 * partial orphan breaks out-of-order-OK logic. 2759 */ 2760 if (skb_is_decrypted(skb)) 2761 return false; 2762 2763 return (skb->destructor == sock_wfree || 2764 (IS_ENABLED(CONFIG_INET) && skb->destructor == tcp_wfree)); 2765 } 2766 2767 /* This helper is used by netem, as it can hold packets in its 2768 * delay queue. We want to allow the owner socket to send more 2769 * packets, as if they were already TX completed by a typical driver. 2770 * But we also want to keep skb->sk set because some packet schedulers 2771 * rely on it (sch_fq for example). 2772 */ 2773 void skb_orphan_partial(struct sk_buff *skb) 2774 { 2775 if (skb_is_tcp_pure_ack(skb)) 2776 return; 2777 2778 if (can_skb_orphan_partial(skb) && skb_set_owner_sk_safe(skb, skb->sk)) 2779 return; 2780 2781 skb_orphan(skb); 2782 } 2783 EXPORT_SYMBOL(skb_orphan_partial); 2784 2785 /* 2786 * Read buffer destructor automatically called from kfree_skb. 2787 */ 2788 void sock_rfree(struct sk_buff *skb) 2789 { 2790 struct sock *sk = skb->sk; 2791 unsigned int len = skb->truesize; 2792 2793 atomic_sub(len, &sk->sk_rmem_alloc); 2794 sk_mem_uncharge(sk, len); 2795 } 2796 EXPORT_SYMBOL(sock_rfree); 2797 2798 /* 2799 * Buffer destructor for skbs that are not used directly in read or write 2800 * path, e.g. for error handler skbs. Automatically called from kfree_skb. 2801 */ 2802 void sock_efree(struct sk_buff *skb) 2803 { 2804 sock_put(skb->sk); 2805 } 2806 EXPORT_SYMBOL(sock_efree); 2807 2808 /* Buffer destructor for prefetch/receive path where reference count may 2809 * not be held, e.g. for listen sockets. 2810 */ 2811 #ifdef CONFIG_INET 2812 void sock_pfree(struct sk_buff *skb) 2813 { 2814 struct sock *sk = skb->sk; 2815 2816 if (!sk_is_refcounted(sk)) 2817 return; 2818 2819 if (sk->sk_state == TCP_NEW_SYN_RECV && inet_reqsk(sk)->syncookie) { 2820 inet_reqsk(sk)->rsk_listener = NULL; 2821 reqsk_free(inet_reqsk(sk)); 2822 return; 2823 } 2824 2825 sock_gen_put(sk); 2826 } 2827 EXPORT_SYMBOL(sock_pfree); 2828 #endif /* CONFIG_INET */ 2829 2830 /* 2831 * Allocate a skb from the socket's send buffer. 2832 */ 2833 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force, 2834 gfp_t priority) 2835 { 2836 if (force || 2837 refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) { 2838 struct sk_buff *skb = alloc_skb(size, priority); 2839 2840 if (skb) { 2841 skb_set_owner_w(skb, sk); 2842 return skb; 2843 } 2844 } 2845 return NULL; 2846 } 2847 EXPORT_SYMBOL(sock_wmalloc); 2848 2849 static void sock_ofree(struct sk_buff *skb) 2850 { 2851 struct sock *sk = skb->sk; 2852 2853 atomic_sub(skb->truesize, &sk->sk_omem_alloc); 2854 } 2855 2856 struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size, 2857 gfp_t priority) 2858 { 2859 struct sk_buff *skb; 2860 2861 /* small safe race: SKB_TRUESIZE may differ from final skb->truesize */ 2862 if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) > 2863 READ_ONCE(sock_net(sk)->core.sysctl_optmem_max)) 2864 return NULL; 2865 2866 skb = alloc_skb(size, priority); 2867 if (!skb) 2868 return NULL; 2869 2870 atomic_add(skb->truesize, &sk->sk_omem_alloc); 2871 skb->sk = sk; 2872 skb->destructor = sock_ofree; 2873 return skb; 2874 } 2875 2876 /* 2877 * Allocate a memory block from the socket's option memory buffer. 2878 */ 2879 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority) 2880 { 2881 int optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max); 2882 2883 if ((unsigned int)size <= optmem_max && 2884 atomic_read(&sk->sk_omem_alloc) + size < optmem_max) { 2885 void *mem; 2886 /* First do the add, to avoid the race if kmalloc 2887 * might sleep. 2888 */ 2889 atomic_add(size, &sk->sk_omem_alloc); 2890 mem = kmalloc(size, priority); 2891 if (mem) 2892 return mem; 2893 atomic_sub(size, &sk->sk_omem_alloc); 2894 } 2895 return NULL; 2896 } 2897 EXPORT_SYMBOL(sock_kmalloc); 2898 2899 /* 2900 * Duplicate the input "src" memory block using the socket's 2901 * option memory buffer. 2902 */ 2903 void *sock_kmemdup(struct sock *sk, const void *src, 2904 int size, gfp_t priority) 2905 { 2906 void *mem; 2907 2908 mem = sock_kmalloc(sk, size, priority); 2909 if (mem) 2910 memcpy(mem, src, size); 2911 return mem; 2912 } 2913 EXPORT_SYMBOL(sock_kmemdup); 2914 2915 /* Free an option memory block. Note, we actually want the inline 2916 * here as this allows gcc to detect the nullify and fold away the 2917 * condition entirely. 2918 */ 2919 static inline void __sock_kfree_s(struct sock *sk, void *mem, int size, 2920 const bool nullify) 2921 { 2922 if (WARN_ON_ONCE(!mem)) 2923 return; 2924 if (nullify) 2925 kfree_sensitive(mem); 2926 else 2927 kfree(mem); 2928 atomic_sub(size, &sk->sk_omem_alloc); 2929 } 2930 2931 void sock_kfree_s(struct sock *sk, void *mem, int size) 2932 { 2933 __sock_kfree_s(sk, mem, size, false); 2934 } 2935 EXPORT_SYMBOL(sock_kfree_s); 2936 2937 void sock_kzfree_s(struct sock *sk, void *mem, int size) 2938 { 2939 __sock_kfree_s(sk, mem, size, true); 2940 } 2941 EXPORT_SYMBOL(sock_kzfree_s); 2942 2943 /* It is almost wait_for_tcp_memory minus release_sock/lock_sock. 2944 I think, these locks should be removed for datagram sockets. 2945 */ 2946 static long sock_wait_for_wmem(struct sock *sk, long timeo) 2947 { 2948 DEFINE_WAIT(wait); 2949 2950 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 2951 for (;;) { 2952 if (!timeo) 2953 break; 2954 if (signal_pending(current)) 2955 break; 2956 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2957 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); 2958 if (refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) 2959 break; 2960 if (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN) 2961 break; 2962 if (READ_ONCE(sk->sk_err)) 2963 break; 2964 timeo = schedule_timeout(timeo); 2965 } 2966 finish_wait(sk_sleep(sk), &wait); 2967 return timeo; 2968 } 2969 2970 2971 /* 2972 * Generic send/receive buffer handlers 2973 */ 2974 2975 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len, 2976 unsigned long data_len, int noblock, 2977 int *errcode, int max_page_order) 2978 { 2979 struct sk_buff *skb; 2980 long timeo; 2981 int err; 2982 2983 timeo = sock_sndtimeo(sk, noblock); 2984 for (;;) { 2985 err = sock_error(sk); 2986 if (err != 0) 2987 goto failure; 2988 2989 err = -EPIPE; 2990 if (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN) 2991 goto failure; 2992 2993 if (sk_wmem_alloc_get(sk) < READ_ONCE(sk->sk_sndbuf)) 2994 break; 2995 2996 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 2997 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2998 err = -EAGAIN; 2999 if (!timeo) 3000 goto failure; 3001 if (signal_pending(current)) 3002 goto interrupted; 3003 timeo = sock_wait_for_wmem(sk, timeo); 3004 } 3005 skb = alloc_skb_with_frags(header_len, data_len, max_page_order, 3006 errcode, sk->sk_allocation); 3007 if (skb) 3008 skb_set_owner_w(skb, sk); 3009 return skb; 3010 3011 interrupted: 3012 err = sock_intr_errno(timeo); 3013 failure: 3014 *errcode = err; 3015 return NULL; 3016 } 3017 EXPORT_SYMBOL(sock_alloc_send_pskb); 3018 3019 int __sock_cmsg_send(struct sock *sk, struct cmsghdr *cmsg, 3020 struct sockcm_cookie *sockc) 3021 { 3022 u32 tsflags; 3023 3024 BUILD_BUG_ON(SOF_TIMESTAMPING_LAST == (1 << 31)); 3025 3026 switch (cmsg->cmsg_type) { 3027 case SO_MARK: 3028 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && 3029 !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 3030 return -EPERM; 3031 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3032 return -EINVAL; 3033 sockc->mark = *(u32 *)CMSG_DATA(cmsg); 3034 break; 3035 case SO_TIMESTAMPING_OLD: 3036 case SO_TIMESTAMPING_NEW: 3037 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3038 return -EINVAL; 3039 3040 tsflags = *(u32 *)CMSG_DATA(cmsg); 3041 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK) 3042 return -EINVAL; 3043 3044 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK; 3045 sockc->tsflags |= tsflags; 3046 break; 3047 case SCM_TXTIME: 3048 if (!sock_flag(sk, SOCK_TXTIME)) 3049 return -EINVAL; 3050 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64))) 3051 return -EINVAL; 3052 sockc->transmit_time = get_unaligned((u64 *)CMSG_DATA(cmsg)); 3053 break; 3054 case SCM_TS_OPT_ID: 3055 if (sk_is_tcp(sk)) 3056 return -EINVAL; 3057 tsflags = READ_ONCE(sk->sk_tsflags); 3058 if (!(tsflags & SOF_TIMESTAMPING_OPT_ID)) 3059 return -EINVAL; 3060 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3061 return -EINVAL; 3062 sockc->ts_opt_id = *(u32 *)CMSG_DATA(cmsg); 3063 sockc->tsflags |= SOCKCM_FLAG_TS_OPT_ID; 3064 break; 3065 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */ 3066 case SCM_RIGHTS: 3067 case SCM_CREDENTIALS: 3068 break; 3069 case SO_PRIORITY: 3070 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3071 return -EINVAL; 3072 if (!sk_set_prio_allowed(sk, *(u32 *)CMSG_DATA(cmsg))) 3073 return -EPERM; 3074 sockc->priority = *(u32 *)CMSG_DATA(cmsg); 3075 break; 3076 case SCM_DEVMEM_DMABUF: 3077 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3078 return -EINVAL; 3079 sockc->dmabuf_id = *(u32 *)CMSG_DATA(cmsg); 3080 break; 3081 default: 3082 return -EINVAL; 3083 } 3084 return 0; 3085 } 3086 EXPORT_SYMBOL(__sock_cmsg_send); 3087 3088 int sock_cmsg_send(struct sock *sk, struct msghdr *msg, 3089 struct sockcm_cookie *sockc) 3090 { 3091 struct cmsghdr *cmsg; 3092 int ret; 3093 3094 for_each_cmsghdr(cmsg, msg) { 3095 if (!CMSG_OK(msg, cmsg)) 3096 return -EINVAL; 3097 if (cmsg->cmsg_level != SOL_SOCKET) 3098 continue; 3099 ret = __sock_cmsg_send(sk, cmsg, sockc); 3100 if (ret) 3101 return ret; 3102 } 3103 return 0; 3104 } 3105 EXPORT_SYMBOL(sock_cmsg_send); 3106 3107 static void sk_enter_memory_pressure(struct sock *sk) 3108 { 3109 if (!sk->sk_prot->enter_memory_pressure) 3110 return; 3111 3112 sk->sk_prot->enter_memory_pressure(sk); 3113 } 3114 3115 static void sk_leave_memory_pressure(struct sock *sk) 3116 { 3117 if (sk->sk_prot->leave_memory_pressure) { 3118 INDIRECT_CALL_INET_1(sk->sk_prot->leave_memory_pressure, 3119 tcp_leave_memory_pressure, sk); 3120 } else { 3121 unsigned long *memory_pressure = sk->sk_prot->memory_pressure; 3122 3123 if (memory_pressure && READ_ONCE(*memory_pressure)) 3124 WRITE_ONCE(*memory_pressure, 0); 3125 } 3126 } 3127 3128 DEFINE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key); 3129 3130 /** 3131 * skb_page_frag_refill - check that a page_frag contains enough room 3132 * @sz: minimum size of the fragment we want to get 3133 * @pfrag: pointer to page_frag 3134 * @gfp: priority for memory allocation 3135 * 3136 * Note: While this allocator tries to use high order pages, there is 3137 * no guarantee that allocations succeed. Therefore, @sz MUST be 3138 * less or equal than PAGE_SIZE. 3139 */ 3140 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp) 3141 { 3142 if (pfrag->page) { 3143 if (page_ref_count(pfrag->page) == 1) { 3144 pfrag->offset = 0; 3145 return true; 3146 } 3147 if (pfrag->offset + sz <= pfrag->size) 3148 return true; 3149 put_page(pfrag->page); 3150 } 3151 3152 pfrag->offset = 0; 3153 if (SKB_FRAG_PAGE_ORDER && 3154 !static_branch_unlikely(&net_high_order_alloc_disable_key)) { 3155 /* Avoid direct reclaim but allow kswapd to wake */ 3156 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) | 3157 __GFP_COMP | __GFP_NOWARN | 3158 __GFP_NORETRY, 3159 SKB_FRAG_PAGE_ORDER); 3160 if (likely(pfrag->page)) { 3161 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER; 3162 return true; 3163 } 3164 } 3165 pfrag->page = alloc_page(gfp); 3166 if (likely(pfrag->page)) { 3167 pfrag->size = PAGE_SIZE; 3168 return true; 3169 } 3170 return false; 3171 } 3172 EXPORT_SYMBOL(skb_page_frag_refill); 3173 3174 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 3175 { 3176 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation))) 3177 return true; 3178 3179 if (!sk->sk_bypass_prot_mem) 3180 sk_enter_memory_pressure(sk); 3181 3182 sk_stream_moderate_sndbuf(sk); 3183 3184 return false; 3185 } 3186 EXPORT_SYMBOL(sk_page_frag_refill); 3187 3188 static void __lock_sock(struct sock *sk) 3189 __releases(&sk->sk_lock.slock) 3190 __acquires(&sk->sk_lock.slock) 3191 { 3192 DEFINE_WAIT(wait); 3193 3194 for (;;) { 3195 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait, 3196 TASK_UNINTERRUPTIBLE); 3197 spin_unlock_bh(&sk->sk_lock.slock); 3198 schedule(); 3199 spin_lock_bh(&sk->sk_lock.slock); 3200 if (!sock_owned_by_user(sk)) 3201 break; 3202 } 3203 finish_wait(&sk->sk_lock.wq, &wait); 3204 } 3205 3206 void __release_sock(struct sock *sk) 3207 __releases(&sk->sk_lock.slock) 3208 __acquires(&sk->sk_lock.slock) 3209 { 3210 struct sk_buff *skb, *next; 3211 int nb = 0; 3212 3213 while ((skb = sk->sk_backlog.head) != NULL) { 3214 sk->sk_backlog.head = sk->sk_backlog.tail = NULL; 3215 3216 spin_unlock_bh(&sk->sk_lock.slock); 3217 3218 while (1) { 3219 next = skb->next; 3220 prefetch(next); 3221 DEBUG_NET_WARN_ON_ONCE(skb_dst_is_noref(skb)); 3222 skb_mark_not_on_list(skb); 3223 sk_backlog_rcv(sk, skb); 3224 3225 skb = next; 3226 if (!skb) 3227 break; 3228 3229 if (!(++nb & 15)) 3230 cond_resched(); 3231 } 3232 3233 spin_lock_bh(&sk->sk_lock.slock); 3234 } 3235 3236 /* 3237 * Doing the zeroing here guarantee we can not loop forever 3238 * while a wild producer attempts to flood us. 3239 */ 3240 sk->sk_backlog.len = 0; 3241 } 3242 3243 void __sk_flush_backlog(struct sock *sk) 3244 { 3245 spin_lock_bh(&sk->sk_lock.slock); 3246 __release_sock(sk); 3247 3248 if (sk->sk_prot->release_cb) 3249 INDIRECT_CALL_INET_1(sk->sk_prot->release_cb, 3250 tcp_release_cb, sk); 3251 3252 spin_unlock_bh(&sk->sk_lock.slock); 3253 } 3254 EXPORT_SYMBOL_GPL(__sk_flush_backlog); 3255 3256 /** 3257 * sk_wait_data - wait for data to arrive at sk_receive_queue 3258 * @sk: sock to wait on 3259 * @timeo: for how long 3260 * @skb: last skb seen on sk_receive_queue 3261 * 3262 * Now socket state including sk->sk_err is changed only under lock, 3263 * hence we may omit checks after joining wait queue. 3264 * We check receive queue before schedule() only as optimization; 3265 * it is very likely that release_sock() added new data. 3266 */ 3267 int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb) 3268 { 3269 DEFINE_WAIT_FUNC(wait, woken_wake_function); 3270 int rc; 3271 3272 add_wait_queue(sk_sleep(sk), &wait); 3273 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 3274 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait); 3275 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 3276 remove_wait_queue(sk_sleep(sk), &wait); 3277 return rc; 3278 } 3279 EXPORT_SYMBOL(sk_wait_data); 3280 3281 /** 3282 * __sk_mem_raise_allocated - increase memory_allocated 3283 * @sk: socket 3284 * @size: memory size to allocate 3285 * @amt: pages to allocate 3286 * @kind: allocation type 3287 * 3288 * Similar to __sk_mem_schedule(), but does not update sk_forward_alloc. 3289 * 3290 * Unlike the globally shared limits among the sockets under same protocol, 3291 * consuming the budget of a memcg won't have direct effect on other ones. 3292 * So be optimistic about memcg's tolerance, and leave the callers to decide 3293 * whether or not to raise allocated through sk_under_memory_pressure() or 3294 * its variants. 3295 */ 3296 int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind) 3297 { 3298 bool memcg_enabled = false, charged = false; 3299 struct proto *prot = sk->sk_prot; 3300 long allocated = 0; 3301 3302 if (!sk->sk_bypass_prot_mem) { 3303 sk_memory_allocated_add(sk, amt); 3304 allocated = sk_memory_allocated(sk); 3305 } 3306 3307 if (mem_cgroup_sk_enabled(sk)) { 3308 memcg_enabled = true; 3309 charged = mem_cgroup_sk_charge(sk, amt, gfp_memcg_charge()); 3310 if (!charged) 3311 goto suppress_allocation; 3312 } 3313 3314 if (!allocated) 3315 return 1; 3316 3317 /* Under limit. */ 3318 if (allocated <= sk_prot_mem_limits(sk, 0)) { 3319 sk_leave_memory_pressure(sk); 3320 return 1; 3321 } 3322 3323 /* Under pressure. */ 3324 if (allocated > sk_prot_mem_limits(sk, 1)) 3325 sk_enter_memory_pressure(sk); 3326 3327 /* Over hard limit. */ 3328 if (allocated > sk_prot_mem_limits(sk, 2)) 3329 goto suppress_allocation; 3330 3331 /* Guarantee minimum buffer size under pressure (either global 3332 * or memcg) to make sure features described in RFC 7323 (TCP 3333 * Extensions for High Performance) work properly. 3334 * 3335 * This rule does NOT stand when exceeds global or memcg's hard 3336 * limit, or else a DoS attack can be taken place by spawning 3337 * lots of sockets whose usage are under minimum buffer size. 3338 */ 3339 if (kind == SK_MEM_RECV) { 3340 if (atomic_read(&sk->sk_rmem_alloc) < sk_get_rmem0(sk, prot)) 3341 return 1; 3342 3343 } else { /* SK_MEM_SEND */ 3344 int wmem0 = sk_get_wmem0(sk, prot); 3345 3346 if (sk->sk_type == SOCK_STREAM) { 3347 if (sk->sk_wmem_queued < wmem0) 3348 return 1; 3349 } else if (refcount_read(&sk->sk_wmem_alloc) < wmem0) { 3350 return 1; 3351 } 3352 } 3353 3354 if (sk_has_memory_pressure(sk)) { 3355 u64 alloc; 3356 3357 /* The following 'average' heuristic is within the 3358 * scope of global accounting, so it only makes 3359 * sense for global memory pressure. 3360 */ 3361 if (!sk_under_global_memory_pressure(sk)) 3362 return 1; 3363 3364 /* Try to be fair among all the sockets under global 3365 * pressure by allowing the ones that below average 3366 * usage to raise. 3367 */ 3368 alloc = sk_sockets_allocated_read_positive(sk); 3369 if (sk_prot_mem_limits(sk, 2) > alloc * 3370 sk_mem_pages(sk->sk_wmem_queued + 3371 atomic_read(&sk->sk_rmem_alloc) + 3372 sk->sk_forward_alloc)) 3373 return 1; 3374 } 3375 3376 suppress_allocation: 3377 3378 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) { 3379 sk_stream_moderate_sndbuf(sk); 3380 3381 /* Fail only if socket is _under_ its sndbuf. 3382 * In this case we cannot block, so that we have to fail. 3383 */ 3384 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf) { 3385 /* Force charge with __GFP_NOFAIL */ 3386 if (memcg_enabled && !charged) 3387 mem_cgroup_sk_charge(sk, amt, 3388 gfp_memcg_charge() | __GFP_NOFAIL); 3389 return 1; 3390 } 3391 } 3392 3393 trace_sock_exceed_buf_limit(sk, prot, allocated, kind); 3394 3395 if (allocated) 3396 sk_memory_allocated_sub(sk, amt); 3397 3398 if (charged) 3399 mem_cgroup_sk_uncharge(sk, amt); 3400 3401 return 0; 3402 } 3403 3404 /** 3405 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated 3406 * @sk: socket 3407 * @size: memory size to allocate 3408 * @kind: allocation type 3409 * 3410 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means 3411 * rmem allocation. This function assumes that protocols which have 3412 * memory_pressure use sk_wmem_queued as write buffer accounting. 3413 */ 3414 int __sk_mem_schedule(struct sock *sk, int size, int kind) 3415 { 3416 int ret, amt = sk_mem_pages(size); 3417 3418 sk_forward_alloc_add(sk, amt << PAGE_SHIFT); 3419 ret = __sk_mem_raise_allocated(sk, size, amt, kind); 3420 if (!ret) 3421 sk_forward_alloc_add(sk, -(amt << PAGE_SHIFT)); 3422 return ret; 3423 } 3424 EXPORT_SYMBOL(__sk_mem_schedule); 3425 3426 /** 3427 * __sk_mem_reduce_allocated - reclaim memory_allocated 3428 * @sk: socket 3429 * @amount: number of quanta 3430 * 3431 * Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc 3432 */ 3433 void __sk_mem_reduce_allocated(struct sock *sk, int amount) 3434 { 3435 if (mem_cgroup_sk_enabled(sk)) 3436 mem_cgroup_sk_uncharge(sk, amount); 3437 3438 if (sk->sk_bypass_prot_mem) 3439 return; 3440 3441 sk_memory_allocated_sub(sk, amount); 3442 3443 if (sk_under_global_memory_pressure(sk) && 3444 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0))) 3445 sk_leave_memory_pressure(sk); 3446 } 3447 3448 /** 3449 * __sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated 3450 * @sk: socket 3451 * @amount: number of bytes (rounded down to a PAGE_SIZE multiple) 3452 */ 3453 void __sk_mem_reclaim(struct sock *sk, int amount) 3454 { 3455 amount >>= PAGE_SHIFT; 3456 sk_forward_alloc_add(sk, -(amount << PAGE_SHIFT)); 3457 __sk_mem_reduce_allocated(sk, amount); 3458 } 3459 EXPORT_SYMBOL(__sk_mem_reclaim); 3460 3461 void __sk_charge(struct sock *sk, gfp_t gfp) 3462 { 3463 int amt; 3464 3465 gfp |= __GFP_NOFAIL; 3466 if (mem_cgroup_from_sk(sk)) { 3467 /* The socket has not been accepted yet, no need 3468 * to look at newsk->sk_wmem_queued. 3469 */ 3470 amt = sk_mem_pages(sk->sk_forward_alloc + 3471 atomic_read(&sk->sk_rmem_alloc)); 3472 if (amt) 3473 mem_cgroup_sk_charge(sk, amt, gfp); 3474 } 3475 3476 kmem_cache_charge(sk, gfp); 3477 } 3478 3479 int sk_set_peek_off(struct sock *sk, int val) 3480 { 3481 WRITE_ONCE(sk->sk_peek_off, val); 3482 return 0; 3483 } 3484 EXPORT_SYMBOL_GPL(sk_set_peek_off); 3485 3486 /* 3487 * Set of default routines for initialising struct proto_ops when 3488 * the protocol does not support a particular function. In certain 3489 * cases where it makes no sense for a protocol to have a "do nothing" 3490 * function, some default processing is provided. 3491 */ 3492 3493 int sock_no_bind(struct socket *sock, struct sockaddr_unsized *saddr, int len) 3494 { 3495 return -EOPNOTSUPP; 3496 } 3497 EXPORT_SYMBOL(sock_no_bind); 3498 3499 int sock_no_connect(struct socket *sock, struct sockaddr_unsized *saddr, 3500 int len, int flags) 3501 { 3502 return -EOPNOTSUPP; 3503 } 3504 EXPORT_SYMBOL(sock_no_connect); 3505 3506 int sock_no_socketpair(struct socket *sock1, struct socket *sock2) 3507 { 3508 return -EOPNOTSUPP; 3509 } 3510 EXPORT_SYMBOL(sock_no_socketpair); 3511 3512 int sock_no_accept(struct socket *sock, struct socket *newsock, 3513 struct proto_accept_arg *arg) 3514 { 3515 return -EOPNOTSUPP; 3516 } 3517 EXPORT_SYMBOL(sock_no_accept); 3518 3519 int sock_no_getname(struct socket *sock, struct sockaddr *saddr, 3520 int peer) 3521 { 3522 return -EOPNOTSUPP; 3523 } 3524 EXPORT_SYMBOL(sock_no_getname); 3525 3526 int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) 3527 { 3528 return -EOPNOTSUPP; 3529 } 3530 EXPORT_SYMBOL(sock_no_ioctl); 3531 3532 int sock_no_listen(struct socket *sock, int backlog) 3533 { 3534 return -EOPNOTSUPP; 3535 } 3536 EXPORT_SYMBOL(sock_no_listen); 3537 3538 int sock_no_shutdown(struct socket *sock, int how) 3539 { 3540 return -EOPNOTSUPP; 3541 } 3542 EXPORT_SYMBOL(sock_no_shutdown); 3543 3544 int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len) 3545 { 3546 return -EOPNOTSUPP; 3547 } 3548 EXPORT_SYMBOL(sock_no_sendmsg); 3549 3550 int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len) 3551 { 3552 return -EOPNOTSUPP; 3553 } 3554 EXPORT_SYMBOL(sock_no_sendmsg_locked); 3555 3556 int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len, 3557 int flags) 3558 { 3559 return -EOPNOTSUPP; 3560 } 3561 EXPORT_SYMBOL(sock_no_recvmsg); 3562 3563 int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma) 3564 { 3565 /* Mirror missing mmap method error code */ 3566 return -ENODEV; 3567 } 3568 EXPORT_SYMBOL(sock_no_mmap); 3569 3570 /* 3571 * When a file is received (via SCM_RIGHTS, etc), we must bump the 3572 * various sock-based usage counts. 3573 */ 3574 void __receive_sock(struct file *file) 3575 { 3576 struct socket *sock; 3577 3578 sock = sock_from_file(file); 3579 if (sock) { 3580 sock_update_netprioidx(&sock->sk->sk_cgrp_data); 3581 sock_update_classid(&sock->sk->sk_cgrp_data); 3582 } 3583 } 3584 3585 /* 3586 * Default Socket Callbacks 3587 */ 3588 3589 static void sock_def_wakeup(struct sock *sk) 3590 { 3591 struct socket_wq *wq; 3592 3593 rcu_read_lock(); 3594 wq = rcu_dereference(sk->sk_wq); 3595 if (skwq_has_sleeper(wq)) 3596 wake_up_interruptible_all(&wq->wait); 3597 rcu_read_unlock(); 3598 } 3599 3600 static void sock_def_error_report(struct sock *sk) 3601 { 3602 struct socket_wq *wq; 3603 3604 rcu_read_lock(); 3605 wq = rcu_dereference(sk->sk_wq); 3606 if (skwq_has_sleeper(wq)) 3607 wake_up_interruptible_poll(&wq->wait, EPOLLERR); 3608 sk_wake_async_rcu(sk, SOCK_WAKE_IO, POLL_ERR); 3609 rcu_read_unlock(); 3610 } 3611 3612 void sock_def_readable(struct sock *sk) 3613 { 3614 struct socket_wq *wq; 3615 3616 trace_sk_data_ready(sk); 3617 3618 rcu_read_lock(); 3619 wq = rcu_dereference(sk->sk_wq); 3620 if (skwq_has_sleeper(wq)) 3621 wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI | 3622 EPOLLRDNORM | EPOLLRDBAND); 3623 sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_IN); 3624 rcu_read_unlock(); 3625 } 3626 3627 static void sock_def_write_space(struct sock *sk) 3628 { 3629 struct socket_wq *wq; 3630 3631 rcu_read_lock(); 3632 3633 /* Do not wake up a writer until he can make "significant" 3634 * progress. --DaveM 3635 */ 3636 if (sock_writeable(sk)) { 3637 wq = rcu_dereference(sk->sk_wq); 3638 if (skwq_has_sleeper(wq)) 3639 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 3640 EPOLLWRNORM | EPOLLWRBAND); 3641 3642 /* Should agree with poll, otherwise some programs break */ 3643 sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT); 3644 } 3645 3646 rcu_read_unlock(); 3647 } 3648 3649 /* An optimised version of sock_def_write_space(), should only be called 3650 * for SOCK_RCU_FREE sockets under RCU read section and after putting 3651 * ->sk_wmem_alloc. 3652 */ 3653 static void sock_def_write_space_wfree(struct sock *sk, int wmem_alloc) 3654 { 3655 /* Do not wake up a writer until he can make "significant" 3656 * progress. --DaveM 3657 */ 3658 if (__sock_writeable(sk, wmem_alloc)) { 3659 struct socket_wq *wq = rcu_dereference(sk->sk_wq); 3660 3661 /* rely on refcount_sub from sock_wfree() */ 3662 smp_mb__after_atomic(); 3663 if (wq && waitqueue_active(&wq->wait)) 3664 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 3665 EPOLLWRNORM | EPOLLWRBAND); 3666 3667 /* Should agree with poll, otherwise some programs break */ 3668 sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT); 3669 } 3670 } 3671 3672 static void sock_def_destruct(struct sock *sk) 3673 { 3674 } 3675 3676 void sk_send_sigurg(struct sock *sk) 3677 { 3678 if (sk->sk_socket && sk->sk_socket->file) 3679 if (send_sigurg(sk->sk_socket->file)) 3680 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI); 3681 } 3682 EXPORT_SYMBOL(sk_send_sigurg); 3683 3684 void sk_reset_timer(struct sock *sk, struct timer_list* timer, 3685 unsigned long expires) 3686 { 3687 if (!mod_timer(timer, expires)) 3688 sock_hold(sk); 3689 } 3690 EXPORT_SYMBOL(sk_reset_timer); 3691 3692 void sk_stop_timer(struct sock *sk, struct timer_list* timer) 3693 { 3694 if (timer_delete(timer)) 3695 __sock_put(sk); 3696 } 3697 EXPORT_SYMBOL(sk_stop_timer); 3698 3699 void sk_stop_timer_sync(struct sock *sk, struct timer_list *timer) 3700 { 3701 if (timer_delete_sync(timer)) 3702 __sock_put(sk); 3703 } 3704 EXPORT_SYMBOL(sk_stop_timer_sync); 3705 3706 void sock_init_data_uid(struct socket *sock, struct sock *sk, kuid_t uid) 3707 { 3708 sk_init_common(sk); 3709 sk->sk_send_head = NULL; 3710 3711 timer_setup(&sk->sk_timer, NULL, 0); 3712 3713 sk->sk_allocation = GFP_KERNEL; 3714 sk->sk_rcvbuf = READ_ONCE(sysctl_rmem_default); 3715 sk->sk_sndbuf = READ_ONCE(sysctl_wmem_default); 3716 sk->sk_state = TCP_CLOSE; 3717 sk->sk_use_task_frag = true; 3718 sk_set_socket(sk, sock); 3719 3720 sock_set_flag(sk, SOCK_ZAPPED); 3721 3722 if (sock) { 3723 sk->sk_type = sock->type; 3724 RCU_INIT_POINTER(sk->sk_wq, &sock->wq); 3725 sock->sk = sk; 3726 } else { 3727 RCU_INIT_POINTER(sk->sk_wq, NULL); 3728 } 3729 sk->sk_uid = uid; 3730 3731 sk->sk_state_change = sock_def_wakeup; 3732 sk->sk_data_ready = sock_def_readable; 3733 sk->sk_write_space = sock_def_write_space; 3734 sk->sk_error_report = sock_def_error_report; 3735 sk->sk_destruct = sock_def_destruct; 3736 3737 sk->sk_frag.page = NULL; 3738 sk->sk_frag.offset = 0; 3739 sk->sk_peek_off = -1; 3740 3741 sk->sk_peer_pid = NULL; 3742 sk->sk_peer_cred = NULL; 3743 spin_lock_init(&sk->sk_peer_lock); 3744 3745 sk->sk_write_pending = 0; 3746 sk->sk_rcvlowat = 1; 3747 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT; 3748 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT; 3749 3750 sk->sk_stamp = SK_DEFAULT_STAMP; 3751 #if BITS_PER_LONG==32 3752 seqlock_init(&sk->sk_stamp_seq); 3753 #endif 3754 atomic_set(&sk->sk_zckey, 0); 3755 3756 #ifdef CONFIG_NET_RX_BUSY_POLL 3757 sk->sk_napi_id = 0; 3758 sk->sk_ll_usec = READ_ONCE(sysctl_net_busy_read); 3759 #endif 3760 3761 sk->sk_max_pacing_rate = ~0UL; 3762 sk->sk_pacing_rate = ~0UL; 3763 WRITE_ONCE(sk->sk_pacing_shift, 10); 3764 sk->sk_incoming_cpu = -1; 3765 3766 sk_rx_queue_clear(sk); 3767 /* 3768 * Before updating sk_refcnt, we must commit prior changes to memory 3769 * (Documentation/RCU/rculist_nulls.rst for details) 3770 */ 3771 smp_wmb(); 3772 refcount_set(&sk->sk_refcnt, 1); 3773 sk_drops_reset(sk); 3774 } 3775 EXPORT_SYMBOL(sock_init_data_uid); 3776 3777 void sock_init_data(struct socket *sock, struct sock *sk) 3778 { 3779 kuid_t uid = sock ? 3780 SOCK_INODE(sock)->i_uid : 3781 make_kuid(sock_net(sk)->user_ns, 0); 3782 3783 sock_init_data_uid(sock, sk, uid); 3784 } 3785 EXPORT_SYMBOL(sock_init_data); 3786 3787 void noinline lock_sock_nested(struct sock *sk, int subclass) 3788 { 3789 /* The sk_lock has mutex_lock() semantics here. */ 3790 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_); 3791 3792 might_sleep(); 3793 #ifdef CONFIG_64BIT 3794 if (sizeof(struct slock_owned) == sizeof(long)) { 3795 socket_lock_t tmp = { 3796 .slock = __SPIN_LOCK_UNLOCKED(tmp.slock), 3797 .owned = 1, 3798 }; 3799 socket_lock_t old = { 3800 .slock = __SPIN_LOCK_UNLOCKED(old.slock), 3801 .owned = 0, 3802 }; 3803 3804 if (likely(try_cmpxchg(&sk->sk_lock.combined, 3805 &old.combined, tmp.combined))) 3806 return; 3807 } 3808 #endif 3809 spin_lock_bh(&sk->sk_lock.slock); 3810 if (unlikely(sock_owned_by_user_nocheck(sk))) 3811 __lock_sock(sk); 3812 sk->sk_lock.owned = 1; 3813 spin_unlock_bh(&sk->sk_lock.slock); 3814 } 3815 EXPORT_SYMBOL(lock_sock_nested); 3816 3817 void release_sock(struct sock *sk) 3818 { 3819 spin_lock_bh(&sk->sk_lock.slock); 3820 3821 if (unlikely(sk->sk_backlog.tail)) 3822 __release_sock(sk); 3823 3824 if (sk->sk_prot->release_cb) { 3825 if (!tcp_release_cb_cond(sk)) 3826 sk->sk_prot->release_cb(sk); 3827 } 3828 sock_release_ownership(sk); 3829 if (unlikely(waitqueue_active(&sk->sk_lock.wq))) 3830 wake_up(&sk->sk_lock.wq); 3831 3832 spin_unlock_bh(&sk->sk_lock.slock); 3833 } 3834 EXPORT_SYMBOL(release_sock); 3835 3836 bool __lock_sock_fast(struct sock *sk) __acquires(&sk->sk_lock.slock) 3837 { 3838 might_sleep(); 3839 spin_lock_bh(&sk->sk_lock.slock); 3840 3841 if (likely(!sock_owned_by_user_nocheck(sk))) { 3842 /* 3843 * Fast path return with bottom halves disabled and 3844 * sock::sk_lock.slock held. 3845 * 3846 * The 'mutex' is not contended and holding 3847 * sock::sk_lock.slock prevents all other lockers to 3848 * proceed so the corresponding unlock_sock_fast() can 3849 * avoid the slow path of release_sock() completely and 3850 * just release slock. 3851 * 3852 * From a semantical POV this is equivalent to 'acquiring' 3853 * the 'mutex', hence the corresponding lockdep 3854 * mutex_release() has to happen in the fast path of 3855 * unlock_sock_fast(). 3856 */ 3857 return false; 3858 } 3859 3860 __lock_sock(sk); 3861 sk->sk_lock.owned = 1; 3862 __acquire(&sk->sk_lock.slock); 3863 spin_unlock_bh(&sk->sk_lock.slock); 3864 return true; 3865 } 3866 EXPORT_SYMBOL(__lock_sock_fast); 3867 3868 int sock_gettstamp(struct socket *sock, void __user *userstamp, 3869 bool timeval, bool time32) 3870 { 3871 struct sock *sk = sock->sk; 3872 struct timespec64 ts; 3873 3874 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 3875 ts = ktime_to_timespec64(sock_read_timestamp(sk)); 3876 if (ts.tv_sec == -1) 3877 return -ENOENT; 3878 if (ts.tv_sec == 0) { 3879 ktime_t kt = ktime_get_real(); 3880 sock_write_timestamp(sk, kt); 3881 ts = ktime_to_timespec64(kt); 3882 } 3883 3884 if (timeval) 3885 ts.tv_nsec /= 1000; 3886 3887 #ifdef CONFIG_COMPAT_32BIT_TIME 3888 if (time32) 3889 return put_old_timespec32(&ts, userstamp); 3890 #endif 3891 #ifdef CONFIG_SPARC64 3892 /* beware of padding in sparc64 timeval */ 3893 if (timeval && !in_compat_syscall()) { 3894 struct __kernel_old_timeval __user tv = { 3895 .tv_sec = ts.tv_sec, 3896 .tv_usec = ts.tv_nsec, 3897 }; 3898 if (copy_to_user(userstamp, &tv, sizeof(tv))) 3899 return -EFAULT; 3900 return 0; 3901 } 3902 #endif 3903 return put_timespec64(&ts, userstamp); 3904 } 3905 EXPORT_SYMBOL(sock_gettstamp); 3906 3907 void sock_enable_timestamp(struct sock *sk, enum sock_flags flag) 3908 { 3909 if (!sock_flag(sk, flag)) { 3910 unsigned long previous_flags = sk->sk_flags; 3911 3912 sock_set_flag(sk, flag); 3913 /* 3914 * we just set one of the two flags which require net 3915 * time stamping, but time stamping might have been on 3916 * already because of the other one 3917 */ 3918 if (sock_needs_netstamp(sk) && 3919 !(previous_flags & SK_FLAGS_TIMESTAMP)) 3920 net_enable_timestamp(); 3921 } 3922 } 3923 3924 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, 3925 int level, int type) 3926 { 3927 struct sock_extended_err ee; 3928 struct sk_buff *skb; 3929 int copied, err; 3930 3931 err = -EAGAIN; 3932 skb = sock_dequeue_err_skb(sk); 3933 if (skb == NULL) 3934 goto out; 3935 3936 copied = skb->len; 3937 if (copied > len) { 3938 msg->msg_flags |= MSG_TRUNC; 3939 copied = len; 3940 } 3941 err = skb_copy_datagram_msg(skb, 0, msg, copied); 3942 if (err) 3943 goto out_free_skb; 3944 3945 sock_recv_timestamp(msg, sk, skb); 3946 3947 /* We must use a bounce buffer for CONFIG_HARDENED_USERCOPY=y */ 3948 ee = SKB_EXT_ERR(skb)->ee; 3949 put_cmsg(msg, level, type, sizeof(ee), &ee); 3950 3951 msg->msg_flags |= MSG_ERRQUEUE; 3952 err = copied; 3953 3954 out_free_skb: 3955 kfree_skb(skb); 3956 out: 3957 return err; 3958 } 3959 EXPORT_SYMBOL(sock_recv_errqueue); 3960 3961 /* 3962 * Get a socket option on an socket. 3963 * 3964 * FIX: POSIX 1003.1g is very ambiguous here. It states that 3965 * asynchronous errors should be reported by getsockopt. We assume 3966 * this means if you specify SO_ERROR (otherwise what is the point of it). 3967 */ 3968 int sock_common_getsockopt(struct socket *sock, int level, int optname, 3969 char __user *optval, int __user *optlen) 3970 { 3971 struct sock *sk = sock->sk; 3972 3973 /* IPV6_ADDRFORM can change sk->sk_prot under us. */ 3974 return READ_ONCE(sk->sk_prot)->getsockopt(sk, level, optname, optval, optlen); 3975 } 3976 EXPORT_SYMBOL(sock_common_getsockopt); 3977 3978 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, 3979 int flags) 3980 { 3981 struct sock *sk = sock->sk; 3982 3983 return sk->sk_prot->recvmsg(sk, msg, size, flags); 3984 } 3985 EXPORT_SYMBOL(sock_common_recvmsg); 3986 3987 /* 3988 * Set socket options on an inet socket. 3989 */ 3990 int sock_common_setsockopt(struct socket *sock, int level, int optname, 3991 sockptr_t optval, unsigned int optlen) 3992 { 3993 struct sock *sk = sock->sk; 3994 3995 /* IPV6_ADDRFORM can change sk->sk_prot under us. */ 3996 return READ_ONCE(sk->sk_prot)->setsockopt(sk, level, optname, optval, optlen); 3997 } 3998 EXPORT_SYMBOL(sock_common_setsockopt); 3999 4000 void sk_common_release(struct sock *sk) 4001 { 4002 if (sk->sk_prot->destroy) 4003 sk->sk_prot->destroy(sk); 4004 4005 /* 4006 * Observation: when sk_common_release is called, processes have 4007 * no access to socket. But net still has. 4008 * Step one, detach it from networking: 4009 * 4010 * A. Remove from hash tables. 4011 */ 4012 4013 sk->sk_prot->unhash(sk); 4014 4015 /* 4016 * In this point socket cannot receive new packets, but it is possible 4017 * that some packets are in flight because some CPU runs receiver and 4018 * did hash table lookup before we unhashed socket. They will achieve 4019 * receive queue and will be purged by socket destructor. 4020 * 4021 * Also we still have packets pending on receive queue and probably, 4022 * our own packets waiting in device queues. sock_destroy will drain 4023 * receive queue, but transmitted packets will delay socket destruction 4024 * until the last reference will be released. 4025 */ 4026 4027 sock_orphan(sk); 4028 4029 xfrm_sk_free_policy(sk); 4030 4031 sock_put(sk); 4032 } 4033 EXPORT_SYMBOL(sk_common_release); 4034 4035 void sk_get_meminfo(const struct sock *sk, u32 *mem) 4036 { 4037 memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS); 4038 4039 mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk); 4040 mem[SK_MEMINFO_RCVBUF] = READ_ONCE(sk->sk_rcvbuf); 4041 mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk); 4042 mem[SK_MEMINFO_SNDBUF] = READ_ONCE(sk->sk_sndbuf); 4043 mem[SK_MEMINFO_FWD_ALLOC] = READ_ONCE(sk->sk_forward_alloc); 4044 mem[SK_MEMINFO_WMEM_QUEUED] = READ_ONCE(sk->sk_wmem_queued); 4045 mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc); 4046 mem[SK_MEMINFO_BACKLOG] = READ_ONCE(sk->sk_backlog.len); 4047 mem[SK_MEMINFO_DROPS] = sk_drops_read(sk); 4048 } 4049 4050 #ifdef CONFIG_PROC_FS 4051 static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR); 4052 4053 int sock_prot_inuse_get(struct net *net, struct proto *prot) 4054 { 4055 int cpu, idx = prot->inuse_idx; 4056 int res = 0; 4057 4058 for_each_possible_cpu(cpu) 4059 res += per_cpu_ptr(net->core.prot_inuse, cpu)->val[idx]; 4060 4061 return res >= 0 ? res : 0; 4062 } 4063 EXPORT_SYMBOL_GPL(sock_prot_inuse_get); 4064 4065 int sock_inuse_get(struct net *net) 4066 { 4067 int cpu, res = 0; 4068 4069 for_each_possible_cpu(cpu) 4070 res += per_cpu_ptr(net->core.prot_inuse, cpu)->all; 4071 4072 return res; 4073 } 4074 4075 EXPORT_SYMBOL_GPL(sock_inuse_get); 4076 4077 static int __net_init sock_inuse_init_net(struct net *net) 4078 { 4079 net->core.prot_inuse = alloc_percpu(struct prot_inuse); 4080 if (net->core.prot_inuse == NULL) 4081 return -ENOMEM; 4082 return 0; 4083 } 4084 4085 static void __net_exit sock_inuse_exit_net(struct net *net) 4086 { 4087 free_percpu(net->core.prot_inuse); 4088 } 4089 4090 static struct pernet_operations net_inuse_ops = { 4091 .init = sock_inuse_init_net, 4092 .exit = sock_inuse_exit_net, 4093 }; 4094 4095 static __init int net_inuse_init(void) 4096 { 4097 if (register_pernet_subsys(&net_inuse_ops)) 4098 panic("Cannot initialize net inuse counters"); 4099 4100 return 0; 4101 } 4102 4103 core_initcall(net_inuse_init); 4104 4105 static int assign_proto_idx(struct proto *prot) 4106 { 4107 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR); 4108 4109 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR)) { 4110 pr_err("PROTO_INUSE_NR exhausted\n"); 4111 return -ENOSPC; 4112 } 4113 4114 set_bit(prot->inuse_idx, proto_inuse_idx); 4115 return 0; 4116 } 4117 4118 static void release_proto_idx(struct proto *prot) 4119 { 4120 if (prot->inuse_idx != PROTO_INUSE_NR) 4121 clear_bit(prot->inuse_idx, proto_inuse_idx); 4122 } 4123 #else 4124 static inline int assign_proto_idx(struct proto *prot) 4125 { 4126 return 0; 4127 } 4128 4129 static inline void release_proto_idx(struct proto *prot) 4130 { 4131 } 4132 4133 #endif 4134 4135 static void tw_prot_cleanup(struct timewait_sock_ops *twsk_prot) 4136 { 4137 if (!twsk_prot) 4138 return; 4139 kfree(twsk_prot->twsk_slab_name); 4140 twsk_prot->twsk_slab_name = NULL; 4141 kmem_cache_destroy(twsk_prot->twsk_slab); 4142 twsk_prot->twsk_slab = NULL; 4143 } 4144 4145 static int tw_prot_init(const struct proto *prot) 4146 { 4147 struct timewait_sock_ops *twsk_prot = prot->twsk_prot; 4148 4149 if (!twsk_prot) 4150 return 0; 4151 4152 twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", 4153 prot->name); 4154 if (!twsk_prot->twsk_slab_name) 4155 return -ENOMEM; 4156 4157 twsk_prot->twsk_slab = 4158 kmem_cache_create(twsk_prot->twsk_slab_name, 4159 twsk_prot->twsk_obj_size, 0, 4160 SLAB_ACCOUNT | prot->slab_flags, 4161 NULL); 4162 if (!twsk_prot->twsk_slab) { 4163 pr_crit("%s: Can't create timewait sock SLAB cache!\n", 4164 prot->name); 4165 return -ENOMEM; 4166 } 4167 4168 return 0; 4169 } 4170 4171 static void req_prot_cleanup(struct request_sock_ops *rsk_prot) 4172 { 4173 if (!rsk_prot) 4174 return; 4175 kfree(rsk_prot->slab_name); 4176 rsk_prot->slab_name = NULL; 4177 kmem_cache_destroy(rsk_prot->slab); 4178 rsk_prot->slab = NULL; 4179 } 4180 4181 static int req_prot_init(const struct proto *prot) 4182 { 4183 struct request_sock_ops *rsk_prot = prot->rsk_prot; 4184 4185 if (!rsk_prot) 4186 return 0; 4187 4188 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s", 4189 prot->name); 4190 if (!rsk_prot->slab_name) 4191 return -ENOMEM; 4192 4193 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name, 4194 rsk_prot->obj_size, 0, 4195 SLAB_ACCOUNT | prot->slab_flags, 4196 NULL); 4197 4198 if (!rsk_prot->slab) { 4199 pr_crit("%s: Can't create request sock SLAB cache!\n", 4200 prot->name); 4201 return -ENOMEM; 4202 } 4203 return 0; 4204 } 4205 4206 int proto_register(struct proto *prot, int alloc_slab) 4207 { 4208 int ret = -ENOBUFS; 4209 4210 if (prot->memory_allocated && !prot->sysctl_mem) { 4211 pr_err("%s: missing sysctl_mem\n", prot->name); 4212 return -EINVAL; 4213 } 4214 if (prot->memory_allocated && !prot->per_cpu_fw_alloc) { 4215 pr_err("%s: missing per_cpu_fw_alloc\n", prot->name); 4216 return -EINVAL; 4217 } 4218 if (alloc_slab) { 4219 struct kmem_cache_args args = { 4220 .useroffset = prot->useroffset, 4221 .usersize = prot->usersize, 4222 .freeptr_offset = prot->freeptr_offset, 4223 .use_freeptr_offset = !!prot->freeptr_offset, 4224 }; 4225 4226 prot->slab = kmem_cache_create(prot->name, prot->obj_size, 4227 &args, 4228 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT | 4229 prot->slab_flags); 4230 if (prot->slab == NULL) { 4231 pr_crit("%s: Can't create sock SLAB cache!\n", 4232 prot->name); 4233 goto out; 4234 } 4235 4236 if (req_prot_init(prot)) 4237 goto out_free_request_sock_slab; 4238 4239 if (tw_prot_init(prot)) 4240 goto out_free_timewait_sock_slab; 4241 } 4242 4243 mutex_lock(&proto_list_mutex); 4244 ret = assign_proto_idx(prot); 4245 if (ret) { 4246 mutex_unlock(&proto_list_mutex); 4247 goto out_free_timewait_sock_slab; 4248 } 4249 list_add(&prot->node, &proto_list); 4250 mutex_unlock(&proto_list_mutex); 4251 return ret; 4252 4253 out_free_timewait_sock_slab: 4254 if (alloc_slab) 4255 tw_prot_cleanup(prot->twsk_prot); 4256 out_free_request_sock_slab: 4257 if (alloc_slab) { 4258 req_prot_cleanup(prot->rsk_prot); 4259 4260 kmem_cache_destroy(prot->slab); 4261 prot->slab = NULL; 4262 } 4263 out: 4264 return ret; 4265 } 4266 EXPORT_SYMBOL(proto_register); 4267 4268 void proto_unregister(struct proto *prot) 4269 { 4270 mutex_lock(&proto_list_mutex); 4271 release_proto_idx(prot); 4272 list_del(&prot->node); 4273 mutex_unlock(&proto_list_mutex); 4274 4275 kmem_cache_destroy(prot->slab); 4276 prot->slab = NULL; 4277 4278 req_prot_cleanup(prot->rsk_prot); 4279 tw_prot_cleanup(prot->twsk_prot); 4280 } 4281 EXPORT_SYMBOL(proto_unregister); 4282 4283 int sock_load_diag_module(int family, int protocol) 4284 { 4285 if (!protocol) { 4286 if (!sock_is_registered(family)) 4287 return -ENOENT; 4288 4289 return request_module("net-pf-%d-proto-%d-type-%d", PF_NETLINK, 4290 NETLINK_SOCK_DIAG, family); 4291 } 4292 4293 #ifdef CONFIG_INET 4294 if (family == AF_INET && 4295 protocol != IPPROTO_RAW && 4296 protocol < MAX_INET_PROTOS && 4297 !rcu_access_pointer(inet_protos[protocol])) 4298 return -ENOENT; 4299 #endif 4300 4301 return request_module("net-pf-%d-proto-%d-type-%d-%d", PF_NETLINK, 4302 NETLINK_SOCK_DIAG, family, protocol); 4303 } 4304 EXPORT_SYMBOL(sock_load_diag_module); 4305 4306 #ifdef CONFIG_PROC_FS 4307 static void *proto_seq_start(struct seq_file *seq, loff_t *pos) 4308 __acquires(proto_list_mutex) 4309 { 4310 mutex_lock(&proto_list_mutex); 4311 return seq_list_start_head(&proto_list, *pos); 4312 } 4313 4314 static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos) 4315 { 4316 return seq_list_next(v, &proto_list, pos); 4317 } 4318 4319 static void proto_seq_stop(struct seq_file *seq, void *v) 4320 __releases(proto_list_mutex) 4321 { 4322 mutex_unlock(&proto_list_mutex); 4323 } 4324 4325 static char proto_method_implemented(const void *method) 4326 { 4327 return method == NULL ? 'n' : 'y'; 4328 } 4329 static long sock_prot_memory_allocated(struct proto *proto) 4330 { 4331 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L; 4332 } 4333 4334 static const char *sock_prot_memory_pressure(struct proto *proto) 4335 { 4336 return proto->memory_pressure != NULL ? 4337 proto_memory_pressure(proto) ? "yes" : "no" : "NI"; 4338 } 4339 4340 static void proto_seq_printf(struct seq_file *seq, struct proto *proto) 4341 { 4342 4343 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s " 4344 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n", 4345 proto->name, 4346 proto->obj_size, 4347 sock_prot_inuse_get(seq_file_net(seq), proto), 4348 sock_prot_memory_allocated(proto), 4349 sock_prot_memory_pressure(proto), 4350 proto->max_header, 4351 proto->slab == NULL ? "no" : "yes", 4352 module_name(proto->owner), 4353 proto_method_implemented(proto->close), 4354 proto_method_implemented(proto->connect), 4355 proto_method_implemented(proto->disconnect), 4356 proto_method_implemented(proto->accept), 4357 proto_method_implemented(proto->ioctl), 4358 proto_method_implemented(proto->init), 4359 proto_method_implemented(proto->destroy), 4360 proto_method_implemented(proto->shutdown), 4361 proto_method_implemented(proto->setsockopt), 4362 proto_method_implemented(proto->getsockopt), 4363 proto_method_implemented(proto->sendmsg), 4364 proto_method_implemented(proto->recvmsg), 4365 proto_method_implemented(proto->bind), 4366 proto_method_implemented(proto->backlog_rcv), 4367 proto_method_implemented(proto->hash), 4368 proto_method_implemented(proto->unhash), 4369 proto_method_implemented(proto->get_port), 4370 proto_method_implemented(proto->enter_memory_pressure)); 4371 } 4372 4373 static int proto_seq_show(struct seq_file *seq, void *v) 4374 { 4375 if (v == &proto_list) 4376 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s", 4377 "protocol", 4378 "size", 4379 "sockets", 4380 "memory", 4381 "press", 4382 "maxhdr", 4383 "slab", 4384 "module", 4385 "cl co di ac io in de sh ss gs se re bi br ha uh gp em\n"); 4386 else 4387 proto_seq_printf(seq, list_entry(v, struct proto, node)); 4388 return 0; 4389 } 4390 4391 static const struct seq_operations proto_seq_ops = { 4392 .start = proto_seq_start, 4393 .next = proto_seq_next, 4394 .stop = proto_seq_stop, 4395 .show = proto_seq_show, 4396 }; 4397 4398 static __net_init int proto_init_net(struct net *net) 4399 { 4400 if (!proc_create_net("protocols", 0444, net->proc_net, &proto_seq_ops, 4401 sizeof(struct seq_net_private))) 4402 return -ENOMEM; 4403 4404 return 0; 4405 } 4406 4407 static __net_exit void proto_exit_net(struct net *net) 4408 { 4409 remove_proc_entry("protocols", net->proc_net); 4410 } 4411 4412 4413 static __net_initdata struct pernet_operations proto_net_ops = { 4414 .init = proto_init_net, 4415 .exit = proto_exit_net, 4416 }; 4417 4418 static int __init proto_init(void) 4419 { 4420 return register_pernet_subsys(&proto_net_ops); 4421 } 4422 4423 subsys_initcall(proto_init); 4424 4425 #endif /* PROC_FS */ 4426 4427 #ifdef CONFIG_NET_RX_BUSY_POLL 4428 bool sk_busy_loop_end(void *p, unsigned long start_time) 4429 { 4430 struct sock *sk = p; 4431 4432 if (!skb_queue_empty_lockless(&sk->sk_receive_queue)) 4433 return true; 4434 4435 if (sk_is_udp(sk) && 4436 !skb_queue_empty_lockless(&udp_sk(sk)->reader_queue)) 4437 return true; 4438 4439 return sk_busy_loop_timeout(sk, start_time); 4440 } 4441 EXPORT_SYMBOL(sk_busy_loop_end); 4442 #endif /* CONFIG_NET_RX_BUSY_POLL */ 4443 4444 int sock_bind_add(struct sock *sk, struct sockaddr_unsized *addr, int addr_len) 4445 { 4446 if (!sk->sk_prot->bind_add) 4447 return -EOPNOTSUPP; 4448 return sk->sk_prot->bind_add(sk, addr, addr_len); 4449 } 4450 EXPORT_SYMBOL(sock_bind_add); 4451 4452 /* Copy 'size' bytes from userspace and return `size` back to userspace */ 4453 int sock_ioctl_inout(struct sock *sk, unsigned int cmd, 4454 void __user *arg, void *karg, size_t size) 4455 { 4456 int ret; 4457 4458 if (copy_from_user(karg, arg, size)) 4459 return -EFAULT; 4460 4461 ret = READ_ONCE(sk->sk_prot)->ioctl(sk, cmd, karg); 4462 if (ret) 4463 return ret; 4464 4465 if (copy_to_user(arg, karg, size)) 4466 return -EFAULT; 4467 4468 return 0; 4469 } 4470 EXPORT_SYMBOL(sock_ioctl_inout); 4471 4472 /* This is the most common ioctl prep function, where the result (4 bytes) is 4473 * copied back to userspace if the ioctl() returns successfully. No input is 4474 * copied from userspace as input argument. 4475 */ 4476 static int sock_ioctl_out(struct sock *sk, unsigned int cmd, void __user *arg) 4477 { 4478 int ret, karg = 0; 4479 4480 ret = READ_ONCE(sk->sk_prot)->ioctl(sk, cmd, &karg); 4481 if (ret) 4482 return ret; 4483 4484 return put_user(karg, (int __user *)arg); 4485 } 4486 4487 /* A wrapper around sock ioctls, which copies the data from userspace 4488 * (depending on the protocol/ioctl), and copies back the result to userspace. 4489 * The main motivation for this function is to pass kernel memory to the 4490 * protocol ioctl callbacks, instead of userspace memory. 4491 */ 4492 int sk_ioctl(struct sock *sk, unsigned int cmd, void __user *arg) 4493 { 4494 int rc = 1; 4495 4496 if (sk->sk_type == SOCK_RAW && sk->sk_family == AF_INET) 4497 rc = ipmr_sk_ioctl(sk, cmd, arg); 4498 else if (sk->sk_type == SOCK_RAW && sk->sk_family == AF_INET6) 4499 rc = ip6mr_sk_ioctl(sk, cmd, arg); 4500 else if (sk_is_phonet(sk)) 4501 rc = phonet_sk_ioctl(sk, cmd, arg); 4502 4503 /* If ioctl was processed, returns its value */ 4504 if (rc <= 0) 4505 return rc; 4506 4507 /* Otherwise call the default handler */ 4508 return sock_ioctl_out(sk, cmd, arg); 4509 } 4510 EXPORT_SYMBOL(sk_ioctl); 4511 4512 static int __init sock_struct_check(void) 4513 { 4514 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_drops); 4515 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_peek_off); 4516 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_error_queue); 4517 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_receive_queue); 4518 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_backlog); 4519 4520 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rx_dst); 4521 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rx_dst_ifindex); 4522 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rx_dst_cookie); 4523 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rcvbuf); 4524 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_filter); 4525 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_wq); 4526 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_data_ready); 4527 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rcvtimeo); 4528 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rcvlowat); 4529 4530 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rxtx, sk_err); 4531 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rxtx, sk_socket); 4532 #ifdef CONFIG_MEMCG 4533 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rxtx, sk_memcg); 4534 #endif 4535 4536 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_lock); 4537 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_reserved_mem); 4538 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_forward_alloc); 4539 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_tsflags); 4540 4541 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_omem_alloc); 4542 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_omem_alloc); 4543 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_err_soft); 4544 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_wmem_queued); 4545 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_wmem_alloc); 4546 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_tsq_flags); 4547 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_send_head); 4548 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_write_queue); 4549 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_write_pending); 4550 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_frag); 4551 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_timer); 4552 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_pacing_rate); 4553 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_zckey); 4554 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_tskey); 4555 4556 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_dst_pending_confirm); 4557 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_pacing_status); 4558 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_max_pacing_rate); 4559 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_sndtimeo); 4560 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_priority); 4561 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_mark); 4562 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_uid); 4563 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_protocol); 4564 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_dst_cache); 4565 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_route_caps); 4566 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_gso_type); 4567 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_gso_max_size); 4568 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_allocation); 4569 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_txhash); 4570 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_sndbuf); 4571 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_gso_max_segs); 4572 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_pacing_shift); 4573 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_use_task_frag); 4574 return 0; 4575 } 4576 4577 core_initcall(sock_struct_check); 4578