1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * Generic socket support routines. Memory allocators, socket lock/release 8 * handler for protocols to use and generic option handler. 9 * 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Florian La Roche, <flla@stud.uni-sb.de> 13 * Alan Cox, <A.Cox@swansea.ac.uk> 14 * 15 * Fixes: 16 * Alan Cox : Numerous verify_area() problems 17 * Alan Cox : Connecting on a connecting socket 18 * now returns an error for tcp. 19 * Alan Cox : sock->protocol is set correctly. 20 * and is not sometimes left as 0. 21 * Alan Cox : connect handles icmp errors on a 22 * connect properly. Unfortunately there 23 * is a restart syscall nasty there. I 24 * can't match BSD without hacking the C 25 * library. Ideas urgently sought! 26 * Alan Cox : Disallow bind() to addresses that are 27 * not ours - especially broadcast ones!! 28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost) 29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets, 30 * instead they leave that for the DESTROY timer. 31 * Alan Cox : Clean up error flag in accept 32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer 33 * was buggy. Put a remove_sock() in the handler 34 * for memory when we hit 0. Also altered the timer 35 * code. The ACK stuff can wait and needs major 36 * TCP layer surgery. 37 * Alan Cox : Fixed TCP ack bug, removed remove sock 38 * and fixed timer/inet_bh race. 39 * Alan Cox : Added zapped flag for TCP 40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code 41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb 42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources 43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing. 44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so... 45 * Rick Sladkey : Relaxed UDP rules for matching packets. 46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support 47 * Pauline Middelink : identd support 48 * Alan Cox : Fixed connect() taking signals I think. 49 * Alan Cox : SO_LINGER supported 50 * Alan Cox : Error reporting fixes 51 * Anonymous : inet_create tidied up (sk->reuse setting) 52 * Alan Cox : inet sockets don't set sk->type! 53 * Alan Cox : Split socket option code 54 * Alan Cox : Callbacks 55 * Alan Cox : Nagle flag for Charles & Johannes stuff 56 * Alex : Removed restriction on inet fioctl 57 * Alan Cox : Splitting INET from NET core 58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt() 59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code 60 * Alan Cox : Split IP from generic code 61 * Alan Cox : New kfree_skbmem() 62 * Alan Cox : Make SO_DEBUG superuser only. 63 * Alan Cox : Allow anyone to clear SO_DEBUG 64 * (compatibility fix) 65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput. 66 * Alan Cox : Allocator for a socket is settable. 67 * Alan Cox : SO_ERROR includes soft errors. 68 * Alan Cox : Allow NULL arguments on some SO_ opts 69 * Alan Cox : Generic socket allocation to make hooks 70 * easier (suggested by Craig Metz). 71 * Michael Pall : SO_ERROR returns positive errno again 72 * Steve Whitehouse: Added default destructor to free 73 * protocol private data. 74 * Steve Whitehouse: Added various other default routines 75 * common to several socket families. 76 * Chris Evans : Call suser() check last on F_SETOWN 77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER. 78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s() 79 * Andi Kleen : Fix write_space callback 80 * Chris Evans : Security fixes - signedness again 81 * Arnaldo C. Melo : cleanups, use skb_queue_purge 82 * 83 * To Fix: 84 */ 85 86 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 87 88 #include <linux/unaligned.h> 89 #include <linux/capability.h> 90 #include <linux/errno.h> 91 #include <linux/errqueue.h> 92 #include <linux/types.h> 93 #include <linux/socket.h> 94 #include <linux/in.h> 95 #include <linux/kernel.h> 96 #include <linux/module.h> 97 #include <linux/proc_fs.h> 98 #include <linux/seq_file.h> 99 #include <linux/sched.h> 100 #include <linux/sched/mm.h> 101 #include <linux/timer.h> 102 #include <linux/string.h> 103 #include <linux/sockios.h> 104 #include <linux/net.h> 105 #include <linux/mm.h> 106 #include <linux/slab.h> 107 #include <linux/interrupt.h> 108 #include <linux/poll.h> 109 #include <linux/tcp.h> 110 #include <linux/udp.h> 111 #include <linux/init.h> 112 #include <linux/highmem.h> 113 #include <linux/user_namespace.h> 114 #include <linux/static_key.h> 115 #include <linux/memcontrol.h> 116 #include <linux/prefetch.h> 117 #include <linux/compat.h> 118 #include <linux/mroute.h> 119 #include <linux/mroute6.h> 120 #include <linux/icmpv6.h> 121 122 #include <linux/uaccess.h> 123 124 #include <linux/netdevice.h> 125 #include <net/protocol.h> 126 #include <linux/skbuff.h> 127 #include <linux/skbuff_ref.h> 128 #include <net/net_namespace.h> 129 #include <net/request_sock.h> 130 #include <net/sock.h> 131 #include <net/proto_memory.h> 132 #include <net/page_pool/types.h> 133 #include <linux/net_tstamp.h> 134 #include <net/xfrm.h> 135 #include <linux/ipsec.h> 136 #include <net/cls_cgroup.h> 137 #include <net/netprio_cgroup.h> 138 #include <linux/sock_diag.h> 139 140 #include <linux/filter.h> 141 #include <net/sock_reuseport.h> 142 #include <net/bpf_sk_storage.h> 143 144 #include <trace/events/sock.h> 145 146 #include <net/psp.h> 147 #include <net/tcp.h> 148 #include <net/busy_poll.h> 149 #include <net/phonet/phonet.h> 150 151 #include <linux/ethtool.h> 152 153 #include <uapi/linux/pidfd.h> 154 155 #include "dev.h" 156 157 static DEFINE_MUTEX(proto_list_mutex); 158 static LIST_HEAD(proto_list); 159 160 static void sock_def_write_space_wfree(struct sock *sk, int wmem_alloc); 161 static void sock_def_write_space(struct sock *sk); 162 163 /** 164 * sk_ns_capable - General socket capability test 165 * @sk: Socket to use a capability on or through 166 * @user_ns: The user namespace of the capability to use 167 * @cap: The capability to use 168 * 169 * Test to see if the opener of the socket had when the socket was 170 * created and the current process has the capability @cap in the user 171 * namespace @user_ns. 172 */ 173 bool sk_ns_capable(const struct sock *sk, 174 struct user_namespace *user_ns, int cap) 175 { 176 return file_ns_capable(sk->sk_socket->file, user_ns, cap) && 177 ns_capable(user_ns, cap); 178 } 179 EXPORT_SYMBOL(sk_ns_capable); 180 181 /** 182 * sk_capable - Socket global capability test 183 * @sk: Socket to use a capability on or through 184 * @cap: The global capability to use 185 * 186 * Test to see if the opener of the socket had when the socket was 187 * created and the current process has the capability @cap in all user 188 * namespaces. 189 */ 190 bool sk_capable(const struct sock *sk, int cap) 191 { 192 return sk_ns_capable(sk, &init_user_ns, cap); 193 } 194 EXPORT_SYMBOL(sk_capable); 195 196 /** 197 * sk_net_capable - Network namespace socket capability test 198 * @sk: Socket to use a capability on or through 199 * @cap: The capability to use 200 * 201 * Test to see if the opener of the socket had when the socket was created 202 * and the current process has the capability @cap over the network namespace 203 * the socket is a member of. 204 */ 205 bool sk_net_capable(const struct sock *sk, int cap) 206 { 207 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap); 208 } 209 EXPORT_SYMBOL(sk_net_capable); 210 211 /* 212 * Each address family might have different locking rules, so we have 213 * one slock key per address family and separate keys for internal and 214 * userspace sockets. 215 */ 216 static struct lock_class_key af_family_keys[AF_MAX]; 217 static struct lock_class_key af_family_kern_keys[AF_MAX]; 218 static struct lock_class_key af_family_slock_keys[AF_MAX]; 219 static struct lock_class_key af_family_kern_slock_keys[AF_MAX]; 220 221 /* 222 * Make lock validator output more readable. (we pre-construct these 223 * strings build-time, so that runtime initialization of socket 224 * locks is fast): 225 */ 226 227 #define _sock_locks(x) \ 228 x "AF_UNSPEC", x "AF_UNIX" , x "AF_INET" , \ 229 x "AF_AX25" , x "AF_IPX" , x "AF_APPLETALK", \ 230 x "AF_NETROM", x "AF_BRIDGE" , x "AF_ATMPVC" , \ 231 x "AF_X25" , x "AF_INET6" , x "AF_ROSE" , \ 232 x "AF_DECnet", x "AF_NETBEUI" , x "AF_SECURITY" , \ 233 x "AF_KEY" , x "AF_NETLINK" , x "AF_PACKET" , \ 234 x "AF_ASH" , x "AF_ECONET" , x "AF_ATMSVC" , \ 235 x "AF_RDS" , x "AF_SNA" , x "AF_IRDA" , \ 236 x "AF_PPPOX" , x "AF_WANPIPE" , x "AF_LLC" , \ 237 x "27" , x "28" , x "AF_CAN" , \ 238 x "AF_TIPC" , x "AF_BLUETOOTH", x "IUCV" , \ 239 x "AF_RXRPC" , x "AF_ISDN" , x "AF_PHONET" , \ 240 x "AF_IEEE802154", x "AF_CAIF" , x "AF_ALG" , \ 241 x "AF_NFC" , x "AF_VSOCK" , x "AF_KCM" , \ 242 x "AF_QIPCRTR", x "AF_SMC" , x "AF_XDP" , \ 243 x "AF_MCTP" , \ 244 x "AF_MAX" 245 246 static const char *const af_family_key_strings[AF_MAX+1] = { 247 _sock_locks("sk_lock-") 248 }; 249 static const char *const af_family_slock_key_strings[AF_MAX+1] = { 250 _sock_locks("slock-") 251 }; 252 static const char *const af_family_clock_key_strings[AF_MAX+1] = { 253 _sock_locks("clock-") 254 }; 255 256 static const char *const af_family_kern_key_strings[AF_MAX+1] = { 257 _sock_locks("k-sk_lock-") 258 }; 259 static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = { 260 _sock_locks("k-slock-") 261 }; 262 static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = { 263 _sock_locks("k-clock-") 264 }; 265 static const char *const af_family_rlock_key_strings[AF_MAX+1] = { 266 _sock_locks("rlock-") 267 }; 268 static const char *const af_family_wlock_key_strings[AF_MAX+1] = { 269 _sock_locks("wlock-") 270 }; 271 static const char *const af_family_elock_key_strings[AF_MAX+1] = { 272 _sock_locks("elock-") 273 }; 274 275 /* 276 * sk_callback_lock and sk queues locking rules are per-address-family, 277 * so split the lock classes by using a per-AF key: 278 */ 279 static struct lock_class_key af_callback_keys[AF_MAX]; 280 static struct lock_class_key af_rlock_keys[AF_MAX]; 281 static struct lock_class_key af_wlock_keys[AF_MAX]; 282 static struct lock_class_key af_elock_keys[AF_MAX]; 283 static struct lock_class_key af_kern_callback_keys[AF_MAX]; 284 285 /* Run time adjustable parameters. */ 286 __u32 sysctl_wmem_max __read_mostly = 4 << 20; 287 EXPORT_SYMBOL(sysctl_wmem_max); 288 __u32 sysctl_rmem_max __read_mostly = 4 << 20; 289 EXPORT_SYMBOL(sysctl_rmem_max); 290 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_DEFAULT; 291 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_DEFAULT; 292 293 DEFINE_STATIC_KEY_FALSE(memalloc_socks_key); 294 EXPORT_SYMBOL_GPL(memalloc_socks_key); 295 296 /** 297 * sk_set_memalloc - sets %SOCK_MEMALLOC 298 * @sk: socket to set it on 299 * 300 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves. 301 * It's the responsibility of the admin to adjust min_free_kbytes 302 * to meet the requirements 303 */ 304 void sk_set_memalloc(struct sock *sk) 305 { 306 sock_set_flag(sk, SOCK_MEMALLOC); 307 sk->sk_allocation |= __GFP_MEMALLOC; 308 static_branch_inc(&memalloc_socks_key); 309 } 310 EXPORT_SYMBOL_GPL(sk_set_memalloc); 311 312 void sk_clear_memalloc(struct sock *sk) 313 { 314 sock_reset_flag(sk, SOCK_MEMALLOC); 315 sk->sk_allocation &= ~__GFP_MEMALLOC; 316 static_branch_dec(&memalloc_socks_key); 317 318 /* 319 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward 320 * progress of swapping. SOCK_MEMALLOC may be cleared while 321 * it has rmem allocations due to the last swapfile being deactivated 322 * but there is a risk that the socket is unusable due to exceeding 323 * the rmem limits. Reclaim the reserves and obey rmem limits again. 324 */ 325 sk_mem_reclaim(sk); 326 } 327 EXPORT_SYMBOL_GPL(sk_clear_memalloc); 328 329 int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb) 330 { 331 int ret; 332 unsigned int noreclaim_flag; 333 334 /* these should have been dropped before queueing */ 335 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC)); 336 337 noreclaim_flag = memalloc_noreclaim_save(); 338 ret = INDIRECT_CALL_INET(sk->sk_backlog_rcv, 339 tcp_v6_do_rcv, 340 tcp_v4_do_rcv, 341 sk, skb); 342 memalloc_noreclaim_restore(noreclaim_flag); 343 344 return ret; 345 } 346 EXPORT_SYMBOL(__sk_backlog_rcv); 347 348 void sk_error_report(struct sock *sk) 349 { 350 sk->sk_error_report(sk); 351 352 switch (sk->sk_family) { 353 case AF_INET: 354 fallthrough; 355 case AF_INET6: 356 trace_inet_sk_error_report(sk); 357 break; 358 default: 359 break; 360 } 361 } 362 EXPORT_SYMBOL(sk_error_report); 363 364 int sock_get_timeout(long timeo, void *optval, bool old_timeval) 365 { 366 struct __kernel_sock_timeval tv; 367 368 if (timeo == MAX_SCHEDULE_TIMEOUT) { 369 tv.tv_sec = 0; 370 tv.tv_usec = 0; 371 } else { 372 tv.tv_sec = timeo / HZ; 373 tv.tv_usec = ((timeo % HZ) * USEC_PER_SEC) / HZ; 374 } 375 376 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) { 377 struct old_timeval32 tv32 = { tv.tv_sec, tv.tv_usec }; 378 *(struct old_timeval32 *)optval = tv32; 379 return sizeof(tv32); 380 } 381 382 if (old_timeval) { 383 struct __kernel_old_timeval old_tv; 384 old_tv.tv_sec = tv.tv_sec; 385 old_tv.tv_usec = tv.tv_usec; 386 *(struct __kernel_old_timeval *)optval = old_tv; 387 return sizeof(old_tv); 388 } 389 390 *(struct __kernel_sock_timeval *)optval = tv; 391 return sizeof(tv); 392 } 393 EXPORT_SYMBOL(sock_get_timeout); 394 395 int sock_copy_user_timeval(struct __kernel_sock_timeval *tv, 396 sockptr_t optval, int optlen, bool old_timeval) 397 { 398 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) { 399 struct old_timeval32 tv32; 400 401 if (optlen < sizeof(tv32)) 402 return -EINVAL; 403 404 if (copy_from_sockptr(&tv32, optval, sizeof(tv32))) 405 return -EFAULT; 406 tv->tv_sec = tv32.tv_sec; 407 tv->tv_usec = tv32.tv_usec; 408 } else if (old_timeval) { 409 struct __kernel_old_timeval old_tv; 410 411 if (optlen < sizeof(old_tv)) 412 return -EINVAL; 413 if (copy_from_sockptr(&old_tv, optval, sizeof(old_tv))) 414 return -EFAULT; 415 tv->tv_sec = old_tv.tv_sec; 416 tv->tv_usec = old_tv.tv_usec; 417 } else { 418 if (optlen < sizeof(*tv)) 419 return -EINVAL; 420 if (copy_from_sockptr(tv, optval, sizeof(*tv))) 421 return -EFAULT; 422 } 423 424 return 0; 425 } 426 EXPORT_SYMBOL(sock_copy_user_timeval); 427 428 static int sock_set_timeout(long *timeo_p, sockptr_t optval, int optlen, 429 bool old_timeval) 430 { 431 struct __kernel_sock_timeval tv; 432 int err = sock_copy_user_timeval(&tv, optval, optlen, old_timeval); 433 long val; 434 435 if (err) 436 return err; 437 438 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC) 439 return -EDOM; 440 441 if (tv.tv_sec < 0) { 442 static int warned __read_mostly; 443 444 WRITE_ONCE(*timeo_p, 0); 445 if (warned < 10 && net_ratelimit()) { 446 warned++; 447 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n", 448 __func__, current->comm, task_pid_nr(current)); 449 } 450 return 0; 451 } 452 val = MAX_SCHEDULE_TIMEOUT; 453 if ((tv.tv_sec || tv.tv_usec) && 454 (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT / HZ - 1))) 455 val = tv.tv_sec * HZ + DIV_ROUND_UP((unsigned long)tv.tv_usec, 456 USEC_PER_SEC / HZ); 457 WRITE_ONCE(*timeo_p, val); 458 return 0; 459 } 460 461 static bool sk_set_prio_allowed(const struct sock *sk, int val) 462 { 463 return ((val >= TC_PRIO_BESTEFFORT && val <= TC_PRIO_INTERACTIVE) || 464 sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) || 465 sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)); 466 } 467 468 static bool sock_needs_netstamp(const struct sock *sk) 469 { 470 switch (sk->sk_family) { 471 case AF_UNSPEC: 472 case AF_UNIX: 473 return false; 474 default: 475 return true; 476 } 477 } 478 479 static void sock_disable_timestamp(struct sock *sk, unsigned long flags) 480 { 481 if (sk->sk_flags & flags) { 482 sk->sk_flags &= ~flags; 483 if (sock_needs_netstamp(sk) && 484 !(sk->sk_flags & SK_FLAGS_TIMESTAMP)) 485 net_disable_timestamp(); 486 } 487 } 488 489 490 int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 491 { 492 unsigned long flags; 493 struct sk_buff_head *list = &sk->sk_receive_queue; 494 495 if (atomic_read(&sk->sk_rmem_alloc) >= READ_ONCE(sk->sk_rcvbuf)) { 496 sk_drops_inc(sk); 497 trace_sock_rcvqueue_full(sk, skb); 498 return -ENOMEM; 499 } 500 501 if (!sk_rmem_schedule(sk, skb, skb->truesize)) { 502 sk_drops_inc(sk); 503 return -ENOBUFS; 504 } 505 506 skb->dev = NULL; 507 skb_set_owner_r(skb, sk); 508 509 /* we escape from rcu protected region, make sure we dont leak 510 * a norefcounted dst 511 */ 512 skb_dst_force(skb); 513 514 spin_lock_irqsave(&list->lock, flags); 515 sock_skb_set_dropcount(sk, skb); 516 __skb_queue_tail(list, skb); 517 spin_unlock_irqrestore(&list->lock, flags); 518 519 if (!sock_flag(sk, SOCK_DEAD)) 520 sk->sk_data_ready(sk); 521 return 0; 522 } 523 EXPORT_SYMBOL(__sock_queue_rcv_skb); 524 525 enum skb_drop_reason 526 sock_queue_rcv_skb_reason(struct sock *sk, struct sk_buff *skb) 527 { 528 enum skb_drop_reason drop_reason; 529 int err; 530 531 drop_reason = sk_filter_reason(sk, skb); 532 if (drop_reason) 533 return drop_reason; 534 535 err = __sock_queue_rcv_skb(sk, skb); 536 switch (err) { 537 case -ENOMEM: 538 return SKB_DROP_REASON_SOCKET_RCVBUFF; 539 case -ENOBUFS: 540 return SKB_DROP_REASON_PROTO_MEM; 541 } 542 return SKB_NOT_DROPPED_YET; 543 } 544 EXPORT_SYMBOL(sock_queue_rcv_skb_reason); 545 546 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, 547 const int nested, unsigned int trim_cap, bool refcounted) 548 { 549 enum skb_drop_reason reason; 550 int rc = NET_RX_SUCCESS; 551 int err; 552 553 reason = sk_filter_trim_cap(sk, skb, trim_cap); 554 if (reason) 555 goto discard_and_relse; 556 557 skb->dev = NULL; 558 559 if (sk_rcvqueues_full(sk, READ_ONCE(sk->sk_rcvbuf))) { 560 sk_drops_inc(sk); 561 reason = SKB_DROP_REASON_SOCKET_RCVBUFF; 562 goto discard_and_relse; 563 } 564 if (nested) 565 bh_lock_sock_nested(sk); 566 else 567 bh_lock_sock(sk); 568 if (!sock_owned_by_user(sk)) { 569 /* 570 * trylock + unlock semantics: 571 */ 572 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_); 573 574 rc = sk_backlog_rcv(sk, skb); 575 576 mutex_release(&sk->sk_lock.dep_map, _RET_IP_); 577 } else if ((err = sk_add_backlog(sk, skb, READ_ONCE(sk->sk_rcvbuf)))) { 578 bh_unlock_sock(sk); 579 if (err == -ENOMEM) 580 reason = SKB_DROP_REASON_PFMEMALLOC; 581 if (err == -ENOBUFS) 582 reason = SKB_DROP_REASON_SOCKET_BACKLOG; 583 sk_drops_inc(sk); 584 goto discard_and_relse; 585 } 586 587 bh_unlock_sock(sk); 588 out: 589 if (refcounted) 590 sock_put(sk); 591 return rc; 592 discard_and_relse: 593 sk_skb_reason_drop(sk, skb, reason); 594 goto out; 595 } 596 EXPORT_SYMBOL(__sk_receive_skb); 597 598 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ip6_dst_check(struct dst_entry *, 599 u32)); 600 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *, 601 u32)); 602 struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie) 603 { 604 struct dst_entry *dst = __sk_dst_get(sk); 605 606 if (dst && READ_ONCE(dst->obsolete) && 607 INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check, 608 dst, cookie) == NULL) { 609 sk_tx_queue_clear(sk); 610 WRITE_ONCE(sk->sk_dst_pending_confirm, 0); 611 RCU_INIT_POINTER(sk->sk_dst_cache, NULL); 612 dst_release(dst); 613 return NULL; 614 } 615 616 return dst; 617 } 618 EXPORT_SYMBOL(__sk_dst_check); 619 620 struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie) 621 { 622 struct dst_entry *dst = sk_dst_get(sk); 623 624 if (dst && READ_ONCE(dst->obsolete) && 625 INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check, 626 dst, cookie) == NULL) { 627 sk_dst_reset(sk); 628 dst_release(dst); 629 return NULL; 630 } 631 632 return dst; 633 } 634 EXPORT_SYMBOL(sk_dst_check); 635 636 static int sock_bindtoindex_locked(struct sock *sk, int ifindex) 637 { 638 int ret = -ENOPROTOOPT; 639 #ifdef CONFIG_NETDEVICES 640 struct net *net = sock_net(sk); 641 642 /* Sorry... */ 643 ret = -EPERM; 644 if (sk->sk_bound_dev_if && !ns_capable(net->user_ns, CAP_NET_RAW)) 645 goto out; 646 647 ret = -EINVAL; 648 if (ifindex < 0) 649 goto out; 650 651 /* Paired with all READ_ONCE() done locklessly. */ 652 WRITE_ONCE(sk->sk_bound_dev_if, ifindex); 653 654 if (sk->sk_prot->rehash) 655 sk->sk_prot->rehash(sk); 656 sk_dst_reset(sk); 657 658 ret = 0; 659 660 out: 661 #endif 662 663 return ret; 664 } 665 666 int sock_bindtoindex(struct sock *sk, int ifindex, bool lock_sk) 667 { 668 int ret; 669 670 if (lock_sk) 671 lock_sock(sk); 672 ret = sock_bindtoindex_locked(sk, ifindex); 673 if (lock_sk) 674 release_sock(sk); 675 676 return ret; 677 } 678 EXPORT_SYMBOL(sock_bindtoindex); 679 680 static int sock_setbindtodevice(struct sock *sk, sockptr_t optval, int optlen) 681 { 682 int ret = -ENOPROTOOPT; 683 #ifdef CONFIG_NETDEVICES 684 struct net *net = sock_net(sk); 685 char devname[IFNAMSIZ]; 686 int index; 687 688 ret = -EINVAL; 689 if (optlen < 0) 690 goto out; 691 692 /* Bind this socket to a particular device like "eth0", 693 * as specified in the passed interface name. If the 694 * name is "" or the option length is zero the socket 695 * is not bound. 696 */ 697 if (optlen > IFNAMSIZ - 1) 698 optlen = IFNAMSIZ - 1; 699 memset(devname, 0, sizeof(devname)); 700 701 ret = -EFAULT; 702 if (copy_from_sockptr(devname, optval, optlen)) 703 goto out; 704 705 index = 0; 706 if (devname[0] != '\0') { 707 struct net_device *dev; 708 709 rcu_read_lock(); 710 dev = dev_get_by_name_rcu(net, devname); 711 if (dev) 712 index = dev->ifindex; 713 rcu_read_unlock(); 714 ret = -ENODEV; 715 if (!dev) 716 goto out; 717 } 718 719 sockopt_lock_sock(sk); 720 ret = sock_bindtoindex_locked(sk, index); 721 sockopt_release_sock(sk); 722 out: 723 #endif 724 725 return ret; 726 } 727 728 static int sock_getbindtodevice(struct sock *sk, sockptr_t optval, 729 sockptr_t optlen, int len) 730 { 731 int ret = -ENOPROTOOPT; 732 #ifdef CONFIG_NETDEVICES 733 int bound_dev_if = READ_ONCE(sk->sk_bound_dev_if); 734 struct net *net = sock_net(sk); 735 char devname[IFNAMSIZ]; 736 737 if (bound_dev_if == 0) { 738 len = 0; 739 goto zero; 740 } 741 742 ret = -EINVAL; 743 if (len < IFNAMSIZ) 744 goto out; 745 746 ret = netdev_get_name(net, devname, bound_dev_if); 747 if (ret) 748 goto out; 749 750 len = strlen(devname) + 1; 751 752 ret = -EFAULT; 753 if (copy_to_sockptr(optval, devname, len)) 754 goto out; 755 756 zero: 757 ret = -EFAULT; 758 if (copy_to_sockptr(optlen, &len, sizeof(int))) 759 goto out; 760 761 ret = 0; 762 763 out: 764 #endif 765 766 return ret; 767 } 768 769 bool sk_mc_loop(const struct sock *sk) 770 { 771 if (dev_recursion_level()) 772 return false; 773 sk = sk_const_to_full_sk(sk); 774 if (!sk) 775 return true; 776 777 switch (READ_ONCE(sk->sk_family)) { 778 case AF_INET: 779 return inet_test_bit(MC_LOOP, sk); 780 #if IS_ENABLED(CONFIG_IPV6) 781 case AF_INET6: 782 return inet6_test_bit(MC6_LOOP, sk); 783 #endif 784 } 785 return true; 786 } 787 EXPORT_SYMBOL(sk_mc_loop); 788 789 void sock_set_reuseaddr(struct sock *sk) 790 { 791 lock_sock(sk); 792 sk->sk_reuse = SK_CAN_REUSE; 793 release_sock(sk); 794 } 795 EXPORT_SYMBOL(sock_set_reuseaddr); 796 797 void sock_set_reuseport(struct sock *sk) 798 { 799 lock_sock(sk); 800 sk->sk_reuseport = true; 801 release_sock(sk); 802 } 803 EXPORT_SYMBOL(sock_set_reuseport); 804 805 void sock_no_linger(struct sock *sk) 806 { 807 lock_sock(sk); 808 WRITE_ONCE(sk->sk_lingertime, 0); 809 sock_set_flag(sk, SOCK_LINGER); 810 release_sock(sk); 811 } 812 EXPORT_SYMBOL(sock_no_linger); 813 814 void sock_set_priority(struct sock *sk, u32 priority) 815 { 816 WRITE_ONCE(sk->sk_priority, priority); 817 } 818 EXPORT_SYMBOL(sock_set_priority); 819 820 void sock_set_sndtimeo(struct sock *sk, s64 secs) 821 { 822 if (secs && secs < MAX_SCHEDULE_TIMEOUT / HZ - 1) 823 WRITE_ONCE(sk->sk_sndtimeo, secs * HZ); 824 else 825 WRITE_ONCE(sk->sk_sndtimeo, MAX_SCHEDULE_TIMEOUT); 826 } 827 EXPORT_SYMBOL(sock_set_sndtimeo); 828 829 static void __sock_set_timestamps(struct sock *sk, bool val, bool new, bool ns) 830 { 831 sock_valbool_flag(sk, SOCK_RCVTSTAMP, val); 832 sock_valbool_flag(sk, SOCK_RCVTSTAMPNS, val && ns); 833 if (val) { 834 sock_valbool_flag(sk, SOCK_TSTAMP_NEW, new); 835 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 836 } 837 } 838 839 void sock_set_timestamp(struct sock *sk, int optname, bool valbool) 840 { 841 switch (optname) { 842 case SO_TIMESTAMP_OLD: 843 __sock_set_timestamps(sk, valbool, false, false); 844 break; 845 case SO_TIMESTAMP_NEW: 846 __sock_set_timestamps(sk, valbool, true, false); 847 break; 848 case SO_TIMESTAMPNS_OLD: 849 __sock_set_timestamps(sk, valbool, false, true); 850 break; 851 case SO_TIMESTAMPNS_NEW: 852 __sock_set_timestamps(sk, valbool, true, true); 853 break; 854 } 855 } 856 857 static int sock_timestamping_bind_phc(struct sock *sk, int phc_index) 858 { 859 struct net *net = sock_net(sk); 860 struct net_device *dev = NULL; 861 bool match = false; 862 int *vclock_index; 863 int i, num; 864 865 if (sk->sk_bound_dev_if) 866 dev = dev_get_by_index(net, sk->sk_bound_dev_if); 867 868 if (!dev) { 869 pr_err("%s: sock not bind to device\n", __func__); 870 return -EOPNOTSUPP; 871 } 872 873 num = ethtool_get_phc_vclocks(dev, &vclock_index); 874 dev_put(dev); 875 876 for (i = 0; i < num; i++) { 877 if (*(vclock_index + i) == phc_index) { 878 match = true; 879 break; 880 } 881 } 882 883 if (num > 0) 884 kfree(vclock_index); 885 886 if (!match) 887 return -EINVAL; 888 889 WRITE_ONCE(sk->sk_bind_phc, phc_index); 890 891 return 0; 892 } 893 894 int sock_set_timestamping(struct sock *sk, int optname, 895 struct so_timestamping timestamping) 896 { 897 int val = timestamping.flags; 898 int ret; 899 900 if (val & ~SOF_TIMESTAMPING_MASK) 901 return -EINVAL; 902 903 if (val & SOF_TIMESTAMPING_OPT_ID_TCP && 904 !(val & SOF_TIMESTAMPING_OPT_ID)) 905 return -EINVAL; 906 907 if (val & SOF_TIMESTAMPING_OPT_ID && 908 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) { 909 if (sk_is_tcp(sk)) { 910 if ((1 << sk->sk_state) & 911 (TCPF_CLOSE | TCPF_LISTEN)) 912 return -EINVAL; 913 if (val & SOF_TIMESTAMPING_OPT_ID_TCP) 914 atomic_set(&sk->sk_tskey, tcp_sk(sk)->write_seq); 915 else 916 atomic_set(&sk->sk_tskey, tcp_sk(sk)->snd_una); 917 } else { 918 atomic_set(&sk->sk_tskey, 0); 919 } 920 } 921 922 if (val & SOF_TIMESTAMPING_OPT_STATS && 923 !(val & SOF_TIMESTAMPING_OPT_TSONLY)) 924 return -EINVAL; 925 926 if (val & SOF_TIMESTAMPING_BIND_PHC) { 927 ret = sock_timestamping_bind_phc(sk, timestamping.bind_phc); 928 if (ret) 929 return ret; 930 } 931 932 WRITE_ONCE(sk->sk_tsflags, val); 933 sock_valbool_flag(sk, SOCK_TSTAMP_NEW, optname == SO_TIMESTAMPING_NEW); 934 sock_valbool_flag(sk, SOCK_TIMESTAMPING_ANY, !!(val & TSFLAGS_ANY)); 935 936 if (val & SOF_TIMESTAMPING_RX_SOFTWARE) 937 sock_enable_timestamp(sk, 938 SOCK_TIMESTAMPING_RX_SOFTWARE); 939 else 940 sock_disable_timestamp(sk, 941 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE)); 942 return 0; 943 } 944 945 #if defined(CONFIG_CGROUP_BPF) 946 void bpf_skops_tx_timestamping(struct sock *sk, struct sk_buff *skb, int op) 947 { 948 struct bpf_sock_ops_kern sock_ops; 949 950 memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); 951 sock_ops.op = op; 952 sock_ops.is_fullsock = 1; 953 sock_ops.sk = sk; 954 bpf_skops_init_skb(&sock_ops, skb, 0); 955 __cgroup_bpf_run_filter_sock_ops(sk, &sock_ops, CGROUP_SOCK_OPS); 956 } 957 #endif 958 959 void sock_set_keepalive(struct sock *sk) 960 { 961 lock_sock(sk); 962 if (sk->sk_prot->keepalive) 963 sk->sk_prot->keepalive(sk, true); 964 sock_valbool_flag(sk, SOCK_KEEPOPEN, true); 965 release_sock(sk); 966 } 967 EXPORT_SYMBOL(sock_set_keepalive); 968 969 static void __sock_set_rcvbuf(struct sock *sk, int val) 970 { 971 struct socket *sock = sk->sk_socket; 972 973 /* Ensure val * 2 fits into an int, to prevent max_t() from treating it 974 * as a negative value. 975 */ 976 val = min_t(int, val, INT_MAX / 2); 977 sk->sk_userlocks |= SOCK_RCVBUF_LOCK; 978 979 /* We double it on the way in to account for "struct sk_buff" etc. 980 * overhead. Applications assume that the SO_RCVBUF setting they make 981 * will allow that much actual data to be received on that socket. 982 * 983 * Applications are unaware that "struct sk_buff" and other overheads 984 * allocate from the receive buffer during socket buffer allocation. 985 * 986 * And after considering the possible alternatives, returning the value 987 * we actually used in getsockopt is the most desirable behavior. 988 */ 989 WRITE_ONCE(sk->sk_rcvbuf, max_t(int, val * 2, SOCK_MIN_RCVBUF)); 990 991 if (sock) { 992 const struct proto_ops *ops = READ_ONCE(sock->ops); 993 994 if (ops->set_rcvbuf) 995 ops->set_rcvbuf(sk, sk->sk_rcvbuf); 996 } 997 } 998 999 void sock_set_rcvbuf(struct sock *sk, int val) 1000 { 1001 lock_sock(sk); 1002 __sock_set_rcvbuf(sk, val); 1003 release_sock(sk); 1004 } 1005 EXPORT_SYMBOL(sock_set_rcvbuf); 1006 1007 static void __sock_set_mark(struct sock *sk, u32 val) 1008 { 1009 if (val != sk->sk_mark) { 1010 WRITE_ONCE(sk->sk_mark, val); 1011 sk_dst_reset(sk); 1012 } 1013 } 1014 1015 void sock_set_mark(struct sock *sk, u32 val) 1016 { 1017 lock_sock(sk); 1018 __sock_set_mark(sk, val); 1019 release_sock(sk); 1020 } 1021 EXPORT_SYMBOL(sock_set_mark); 1022 1023 static void sock_release_reserved_memory(struct sock *sk, int bytes) 1024 { 1025 /* Round down bytes to multiple of pages */ 1026 bytes = round_down(bytes, PAGE_SIZE); 1027 1028 WARN_ON(bytes > sk->sk_reserved_mem); 1029 WRITE_ONCE(sk->sk_reserved_mem, sk->sk_reserved_mem - bytes); 1030 sk_mem_reclaim(sk); 1031 } 1032 1033 static int sock_reserve_memory(struct sock *sk, int bytes) 1034 { 1035 long allocated; 1036 bool charged; 1037 int pages; 1038 1039 if (!mem_cgroup_sk_enabled(sk) || !sk_is_tcp(sk)) 1040 return -EOPNOTSUPP; 1041 1042 if (!bytes) 1043 return 0; 1044 1045 pages = sk_mem_pages(bytes); 1046 1047 /* pre-charge to memcg */ 1048 charged = mem_cgroup_sk_charge(sk, pages, 1049 GFP_KERNEL | __GFP_RETRY_MAYFAIL); 1050 if (!charged) 1051 return -ENOMEM; 1052 1053 if (sk->sk_bypass_prot_mem) 1054 goto success; 1055 1056 /* pre-charge to forward_alloc */ 1057 sk_memory_allocated_add(sk, pages); 1058 allocated = sk_memory_allocated(sk); 1059 1060 /* If the system goes into memory pressure with this 1061 * precharge, give up and return error. 1062 */ 1063 if (allocated > sk_prot_mem_limits(sk, 1)) { 1064 sk_memory_allocated_sub(sk, pages); 1065 mem_cgroup_sk_uncharge(sk, pages); 1066 return -ENOMEM; 1067 } 1068 1069 success: 1070 sk_forward_alloc_add(sk, pages << PAGE_SHIFT); 1071 1072 WRITE_ONCE(sk->sk_reserved_mem, 1073 sk->sk_reserved_mem + (pages << PAGE_SHIFT)); 1074 1075 return 0; 1076 } 1077 1078 #ifdef CONFIG_PAGE_POOL 1079 1080 /* This is the number of tokens and frags that the user can SO_DEVMEM_DONTNEED 1081 * in 1 syscall. The limit exists to limit the amount of memory the kernel 1082 * allocates to copy these tokens, and to prevent looping over the frags for 1083 * too long. 1084 */ 1085 #define MAX_DONTNEED_TOKENS 128 1086 #define MAX_DONTNEED_FRAGS 1024 1087 1088 static noinline_for_stack int 1089 sock_devmem_dontneed(struct sock *sk, sockptr_t optval, unsigned int optlen) 1090 { 1091 unsigned int num_tokens, i, j, netmem_num = 0; 1092 struct dmabuf_token *tokens; 1093 int ret = 0, num_frags = 0; 1094 netmem_ref netmems[16]; 1095 1096 if (!sk_is_tcp(sk)) 1097 return -EBADF; 1098 1099 if (optlen % sizeof(*tokens) || 1100 optlen > sizeof(*tokens) * MAX_DONTNEED_TOKENS) 1101 return -EINVAL; 1102 1103 num_tokens = optlen / sizeof(*tokens); 1104 tokens = kvmalloc_objs(*tokens, num_tokens); 1105 if (!tokens) 1106 return -ENOMEM; 1107 1108 if (copy_from_sockptr(tokens, optval, optlen)) { 1109 kvfree(tokens); 1110 return -EFAULT; 1111 } 1112 1113 xa_lock_bh(&sk->sk_user_frags); 1114 for (i = 0; i < num_tokens; i++) { 1115 for (j = 0; j < tokens[i].token_count; j++) { 1116 if (++num_frags > MAX_DONTNEED_FRAGS) 1117 goto frag_limit_reached; 1118 1119 netmem_ref netmem = (__force netmem_ref)__xa_erase( 1120 &sk->sk_user_frags, tokens[i].token_start + j); 1121 1122 if (!netmem || WARN_ON_ONCE(!netmem_is_net_iov(netmem))) 1123 continue; 1124 1125 netmems[netmem_num++] = netmem; 1126 if (netmem_num == ARRAY_SIZE(netmems)) { 1127 xa_unlock_bh(&sk->sk_user_frags); 1128 page_pool_put_netmem_bulk(netmems, 1129 ARRAY_SIZE(netmems)); 1130 netmem_num = 0; 1131 xa_lock_bh(&sk->sk_user_frags); 1132 } 1133 ret++; 1134 } 1135 } 1136 1137 frag_limit_reached: 1138 xa_unlock_bh(&sk->sk_user_frags); 1139 page_pool_put_netmem_bulk(netmems, netmem_num); 1140 1141 kvfree(tokens); 1142 return ret; 1143 } 1144 #endif 1145 1146 void sockopt_lock_sock(struct sock *sk) 1147 { 1148 /* When current->bpf_ctx is set, the setsockopt is called from 1149 * a bpf prog. bpf has ensured the sk lock has been 1150 * acquired before calling setsockopt(). 1151 */ 1152 if (has_current_bpf_ctx()) 1153 return; 1154 1155 lock_sock(sk); 1156 } 1157 EXPORT_SYMBOL(sockopt_lock_sock); 1158 1159 void sockopt_release_sock(struct sock *sk) 1160 { 1161 if (has_current_bpf_ctx()) 1162 return; 1163 1164 release_sock(sk); 1165 } 1166 EXPORT_SYMBOL(sockopt_release_sock); 1167 1168 bool sockopt_ns_capable(struct user_namespace *ns, int cap) 1169 { 1170 return has_current_bpf_ctx() || ns_capable(ns, cap); 1171 } 1172 EXPORT_SYMBOL(sockopt_ns_capable); 1173 1174 bool sockopt_capable(int cap) 1175 { 1176 return has_current_bpf_ctx() || capable(cap); 1177 } 1178 EXPORT_SYMBOL(sockopt_capable); 1179 1180 static int sockopt_validate_clockid(__kernel_clockid_t value) 1181 { 1182 switch (value) { 1183 case CLOCK_REALTIME: 1184 case CLOCK_MONOTONIC: 1185 case CLOCK_TAI: 1186 return 0; 1187 } 1188 return -EINVAL; 1189 } 1190 1191 /* 1192 * This is meant for all protocols to use and covers goings on 1193 * at the socket level. Everything here is generic. 1194 */ 1195 1196 int sk_setsockopt(struct sock *sk, int level, int optname, 1197 sockptr_t optval, unsigned int optlen) 1198 { 1199 struct so_timestamping timestamping; 1200 struct socket *sock = sk->sk_socket; 1201 struct sock_txtime sk_txtime; 1202 int val; 1203 int valbool; 1204 struct linger ling; 1205 int ret = 0; 1206 1207 /* 1208 * Options without arguments 1209 */ 1210 1211 if (optname == SO_BINDTODEVICE) 1212 return sock_setbindtodevice(sk, optval, optlen); 1213 1214 if (optlen < sizeof(int)) 1215 return -EINVAL; 1216 1217 if (copy_from_sockptr(&val, optval, sizeof(val))) 1218 return -EFAULT; 1219 1220 valbool = val ? 1 : 0; 1221 1222 /* handle options which do not require locking the socket. */ 1223 switch (optname) { 1224 case SO_PRIORITY: 1225 if (sk_set_prio_allowed(sk, val)) { 1226 sock_set_priority(sk, val); 1227 return 0; 1228 } 1229 return -EPERM; 1230 case SO_TYPE: 1231 case SO_PROTOCOL: 1232 case SO_DOMAIN: 1233 case SO_ERROR: 1234 return -ENOPROTOOPT; 1235 #ifdef CONFIG_NET_RX_BUSY_POLL 1236 case SO_BUSY_POLL: 1237 if (val < 0) 1238 return -EINVAL; 1239 WRITE_ONCE(sk->sk_ll_usec, val); 1240 return 0; 1241 case SO_PREFER_BUSY_POLL: 1242 if (valbool && !sockopt_capable(CAP_NET_ADMIN)) 1243 return -EPERM; 1244 WRITE_ONCE(sk->sk_prefer_busy_poll, valbool); 1245 return 0; 1246 case SO_BUSY_POLL_BUDGET: 1247 if (val > READ_ONCE(sk->sk_busy_poll_budget) && 1248 !sockopt_capable(CAP_NET_ADMIN)) 1249 return -EPERM; 1250 if (val < 0 || val > U16_MAX) 1251 return -EINVAL; 1252 WRITE_ONCE(sk->sk_busy_poll_budget, val); 1253 return 0; 1254 #endif 1255 case SO_MAX_PACING_RATE: 1256 { 1257 unsigned long ulval = (val == ~0U) ? ~0UL : (unsigned int)val; 1258 unsigned long pacing_rate; 1259 1260 if (sizeof(ulval) != sizeof(val) && 1261 optlen >= sizeof(ulval) && 1262 copy_from_sockptr(&ulval, optval, sizeof(ulval))) { 1263 return -EFAULT; 1264 } 1265 if (ulval != ~0UL) 1266 cmpxchg(&sk->sk_pacing_status, 1267 SK_PACING_NONE, 1268 SK_PACING_NEEDED); 1269 /* Pairs with READ_ONCE() from sk_getsockopt() */ 1270 WRITE_ONCE(sk->sk_max_pacing_rate, ulval); 1271 pacing_rate = READ_ONCE(sk->sk_pacing_rate); 1272 if (ulval < pacing_rate) 1273 WRITE_ONCE(sk->sk_pacing_rate, ulval); 1274 return 0; 1275 } 1276 case SO_TXREHASH: 1277 if (!sk_is_tcp(sk)) 1278 return -EOPNOTSUPP; 1279 if (val < -1 || val > 1) 1280 return -EINVAL; 1281 if ((u8)val == SOCK_TXREHASH_DEFAULT) 1282 val = READ_ONCE(sock_net(sk)->core.sysctl_txrehash); 1283 /* Paired with READ_ONCE() in tcp_rtx_synack() 1284 * and sk_getsockopt(). 1285 */ 1286 WRITE_ONCE(sk->sk_txrehash, (u8)val); 1287 return 0; 1288 case SO_PEEK_OFF: 1289 { 1290 int (*set_peek_off)(struct sock *sk, int val); 1291 1292 set_peek_off = READ_ONCE(sock->ops)->set_peek_off; 1293 if (set_peek_off) 1294 ret = set_peek_off(sk, val); 1295 else 1296 ret = -EOPNOTSUPP; 1297 return ret; 1298 } 1299 #ifdef CONFIG_PAGE_POOL 1300 case SO_DEVMEM_DONTNEED: 1301 return sock_devmem_dontneed(sk, optval, optlen); 1302 #endif 1303 case SO_SNDTIMEO_OLD: 1304 case SO_SNDTIMEO_NEW: 1305 return sock_set_timeout(&sk->sk_sndtimeo, optval, 1306 optlen, optname == SO_SNDTIMEO_OLD); 1307 case SO_RCVTIMEO_OLD: 1308 case SO_RCVTIMEO_NEW: 1309 return sock_set_timeout(&sk->sk_rcvtimeo, optval, 1310 optlen, optname == SO_RCVTIMEO_OLD); 1311 } 1312 1313 sockopt_lock_sock(sk); 1314 1315 switch (optname) { 1316 case SO_DEBUG: 1317 if (val && !sockopt_capable(CAP_NET_ADMIN)) 1318 ret = -EACCES; 1319 else 1320 sock_valbool_flag(sk, SOCK_DBG, valbool); 1321 break; 1322 case SO_REUSEADDR: 1323 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE); 1324 break; 1325 case SO_REUSEPORT: 1326 if (valbool && !sk_is_inet(sk)) 1327 ret = -EOPNOTSUPP; 1328 else 1329 sk->sk_reuseport = valbool; 1330 break; 1331 case SO_DONTROUTE: 1332 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool); 1333 sk_dst_reset(sk); 1334 break; 1335 case SO_BROADCAST: 1336 sock_valbool_flag(sk, SOCK_BROADCAST, valbool); 1337 break; 1338 case SO_SNDBUF: 1339 /* Don't error on this BSD doesn't and if you think 1340 * about it this is right. Otherwise apps have to 1341 * play 'guess the biggest size' games. RCVBUF/SNDBUF 1342 * are treated in BSD as hints 1343 */ 1344 val = min_t(u32, val, READ_ONCE(sysctl_wmem_max)); 1345 set_sndbuf: 1346 /* Ensure val * 2 fits into an int, to prevent max_t() 1347 * from treating it as a negative value. 1348 */ 1349 val = min_t(int, val, INT_MAX / 2); 1350 sk->sk_userlocks |= SOCK_SNDBUF_LOCK; 1351 WRITE_ONCE(sk->sk_sndbuf, 1352 max_t(int, val * 2, SOCK_MIN_SNDBUF)); 1353 /* Wake up sending tasks if we upped the value. */ 1354 sk->sk_write_space(sk); 1355 break; 1356 1357 case SO_SNDBUFFORCE: 1358 if (!sockopt_capable(CAP_NET_ADMIN)) { 1359 ret = -EPERM; 1360 break; 1361 } 1362 1363 /* No negative values (to prevent underflow, as val will be 1364 * multiplied by 2). 1365 */ 1366 if (val < 0) 1367 val = 0; 1368 goto set_sndbuf; 1369 1370 case SO_RCVBUF: 1371 /* Don't error on this BSD doesn't and if you think 1372 * about it this is right. Otherwise apps have to 1373 * play 'guess the biggest size' games. RCVBUF/SNDBUF 1374 * are treated in BSD as hints 1375 */ 1376 __sock_set_rcvbuf(sk, min_t(u32, val, READ_ONCE(sysctl_rmem_max))); 1377 break; 1378 1379 case SO_RCVBUFFORCE: 1380 if (!sockopt_capable(CAP_NET_ADMIN)) { 1381 ret = -EPERM; 1382 break; 1383 } 1384 1385 /* No negative values (to prevent underflow, as val will be 1386 * multiplied by 2). 1387 */ 1388 __sock_set_rcvbuf(sk, max(val, 0)); 1389 break; 1390 1391 case SO_KEEPALIVE: 1392 if (sk->sk_prot->keepalive) 1393 sk->sk_prot->keepalive(sk, valbool); 1394 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool); 1395 break; 1396 1397 case SO_OOBINLINE: 1398 sock_valbool_flag(sk, SOCK_URGINLINE, valbool); 1399 break; 1400 1401 case SO_NO_CHECK: 1402 sk->sk_no_check_tx = valbool; 1403 break; 1404 1405 case SO_LINGER: 1406 if (optlen < sizeof(ling)) { 1407 ret = -EINVAL; /* 1003.1g */ 1408 break; 1409 } 1410 if (copy_from_sockptr(&ling, optval, sizeof(ling))) { 1411 ret = -EFAULT; 1412 break; 1413 } 1414 if (!ling.l_onoff) { 1415 sock_reset_flag(sk, SOCK_LINGER); 1416 } else { 1417 unsigned long t_sec = ling.l_linger; 1418 1419 if (t_sec >= MAX_SCHEDULE_TIMEOUT / HZ) 1420 WRITE_ONCE(sk->sk_lingertime, MAX_SCHEDULE_TIMEOUT); 1421 else 1422 WRITE_ONCE(sk->sk_lingertime, t_sec * HZ); 1423 sock_set_flag(sk, SOCK_LINGER); 1424 } 1425 break; 1426 1427 case SO_BSDCOMPAT: 1428 break; 1429 1430 case SO_TIMESTAMP_OLD: 1431 case SO_TIMESTAMP_NEW: 1432 case SO_TIMESTAMPNS_OLD: 1433 case SO_TIMESTAMPNS_NEW: 1434 sock_set_timestamp(sk, optname, valbool); 1435 break; 1436 1437 case SO_TIMESTAMPING_NEW: 1438 case SO_TIMESTAMPING_OLD: 1439 if (optlen == sizeof(timestamping)) { 1440 if (copy_from_sockptr(×tamping, optval, 1441 sizeof(timestamping))) { 1442 ret = -EFAULT; 1443 break; 1444 } 1445 } else { 1446 memset(×tamping, 0, sizeof(timestamping)); 1447 timestamping.flags = val; 1448 } 1449 ret = sock_set_timestamping(sk, optname, timestamping); 1450 break; 1451 1452 case SO_RCVLOWAT: 1453 { 1454 int (*set_rcvlowat)(struct sock *sk, int val) = NULL; 1455 1456 if (val < 0) 1457 val = INT_MAX; 1458 if (sock) 1459 set_rcvlowat = READ_ONCE(sock->ops)->set_rcvlowat; 1460 if (set_rcvlowat) 1461 ret = set_rcvlowat(sk, val); 1462 else 1463 WRITE_ONCE(sk->sk_rcvlowat, val ? : 1); 1464 break; 1465 } 1466 case SO_ATTACH_FILTER: { 1467 struct sock_fprog fprog; 1468 1469 if (sk_is_tcp(sk) && 1470 !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1471 ret = -EPERM; 1472 break; 1473 } 1474 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen); 1475 if (!ret) 1476 ret = sk_attach_filter(&fprog, sk); 1477 break; 1478 } 1479 case SO_ATTACH_BPF: 1480 ret = -EINVAL; 1481 if (optlen == sizeof(u32)) { 1482 u32 ufd; 1483 1484 ret = -EFAULT; 1485 if (copy_from_sockptr(&ufd, optval, sizeof(ufd))) 1486 break; 1487 1488 ret = sk_attach_bpf(ufd, sk); 1489 } 1490 break; 1491 1492 case SO_ATTACH_REUSEPORT_CBPF: { 1493 struct sock_fprog fprog; 1494 1495 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen); 1496 if (!ret) 1497 ret = sk_reuseport_attach_filter(&fprog, sk); 1498 break; 1499 } 1500 case SO_ATTACH_REUSEPORT_EBPF: 1501 ret = -EINVAL; 1502 if (optlen == sizeof(u32)) { 1503 u32 ufd; 1504 1505 ret = -EFAULT; 1506 if (copy_from_sockptr(&ufd, optval, sizeof(ufd))) 1507 break; 1508 1509 ret = sk_reuseport_attach_bpf(ufd, sk); 1510 } 1511 break; 1512 1513 case SO_DETACH_REUSEPORT_BPF: 1514 ret = reuseport_detach_prog(sk); 1515 break; 1516 1517 case SO_DETACH_FILTER: 1518 ret = sk_detach_filter(sk); 1519 break; 1520 1521 case SO_LOCK_FILTER: 1522 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool) 1523 ret = -EPERM; 1524 else 1525 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool); 1526 break; 1527 1528 case SO_MARK: 1529 if (!sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && 1530 !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1531 ret = -EPERM; 1532 break; 1533 } 1534 1535 __sock_set_mark(sk, val); 1536 break; 1537 case SO_RCVMARK: 1538 sock_valbool_flag(sk, SOCK_RCVMARK, valbool); 1539 break; 1540 1541 case SO_RCVPRIORITY: 1542 sock_valbool_flag(sk, SOCK_RCVPRIORITY, valbool); 1543 break; 1544 1545 case SO_RXQ_OVFL: 1546 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool); 1547 break; 1548 1549 case SO_WIFI_STATUS: 1550 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool); 1551 break; 1552 1553 case SO_NOFCS: 1554 sock_valbool_flag(sk, SOCK_NOFCS, valbool); 1555 break; 1556 1557 case SO_SELECT_ERR_QUEUE: 1558 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool); 1559 break; 1560 1561 case SO_PASSCRED: 1562 if (sk_may_scm_recv(sk)) 1563 sk->sk_scm_credentials = valbool; 1564 else 1565 ret = -EOPNOTSUPP; 1566 break; 1567 1568 case SO_PASSSEC: 1569 if (IS_ENABLED(CONFIG_SECURITY_NETWORK) && sk_may_scm_recv(sk)) 1570 sk->sk_scm_security = valbool; 1571 else 1572 ret = -EOPNOTSUPP; 1573 break; 1574 1575 case SO_PASSPIDFD: 1576 if (sk_is_unix(sk)) 1577 sk->sk_scm_pidfd = valbool; 1578 else 1579 ret = -EOPNOTSUPP; 1580 break; 1581 1582 case SO_PASSPIDFD_THREAD: 1583 if (sk_is_unix(sk)) 1584 sk->sk_scm_pidfd_thread = valbool; 1585 else 1586 ret = -EOPNOTSUPP; 1587 break; 1588 1589 case SO_PASSRIGHTS: 1590 if (sk_is_unix(sk)) 1591 sk->sk_scm_rights = valbool; 1592 else 1593 ret = -EOPNOTSUPP; 1594 break; 1595 1596 case SO_INCOMING_CPU: 1597 reuseport_update_incoming_cpu(sk, val); 1598 break; 1599 1600 case SO_CNX_ADVICE: 1601 if (val == 1) 1602 dst_negative_advice(sk); 1603 break; 1604 1605 case SO_ZEROCOPY: 1606 if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) { 1607 if (!(sk_is_tcp(sk) || 1608 (sk->sk_type == SOCK_DGRAM && 1609 sk->sk_protocol == IPPROTO_UDP))) 1610 ret = -EOPNOTSUPP; 1611 } else if (sk->sk_family != PF_RDS) { 1612 ret = -EOPNOTSUPP; 1613 } 1614 if (!ret) { 1615 if (val < 0 || val > 1) 1616 ret = -EINVAL; 1617 else 1618 sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool); 1619 } 1620 break; 1621 1622 case SO_TXTIME: 1623 if (optlen != sizeof(struct sock_txtime)) { 1624 ret = -EINVAL; 1625 break; 1626 } else if (copy_from_sockptr(&sk_txtime, optval, 1627 sizeof(struct sock_txtime))) { 1628 ret = -EFAULT; 1629 break; 1630 } else if (sk_txtime.flags & ~SOF_TXTIME_FLAGS_MASK) { 1631 ret = -EINVAL; 1632 break; 1633 } 1634 /* CLOCK_MONOTONIC is only used by sch_fq, and this packet 1635 * scheduler has enough safe guards. 1636 */ 1637 if (sk_txtime.clockid != CLOCK_MONOTONIC && 1638 !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1639 ret = -EPERM; 1640 break; 1641 } 1642 1643 ret = sockopt_validate_clockid(sk_txtime.clockid); 1644 if (ret) 1645 break; 1646 1647 sock_valbool_flag(sk, SOCK_TXTIME, true); 1648 sk->sk_clockid = sk_txtime.clockid; 1649 sk->sk_txtime_deadline_mode = 1650 !!(sk_txtime.flags & SOF_TXTIME_DEADLINE_MODE); 1651 sk->sk_txtime_report_errors = 1652 !!(sk_txtime.flags & SOF_TXTIME_REPORT_ERRORS); 1653 break; 1654 1655 case SO_BINDTOIFINDEX: 1656 ret = sock_bindtoindex_locked(sk, val); 1657 break; 1658 1659 case SO_BUF_LOCK: 1660 if (val & ~SOCK_BUF_LOCK_MASK) { 1661 ret = -EINVAL; 1662 break; 1663 } 1664 sk->sk_userlocks = val | (sk->sk_userlocks & 1665 ~SOCK_BUF_LOCK_MASK); 1666 break; 1667 1668 case SO_RESERVE_MEM: 1669 { 1670 int delta; 1671 1672 if (val < 0 || val > SZ_1G) { 1673 ret = -EINVAL; 1674 break; 1675 } 1676 1677 delta = val - sk->sk_reserved_mem; 1678 if (delta < 0) 1679 sock_release_reserved_memory(sk, -delta); 1680 else 1681 ret = sock_reserve_memory(sk, delta); 1682 break; 1683 } 1684 1685 default: 1686 ret = -ENOPROTOOPT; 1687 break; 1688 } 1689 sockopt_release_sock(sk); 1690 return ret; 1691 } 1692 1693 int sock_setsockopt(struct socket *sock, int level, int optname, 1694 sockptr_t optval, unsigned int optlen) 1695 { 1696 return sk_setsockopt(sock->sk, level, optname, 1697 optval, optlen); 1698 } 1699 EXPORT_SYMBOL(sock_setsockopt); 1700 1701 static const struct cred *sk_get_peer_cred(struct sock *sk) 1702 { 1703 const struct cred *cred; 1704 1705 spin_lock(&sk->sk_peer_lock); 1706 cred = get_cred(sk->sk_peer_cred); 1707 spin_unlock(&sk->sk_peer_lock); 1708 1709 return cred; 1710 } 1711 1712 static void cred_to_ucred(struct pid *pid, const struct cred *cred, 1713 struct ucred *ucred) 1714 { 1715 ucred->pid = pid_vnr(pid); 1716 ucred->uid = ucred->gid = -1; 1717 if (cred) { 1718 struct user_namespace *current_ns = current_user_ns(); 1719 1720 ucred->uid = from_kuid_munged(current_ns, cred->euid); 1721 ucred->gid = from_kgid_munged(current_ns, cred->egid); 1722 } 1723 } 1724 1725 static int groups_to_user(sockptr_t dst, const struct group_info *src) 1726 { 1727 struct user_namespace *user_ns = current_user_ns(); 1728 int i; 1729 1730 for (i = 0; i < src->ngroups; i++) { 1731 gid_t gid = from_kgid_munged(user_ns, src->gid[i]); 1732 1733 if (copy_to_sockptr_offset(dst, i * sizeof(gid), &gid, sizeof(gid))) 1734 return -EFAULT; 1735 } 1736 1737 return 0; 1738 } 1739 1740 /* Hand out a pidfd for @type of the socket's peer via SO_PEERPIDFD*. */ 1741 static int sk_getsockopt_peerpidfd(struct sock *sk, enum pid_type type, 1742 sockptr_t optval, sockptr_t optlen, int len) 1743 { 1744 struct file *pidfd_file = NULL; 1745 unsigned int flags = 0; 1746 struct pid *peer_pid; 1747 int pidfd; 1748 1749 if (len > sizeof(pidfd)) 1750 len = sizeof(pidfd); 1751 1752 spin_lock(&sk->sk_peer_lock); 1753 peer_pid = get_pid(sk->sk_peer_pid[type]); 1754 spin_unlock(&sk->sk_peer_lock); 1755 1756 if (!peer_pid) 1757 return -ENODATA; 1758 1759 /* The use of PIDFD_STALE requires stashing of struct pid on pidfs 1760 * with pidfs_register_pid() and only AF_UNIX is prepared for this. 1761 */ 1762 if (sk_is_unix(sk)) 1763 flags |= PIDFD_STALE; 1764 if (type == PIDTYPE_PID) 1765 flags |= PIDFD_THREAD; 1766 1767 pidfd = pidfd_prepare(peer_pid, flags, &pidfd_file); 1768 put_pid(peer_pid); 1769 if (pidfd < 0) 1770 return pidfd; 1771 1772 if (copy_to_sockptr(optval, &pidfd, len) || 1773 copy_to_sockptr(optlen, &len, sizeof(int))) { 1774 put_unused_fd(pidfd); 1775 fput(pidfd_file); 1776 1777 return -EFAULT; 1778 } 1779 1780 fd_install(pidfd, pidfd_file); 1781 return 0; 1782 } 1783 1784 int sk_getsockopt(struct sock *sk, int level, int optname, 1785 sockptr_t optval, sockptr_t optlen) 1786 { 1787 struct socket *sock = sk->sk_socket; 1788 1789 union { 1790 int val; 1791 u64 val64; 1792 unsigned long ulval; 1793 struct linger ling; 1794 struct old_timeval32 tm32; 1795 struct __kernel_old_timeval tm; 1796 struct __kernel_sock_timeval stm; 1797 struct sock_txtime txtime; 1798 struct so_timestamping timestamping; 1799 } v; 1800 1801 int lv = sizeof(int); 1802 int len; 1803 1804 if (copy_from_sockptr(&len, optlen, sizeof(int))) 1805 return -EFAULT; 1806 if (len < 0) 1807 return -EINVAL; 1808 1809 memset(&v, 0, sizeof(v)); 1810 1811 switch (optname) { 1812 case SO_DEBUG: 1813 v.val = sock_flag(sk, SOCK_DBG); 1814 break; 1815 1816 case SO_DONTROUTE: 1817 v.val = sock_flag(sk, SOCK_LOCALROUTE); 1818 break; 1819 1820 case SO_BROADCAST: 1821 v.val = sock_flag(sk, SOCK_BROADCAST); 1822 break; 1823 1824 case SO_SNDBUF: 1825 v.val = READ_ONCE(sk->sk_sndbuf); 1826 break; 1827 1828 case SO_RCVBUF: 1829 v.val = READ_ONCE(sk->sk_rcvbuf); 1830 break; 1831 1832 case SO_REUSEADDR: 1833 v.val = sk->sk_reuse; 1834 break; 1835 1836 case SO_REUSEPORT: 1837 v.val = sk->sk_reuseport; 1838 break; 1839 1840 case SO_KEEPALIVE: 1841 v.val = sock_flag(sk, SOCK_KEEPOPEN); 1842 break; 1843 1844 case SO_TYPE: 1845 v.val = sk->sk_type; 1846 break; 1847 1848 case SO_PROTOCOL: 1849 v.val = sk->sk_protocol; 1850 break; 1851 1852 case SO_DOMAIN: 1853 v.val = sk->sk_family; 1854 break; 1855 1856 case SO_ERROR: 1857 v.val = -sock_error(sk); 1858 if (v.val == 0) 1859 v.val = xchg(&sk->sk_err_soft, 0); 1860 break; 1861 1862 case SO_OOBINLINE: 1863 v.val = sock_flag(sk, SOCK_URGINLINE); 1864 break; 1865 1866 case SO_NO_CHECK: 1867 v.val = sk->sk_no_check_tx; 1868 break; 1869 1870 case SO_PRIORITY: 1871 v.val = READ_ONCE(sk->sk_priority); 1872 break; 1873 1874 case SO_LINGER: 1875 lv = sizeof(v.ling); 1876 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER); 1877 v.ling.l_linger = READ_ONCE(sk->sk_lingertime) / HZ; 1878 break; 1879 1880 case SO_BSDCOMPAT: 1881 break; 1882 1883 case SO_TIMESTAMP_OLD: 1884 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && 1885 !sock_flag(sk, SOCK_TSTAMP_NEW) && 1886 !sock_flag(sk, SOCK_RCVTSTAMPNS); 1887 break; 1888 1889 case SO_TIMESTAMPNS_OLD: 1890 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && !sock_flag(sk, SOCK_TSTAMP_NEW); 1891 break; 1892 1893 case SO_TIMESTAMP_NEW: 1894 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && sock_flag(sk, SOCK_TSTAMP_NEW); 1895 break; 1896 1897 case SO_TIMESTAMPNS_NEW: 1898 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && sock_flag(sk, SOCK_TSTAMP_NEW); 1899 break; 1900 1901 case SO_TIMESTAMPING_OLD: 1902 case SO_TIMESTAMPING_NEW: 1903 lv = sizeof(v.timestamping); 1904 /* For the later-added case SO_TIMESTAMPING_NEW: Be strict about only 1905 * returning the flags when they were set through the same option. 1906 * Don't change the beviour for the old case SO_TIMESTAMPING_OLD. 1907 */ 1908 if (optname == SO_TIMESTAMPING_OLD || sock_flag(sk, SOCK_TSTAMP_NEW)) { 1909 v.timestamping.flags = READ_ONCE(sk->sk_tsflags); 1910 v.timestamping.bind_phc = READ_ONCE(sk->sk_bind_phc); 1911 } 1912 break; 1913 1914 case SO_RCVTIMEO_OLD: 1915 case SO_RCVTIMEO_NEW: 1916 lv = sock_get_timeout(READ_ONCE(sk->sk_rcvtimeo), &v, 1917 SO_RCVTIMEO_OLD == optname); 1918 break; 1919 1920 case SO_SNDTIMEO_OLD: 1921 case SO_SNDTIMEO_NEW: 1922 lv = sock_get_timeout(READ_ONCE(sk->sk_sndtimeo), &v, 1923 SO_SNDTIMEO_OLD == optname); 1924 break; 1925 1926 case SO_RCVLOWAT: 1927 v.val = READ_ONCE(sk->sk_rcvlowat); 1928 break; 1929 1930 case SO_SNDLOWAT: 1931 v.val = 1; 1932 break; 1933 1934 case SO_PASSCRED: 1935 if (!sk_may_scm_recv(sk)) 1936 return -EOPNOTSUPP; 1937 1938 v.val = sk->sk_scm_credentials; 1939 break; 1940 1941 case SO_PASSPIDFD: 1942 if (!sk_is_unix(sk)) 1943 return -EOPNOTSUPP; 1944 1945 v.val = sk->sk_scm_pidfd; 1946 break; 1947 1948 case SO_PASSPIDFD_THREAD: 1949 if (!sk_is_unix(sk)) 1950 return -EOPNOTSUPP; 1951 1952 v.val = sk->sk_scm_pidfd_thread; 1953 break; 1954 1955 case SO_PASSRIGHTS: 1956 if (!sk_is_unix(sk)) 1957 return -EOPNOTSUPP; 1958 1959 v.val = sk->sk_scm_rights; 1960 break; 1961 1962 case SO_PEERCRED: 1963 { 1964 struct ucred peercred; 1965 if (len > sizeof(peercred)) 1966 len = sizeof(peercred); 1967 1968 spin_lock(&sk->sk_peer_lock); 1969 cred_to_ucred(sk->sk_peer_pid[PIDTYPE_TGID], sk->sk_peer_cred, 1970 &peercred); 1971 spin_unlock(&sk->sk_peer_lock); 1972 1973 if (copy_to_sockptr(optval, &peercred, len)) 1974 return -EFAULT; 1975 goto lenout; 1976 } 1977 1978 case SO_PEERPIDFD: 1979 return sk_getsockopt_peerpidfd(sk, PIDTYPE_TGID, optval, optlen, len); 1980 1981 case SO_PEERPIDFD_THREAD: 1982 /* Only AF_UNIX records the peer's connecting thread. */ 1983 if (!sk_is_unix(sk)) 1984 return -EOPNOTSUPP; 1985 1986 return sk_getsockopt_peerpidfd(sk, PIDTYPE_PID, optval, optlen, len); 1987 1988 case SO_PEERGROUPS: 1989 { 1990 const struct cred *cred; 1991 int ret, n; 1992 1993 cred = sk_get_peer_cred(sk); 1994 if (!cred) 1995 return -ENODATA; 1996 1997 n = cred->group_info->ngroups; 1998 if (len < n * sizeof(gid_t)) { 1999 len = n * sizeof(gid_t); 2000 put_cred(cred); 2001 return copy_to_sockptr(optlen, &len, sizeof(int)) ? -EFAULT : -ERANGE; 2002 } 2003 len = n * sizeof(gid_t); 2004 2005 ret = groups_to_user(optval, cred->group_info); 2006 put_cred(cred); 2007 if (ret) 2008 return ret; 2009 goto lenout; 2010 } 2011 2012 case SO_PEERNAME: 2013 { 2014 struct sockaddr_storage address; 2015 2016 lv = READ_ONCE(sock->ops)->getname(sock, (struct sockaddr *)&address, 2); 2017 if (lv < 0) 2018 return -ENOTCONN; 2019 if (lv < len) 2020 return -EINVAL; 2021 if (copy_to_sockptr(optval, &address, len)) 2022 return -EFAULT; 2023 goto lenout; 2024 } 2025 2026 /* Dubious BSD thing... Probably nobody even uses it, but 2027 * the UNIX standard wants it for whatever reason... -DaveM 2028 */ 2029 case SO_ACCEPTCONN: 2030 v.val = sk->sk_state == TCP_LISTEN; 2031 break; 2032 2033 case SO_PASSSEC: 2034 if (!IS_ENABLED(CONFIG_SECURITY_NETWORK) || !sk_may_scm_recv(sk)) 2035 return -EOPNOTSUPP; 2036 2037 v.val = sk->sk_scm_security; 2038 break; 2039 2040 case SO_PEERSEC: 2041 return security_socket_getpeersec_stream(sock, 2042 optval, optlen, len); 2043 2044 case SO_MARK: 2045 v.val = READ_ONCE(sk->sk_mark); 2046 break; 2047 2048 case SO_RCVMARK: 2049 v.val = sock_flag(sk, SOCK_RCVMARK); 2050 break; 2051 2052 case SO_RCVPRIORITY: 2053 v.val = sock_flag(sk, SOCK_RCVPRIORITY); 2054 break; 2055 2056 case SO_RXQ_OVFL: 2057 v.val = sock_flag(sk, SOCK_RXQ_OVFL); 2058 break; 2059 2060 case SO_WIFI_STATUS: 2061 v.val = sock_flag(sk, SOCK_WIFI_STATUS); 2062 break; 2063 2064 case SO_PEEK_OFF: 2065 if (!READ_ONCE(sock->ops)->set_peek_off) 2066 return -EOPNOTSUPP; 2067 2068 v.val = READ_ONCE(sk->sk_peek_off); 2069 break; 2070 case SO_NOFCS: 2071 v.val = sock_flag(sk, SOCK_NOFCS); 2072 break; 2073 2074 case SO_BINDTODEVICE: 2075 return sock_getbindtodevice(sk, optval, optlen, len); 2076 2077 case SO_GET_FILTER: 2078 len = sk_get_filter(sk, optval, len); 2079 if (len < 0) 2080 return len; 2081 2082 goto lenout; 2083 2084 case SO_LOCK_FILTER: 2085 v.val = sock_flag(sk, SOCK_FILTER_LOCKED); 2086 break; 2087 2088 case SO_BPF_EXTENSIONS: 2089 v.val = bpf_tell_extensions(); 2090 break; 2091 2092 case SO_SELECT_ERR_QUEUE: 2093 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE); 2094 break; 2095 2096 #ifdef CONFIG_NET_RX_BUSY_POLL 2097 case SO_BUSY_POLL: 2098 v.val = READ_ONCE(sk->sk_ll_usec); 2099 break; 2100 case SO_PREFER_BUSY_POLL: 2101 v.val = READ_ONCE(sk->sk_prefer_busy_poll); 2102 break; 2103 #endif 2104 2105 case SO_MAX_PACING_RATE: 2106 /* The READ_ONCE() pair with the WRITE_ONCE() in sk_setsockopt() */ 2107 if (sizeof(v.ulval) != sizeof(v.val) && len >= sizeof(v.ulval)) { 2108 lv = sizeof(v.ulval); 2109 v.ulval = READ_ONCE(sk->sk_max_pacing_rate); 2110 } else { 2111 /* 32bit version */ 2112 v.val = min_t(unsigned long, ~0U, 2113 READ_ONCE(sk->sk_max_pacing_rate)); 2114 } 2115 break; 2116 2117 case SO_INCOMING_CPU: 2118 v.val = READ_ONCE(sk->sk_incoming_cpu); 2119 break; 2120 2121 case SO_MEMINFO: 2122 { 2123 u32 meminfo[SK_MEMINFO_VARS]; 2124 2125 sk_get_meminfo(sk, meminfo); 2126 2127 len = min_t(unsigned int, len, sizeof(meminfo)); 2128 if (copy_to_sockptr(optval, &meminfo, len)) 2129 return -EFAULT; 2130 2131 goto lenout; 2132 } 2133 2134 #ifdef CONFIG_NET_RX_BUSY_POLL 2135 case SO_INCOMING_NAPI_ID: 2136 v.val = READ_ONCE(sk->sk_napi_id); 2137 2138 /* aggregate non-NAPI IDs down to 0 */ 2139 if (!napi_id_valid(v.val)) 2140 v.val = 0; 2141 2142 break; 2143 #endif 2144 2145 case SO_COOKIE: 2146 lv = sizeof(u64); 2147 if (len < lv) 2148 return -EINVAL; 2149 v.val64 = sock_gen_cookie(sk); 2150 break; 2151 2152 case SO_ZEROCOPY: 2153 v.val = sock_flag(sk, SOCK_ZEROCOPY); 2154 break; 2155 2156 case SO_TXTIME: 2157 lv = sizeof(v.txtime); 2158 v.txtime.clockid = sk->sk_clockid; 2159 v.txtime.flags |= sk->sk_txtime_deadline_mode ? 2160 SOF_TXTIME_DEADLINE_MODE : 0; 2161 v.txtime.flags |= sk->sk_txtime_report_errors ? 2162 SOF_TXTIME_REPORT_ERRORS : 0; 2163 break; 2164 2165 case SO_BINDTOIFINDEX: 2166 v.val = READ_ONCE(sk->sk_bound_dev_if); 2167 break; 2168 2169 case SO_NETNS_COOKIE: 2170 lv = sizeof(u64); 2171 if (len != lv) 2172 return -EINVAL; 2173 v.val64 = sock_net(sk)->net_cookie; 2174 break; 2175 2176 case SO_BUF_LOCK: 2177 v.val = sk->sk_userlocks & SOCK_BUF_LOCK_MASK; 2178 break; 2179 2180 case SO_RESERVE_MEM: 2181 v.val = READ_ONCE(sk->sk_reserved_mem); 2182 break; 2183 2184 case SO_TXREHASH: 2185 if (!sk_is_tcp(sk)) 2186 return -EOPNOTSUPP; 2187 2188 /* Paired with WRITE_ONCE() in sk_setsockopt() */ 2189 v.val = READ_ONCE(sk->sk_txrehash); 2190 break; 2191 2192 default: 2193 /* We implement the SO_SNDLOWAT etc to not be settable 2194 * (1003.1g 7). 2195 */ 2196 return -ENOPROTOOPT; 2197 } 2198 2199 if (len > lv) 2200 len = lv; 2201 if (copy_to_sockptr(optval, &v, len)) 2202 return -EFAULT; 2203 lenout: 2204 if (copy_to_sockptr(optlen, &len, sizeof(int))) 2205 return -EFAULT; 2206 return 0; 2207 } 2208 2209 /* 2210 * Initialize an sk_lock. 2211 * 2212 * (We also register the sk_lock with the lock validator.) 2213 */ 2214 static inline void sock_lock_init(struct sock *sk) 2215 { 2216 sk_owner_clear(sk); 2217 2218 if (sk->sk_kern_sock) 2219 sock_lock_init_class_and_name( 2220 sk, 2221 af_family_kern_slock_key_strings[sk->sk_family], 2222 af_family_kern_slock_keys + sk->sk_family, 2223 af_family_kern_key_strings[sk->sk_family], 2224 af_family_kern_keys + sk->sk_family); 2225 else 2226 sock_lock_init_class_and_name( 2227 sk, 2228 af_family_slock_key_strings[sk->sk_family], 2229 af_family_slock_keys + sk->sk_family, 2230 af_family_key_strings[sk->sk_family], 2231 af_family_keys + sk->sk_family); 2232 } 2233 2234 /* 2235 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet, 2236 * even temporarily, because of RCU lookups. sk_node should also be left as is. 2237 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end 2238 */ 2239 static void sock_copy(struct sock *nsk, const struct sock *osk) 2240 { 2241 const struct proto *prot = READ_ONCE(osk->sk_prot); 2242 #ifdef CONFIG_SECURITY_NETWORK 2243 void *sptr = nsk->sk_security; 2244 #endif 2245 2246 /* If we move sk_tx_queue_mapping out of the private section, 2247 * we must check if sk_tx_queue_clear() is called after 2248 * sock_copy() in sk_clone_lock(). 2249 */ 2250 BUILD_BUG_ON(offsetof(struct sock, sk_tx_queue_mapping) < 2251 offsetof(struct sock, sk_dontcopy_begin) || 2252 offsetof(struct sock, sk_tx_queue_mapping) >= 2253 offsetof(struct sock, sk_dontcopy_end)); 2254 2255 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin)); 2256 2257 unsafe_memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end, 2258 prot->obj_size - offsetof(struct sock, sk_dontcopy_end), 2259 /* alloc is larger than struct, see sk_prot_alloc() */); 2260 2261 #ifdef CONFIG_SECURITY_NETWORK 2262 nsk->sk_security = sptr; 2263 security_sk_clone(osk, nsk); 2264 #endif 2265 } 2266 2267 static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority, 2268 int family) 2269 { 2270 struct sock *sk; 2271 struct kmem_cache *slab; 2272 2273 slab = prot->slab; 2274 if (slab != NULL) { 2275 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO); 2276 if (!sk) 2277 return sk; 2278 if (want_init_on_alloc(priority)) 2279 sk_prot_clear_nulls(sk, prot->obj_size); 2280 } else 2281 sk = kmalloc(prot->obj_size, priority); 2282 2283 if (sk != NULL) { 2284 if (security_sk_alloc(sk, family, priority)) 2285 goto out_free; 2286 2287 if (!try_module_get(prot->owner)) 2288 goto out_free_sec; 2289 } 2290 2291 return sk; 2292 2293 out_free_sec: 2294 security_sk_free(sk); 2295 out_free: 2296 if (slab != NULL) 2297 kmem_cache_free(slab, sk); 2298 else 2299 kfree(sk); 2300 return NULL; 2301 } 2302 2303 static void sk_prot_free(struct proto *prot, struct sock *sk) 2304 { 2305 struct kmem_cache *slab; 2306 struct module *owner; 2307 2308 owner = prot->owner; 2309 slab = prot->slab; 2310 2311 cgroup_sk_free(&sk->sk_cgrp_data); 2312 mem_cgroup_sk_free(sk); 2313 security_sk_free(sk); 2314 2315 sk_owner_put(sk); 2316 2317 if (slab != NULL) 2318 kmem_cache_free(slab, sk); 2319 else 2320 kfree(sk); 2321 module_put(owner); 2322 } 2323 2324 /** 2325 * sk_alloc - All socket objects are allocated here 2326 * @net: the applicable net namespace 2327 * @family: protocol family 2328 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 2329 * @prot: struct proto associated with this new sock instance 2330 * @kern: is this to be a kernel socket? 2331 */ 2332 struct sock *sk_alloc(struct net *net, int family, gfp_t priority, 2333 struct proto *prot, int kern) 2334 { 2335 struct sock *sk; 2336 2337 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family); 2338 if (sk) { 2339 sk->sk_family = family; 2340 /* 2341 * See comment in struct sock definition to understand 2342 * why we need sk_prot_creator -acme 2343 */ 2344 sk->sk_prot = sk->sk_prot_creator = prot; 2345 2346 if (READ_ONCE(net->core.sysctl_bypass_prot_mem)) 2347 sk->sk_bypass_prot_mem = 1; 2348 2349 sk->sk_kern_sock = kern; 2350 sock_lock_init(sk); 2351 2352 sk->sk_net_refcnt = kern ? 0 : 1; 2353 if (likely(sk->sk_net_refcnt)) { 2354 get_net_track(net, &sk->ns_tracker, priority); 2355 sock_inuse_add(net, 1); 2356 } else { 2357 net_passive_inc(net); 2358 __netns_tracker_alloc(net, &sk->ns_tracker, 2359 false, priority); 2360 } 2361 2362 sock_net_set(sk, net); 2363 refcount_set(&sk->sk_wmem_alloc, SK_WMEM_ALLOC_BIAS); 2364 2365 mem_cgroup_sk_alloc(sk); 2366 cgroup_sk_alloc(&sk->sk_cgrp_data); 2367 sock_update_classid(&sk->sk_cgrp_data); 2368 sock_update_netprioidx(&sk->sk_cgrp_data); 2369 sk_tx_queue_clear(sk); 2370 } 2371 2372 return sk; 2373 } 2374 EXPORT_SYMBOL(sk_alloc); 2375 2376 /* Sockets having SOCK_RCU_FREE will call this function after one RCU 2377 * grace period. This is the case for UDP sockets and TCP listeners. 2378 */ 2379 static void __sk_destruct(struct rcu_head *head) 2380 { 2381 struct sock *sk = container_of(head, struct sock, sk_rcu); 2382 struct net *net = sock_net(sk); 2383 struct sk_filter *filter; 2384 2385 if (sk->sk_destruct) 2386 sk->sk_destruct(sk); 2387 2388 filter = rcu_dereference_check(sk->sk_filter, 2389 refcount_read(&sk->sk_wmem_alloc) == 0); 2390 if (filter) { 2391 sk_filter_uncharge(sk, filter); 2392 RCU_INIT_POINTER(sk->sk_filter, NULL); 2393 } 2394 2395 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP); 2396 2397 #ifdef CONFIG_BPF_SYSCALL 2398 bpf_sk_storage_free(sk); 2399 #endif 2400 2401 if (atomic_read(&sk->sk_omem_alloc)) 2402 pr_debug("%s: optmem leakage (%d bytes) detected\n", 2403 __func__, atomic_read(&sk->sk_omem_alloc)); 2404 2405 if (sk->sk_frag.page) { 2406 put_page(sk->sk_frag.page); 2407 sk->sk_frag.page = NULL; 2408 } 2409 2410 /* We do not need to acquire sk->sk_peer_lock, we are the last user. */ 2411 put_cred(sk->sk_peer_cred); 2412 put_pids(sk->sk_peer_pid); 2413 2414 if (likely(sk->sk_net_refcnt)) { 2415 put_net_track(net, &sk->ns_tracker); 2416 } else { 2417 __netns_tracker_free(net, &sk->ns_tracker, false); 2418 net_passive_dec(net); 2419 } 2420 sk_prot_free(sk->sk_prot_creator, sk); 2421 } 2422 2423 void sk_net_refcnt_upgrade(struct sock *sk) 2424 { 2425 struct net *net = sock_net(sk); 2426 2427 WARN_ON_ONCE(sk->sk_net_refcnt); 2428 __netns_tracker_free(net, &sk->ns_tracker, false); 2429 net_passive_dec(net); 2430 sk->sk_net_refcnt = 1; 2431 get_net_track(net, &sk->ns_tracker, GFP_KERNEL); 2432 sock_inuse_add(net, 1); 2433 } 2434 EXPORT_SYMBOL_GPL(sk_net_refcnt_upgrade); 2435 2436 void sk_destruct(struct sock *sk) 2437 { 2438 bool use_call_rcu = sock_flag(sk, SOCK_RCU_FREE); 2439 2440 if (rcu_access_pointer(sk->sk_reuseport_cb)) { 2441 reuseport_detach_sock(sk); 2442 use_call_rcu = true; 2443 } 2444 2445 if (use_call_rcu) 2446 call_rcu(&sk->sk_rcu, __sk_destruct); 2447 else 2448 __sk_destruct(&sk->sk_rcu); 2449 } 2450 2451 static void __sk_free(struct sock *sk) 2452 { 2453 if (likely(sk->sk_net_refcnt)) 2454 sock_inuse_add(sock_net(sk), -1); 2455 2456 if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk))) 2457 sock_diag_broadcast_destroy(sk); 2458 else 2459 sk_destruct(sk); 2460 } 2461 2462 void sk_free(struct sock *sk) 2463 { 2464 /* 2465 * We subtract one from sk_wmem_alloc and can know if 2466 * some packets are still in some tx queue. 2467 * If not null, sock_wfree() will call __sk_free(sk) later 2468 */ 2469 if (refcount_dec_and_test(&sk->sk_wmem_alloc)) 2470 __sk_free(sk); 2471 } 2472 EXPORT_SYMBOL(sk_free); 2473 2474 static void sk_init_common(struct sock *sk) 2475 { 2476 skb_queue_head_init(&sk->sk_receive_queue); 2477 skb_queue_head_init(&sk->sk_write_queue); 2478 skb_queue_head_init(&sk->sk_error_queue); 2479 2480 rwlock_init(&sk->sk_callback_lock); 2481 lockdep_set_class_and_name(&sk->sk_receive_queue.lock, 2482 af_rlock_keys + sk->sk_family, 2483 af_family_rlock_key_strings[sk->sk_family]); 2484 lockdep_set_class_and_name(&sk->sk_write_queue.lock, 2485 af_wlock_keys + sk->sk_family, 2486 af_family_wlock_key_strings[sk->sk_family]); 2487 lockdep_set_class_and_name(&sk->sk_error_queue.lock, 2488 af_elock_keys + sk->sk_family, 2489 af_family_elock_key_strings[sk->sk_family]); 2490 if (sk->sk_kern_sock) 2491 lockdep_set_class_and_name(&sk->sk_callback_lock, 2492 af_kern_callback_keys + sk->sk_family, 2493 af_family_kern_clock_key_strings[sk->sk_family]); 2494 else 2495 lockdep_set_class_and_name(&sk->sk_callback_lock, 2496 af_callback_keys + sk->sk_family, 2497 af_family_clock_key_strings[sk->sk_family]); 2498 } 2499 2500 /** 2501 * sk_clone - clone a socket 2502 * @sk: the socket to clone 2503 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 2504 * @lock: if true, lock the cloned sk 2505 * 2506 * If @lock is true, the clone is locked by bh_lock_sock(), and 2507 * caller must unlock socket even in error path by bh_unlock_sock(). 2508 */ 2509 struct sock *sk_clone(const struct sock *sk, const gfp_t priority, 2510 bool lock) 2511 { 2512 struct proto *prot = READ_ONCE(sk->sk_prot); 2513 struct sk_filter *filter; 2514 bool is_charged = true; 2515 struct sock *newsk; 2516 2517 newsk = sk_prot_alloc(prot, priority, sk->sk_family); 2518 if (!newsk) 2519 goto out; 2520 2521 sock_copy(newsk, sk); 2522 2523 newsk->sk_prot_creator = prot; 2524 #ifdef CONFIG_BPF_SYSCALL 2525 RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL); 2526 #endif 2527 #if IS_ENABLED(CONFIG_INET_PSP) 2528 DEBUG_NET_WARN_ON_ONCE(rcu_access_pointer(sk->psp_assoc)); 2529 #endif 2530 2531 /* SANITY */ 2532 if (likely(newsk->sk_net_refcnt)) { 2533 get_net_track(sock_net(newsk), &newsk->ns_tracker, priority); 2534 sock_inuse_add(sock_net(newsk), 1); 2535 } else { 2536 /* Kernel sockets are not elevating the struct net refcount. 2537 * Instead, use a tracker to more easily detect if a layer 2538 * is not properly dismantling its kernel sockets at netns 2539 * destroy time. 2540 */ 2541 net_passive_inc(sock_net(newsk)); 2542 __netns_tracker_alloc(sock_net(newsk), &newsk->ns_tracker, 2543 false, priority); 2544 } 2545 2546 sk_node_init(&newsk->sk_node); 2547 sock_lock_init(newsk); 2548 2549 if (lock) 2550 bh_lock_sock(newsk); 2551 2552 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL; 2553 newsk->sk_backlog.len = 0; 2554 2555 atomic_set(&newsk->sk_rmem_alloc, 0); 2556 2557 refcount_set(&newsk->sk_wmem_alloc, SK_WMEM_ALLOC_BIAS); 2558 2559 atomic_set(&newsk->sk_omem_alloc, 0); 2560 sk_init_common(newsk); 2561 2562 newsk->sk_dst_cache = NULL; 2563 newsk->sk_dst_pending_confirm = 0; 2564 newsk->sk_wmem_queued = 0; 2565 newsk->sk_forward_alloc = 0; 2566 newsk->sk_reserved_mem = 0; 2567 DEBUG_NET_WARN_ON_ONCE(newsk->sk_drop_counters); 2568 sk_drops_reset(newsk); 2569 newsk->sk_send_head = NULL; 2570 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK; 2571 atomic_set(&newsk->sk_zckey, 0); 2572 2573 sock_reset_flag(newsk, SOCK_DONE); 2574 2575 #ifdef CONFIG_MEMCG 2576 /* sk->sk_memcg will be populated at accept() time */ 2577 newsk->sk_memcg = NULL; 2578 #endif 2579 2580 cgroup_sk_clone(&newsk->sk_cgrp_data); 2581 2582 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL); 2583 2584 if (sock_needs_netstamp(sk) && newsk->sk_flags & SK_FLAGS_TIMESTAMP) 2585 net_enable_timestamp(); 2586 2587 rcu_read_lock(); 2588 filter = rcu_dereference(sk->sk_filter); 2589 if (filter != NULL) 2590 /* though it's an empty new sock, the charging may fail 2591 * if sysctl_optmem_max was changed between creation of 2592 * original socket and cloning 2593 */ 2594 is_charged = sk_filter_charge(newsk, filter); 2595 RCU_INIT_POINTER(newsk->sk_filter, filter); 2596 rcu_read_unlock(); 2597 2598 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) { 2599 /* We need to make sure that we don't uncharge the new 2600 * socket if we couldn't charge it in the first place 2601 * as otherwise we uncharge the parent's filter. 2602 */ 2603 if (!is_charged) 2604 RCU_INIT_POINTER(newsk->sk_filter, NULL); 2605 2606 goto free; 2607 } 2608 2609 if (bpf_sk_storage_clone(sk, newsk)) 2610 goto free; 2611 2612 /* Clear sk_user_data if parent had the pointer tagged 2613 * as not suitable for copying when cloning. 2614 */ 2615 if (sk_user_data_is_nocopy(newsk)) 2616 newsk->sk_user_data = NULL; 2617 2618 newsk->sk_err = 0; 2619 newsk->sk_err_soft = 0; 2620 newsk->sk_priority = 0; 2621 newsk->sk_incoming_cpu = raw_smp_processor_id(); 2622 2623 /* Before updating sk_refcnt, we must commit prior changes to memory 2624 * (Documentation/RCU/rculist_nulls.rst for details) 2625 */ 2626 smp_wmb(); 2627 refcount_set(&newsk->sk_refcnt, 2); 2628 2629 sk_set_socket(newsk, NULL); 2630 sk_tx_queue_clear(newsk); 2631 sk_rx_queue_clear(newsk); 2632 RCU_INIT_POINTER(newsk->sk_wq, NULL); 2633 2634 if (newsk->sk_prot->sockets_allocated) 2635 sk_sockets_allocated_inc(newsk); 2636 out: 2637 return newsk; 2638 free: 2639 /* It is still raw copy of parent, so invalidate 2640 * destructor and make plain sk_free() 2641 */ 2642 newsk->sk_destruct = NULL; 2643 if (lock) 2644 bh_unlock_sock(newsk); 2645 sk_free(newsk); 2646 newsk = NULL; 2647 goto out; 2648 } 2649 EXPORT_SYMBOL_GPL(sk_clone); 2650 2651 static u32 sk_dst_gso_max_size(struct sock *sk, const struct net_device *dev) 2652 { 2653 bool is_ipv6 = false; 2654 u32 max_size; 2655 2656 #if IS_ENABLED(CONFIG_IPV6) 2657 is_ipv6 = (sk->sk_family == AF_INET6 && 2658 !ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)); 2659 #endif 2660 /* pairs with the WRITE_ONCE() in netif_set_gso(_ipv4)_max_size() */ 2661 max_size = is_ipv6 ? READ_ONCE(dev->gso_max_size) : 2662 READ_ONCE(dev->gso_ipv4_max_size); 2663 if (max_size > GSO_LEGACY_MAX_SIZE && !sk_is_tcp(sk)) 2664 max_size = GSO_LEGACY_MAX_SIZE; 2665 2666 return max_size - (MAX_TCP_HEADER + 1); 2667 } 2668 2669 void sk_setup_caps(struct sock *sk, struct dst_entry *dst) 2670 { 2671 const struct net_device *dev; 2672 u32 max_segs = 1; 2673 2674 rcu_read_lock(); 2675 dev = dst_dev_rcu(dst); 2676 sk->sk_route_caps = dev->features; 2677 if (sk_is_tcp(sk)) { 2678 struct inet_connection_sock *icsk = inet_csk(sk); 2679 2680 sk->sk_route_caps |= NETIF_F_GSO; 2681 icsk->icsk_ack.dst_quick_ack = dst_metric(dst, RTAX_QUICKACK); 2682 } 2683 if (sk->sk_route_caps & NETIF_F_GSO) 2684 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE; 2685 if (unlikely(sk->sk_gso_disabled)) 2686 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 2687 if (sk_can_gso(sk)) { 2688 if (dst->header_len && !xfrm_dst_offload_ok(dst)) { 2689 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 2690 } else { 2691 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM; 2692 sk->sk_gso_max_size = sk_dst_gso_max_size(sk, dev); 2693 /* pairs with the WRITE_ONCE() in netif_set_gso_max_segs() */ 2694 max_segs = max_t(u32, READ_ONCE(dev->gso_max_segs), 1); 2695 } 2696 } 2697 sk->sk_gso_max_segs = max_segs; 2698 sk_dst_set(sk, dst); 2699 rcu_read_unlock(); 2700 } 2701 EXPORT_SYMBOL_GPL(sk_setup_caps); 2702 2703 bool sk_has_decrypt_user(const struct sock *sk) 2704 { 2705 return psp_sk_assoc(sk) || 2706 (sk_is_inet(sk) && inet_csk_has_ulp(sk)); /* for tls */ 2707 } 2708 2709 /* 2710 * Simple resource managers for sockets. 2711 */ 2712 2713 2714 /* 2715 * Write buffer destructor automatically called from kfree_skb. 2716 */ 2717 void sock_wfree(struct sk_buff *skb) 2718 { 2719 unsigned int len = skb->truesize; 2720 struct sock *sk = skb->sk; 2721 bool free; 2722 int old; 2723 2724 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) { 2725 void (*sk_write_space)(struct sock *sk); 2726 2727 sk_write_space = READ_ONCE(sk->sk_write_space); 2728 2729 if (sock_flag(sk, SOCK_RCU_FREE) && 2730 sk_write_space == sock_def_write_space) { 2731 rcu_read_lock(); 2732 free = __refcount_sub_and_test(len, &sk->sk_wmem_alloc, 2733 &old); 2734 sock_def_write_space_wfree(sk, old - len); 2735 rcu_read_unlock(); 2736 if (unlikely(free)) 2737 __sk_free(sk); 2738 return; 2739 } 2740 2741 /* 2742 * Keep a reference on sk_wmem_alloc, this will be released 2743 * after sk_write_space() call 2744 */ 2745 WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc)); 2746 sk_write_space(sk); 2747 len = 1; 2748 } 2749 /* 2750 * if sk_wmem_alloc reaches 0, we must finish what sk_free() 2751 * could not do because of in-flight packets 2752 */ 2753 if (refcount_sub_and_test(len, &sk->sk_wmem_alloc)) 2754 __sk_free(sk); 2755 } 2756 EXPORT_SYMBOL(sock_wfree); 2757 2758 /* This variant of sock_wfree() is used by TCP, 2759 * since it sets SOCK_USE_WRITE_QUEUE. 2760 */ 2761 #ifdef CONFIG_INET 2762 void __sock_wfree(struct sk_buff *skb) 2763 { 2764 struct sock *sk = skb->sk; 2765 2766 if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)) 2767 __sk_free(sk); 2768 } 2769 EXPORT_SYMBOL_GPL(__sock_wfree); 2770 #endif 2771 2772 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk) 2773 { 2774 int old_wmem; 2775 2776 skb_orphan(skb); 2777 #ifdef CONFIG_INET 2778 if (unlikely(!sk_fullsock(sk))) 2779 return skb_set_owner_edemux(skb, sk); 2780 #endif 2781 skb->sk = sk; 2782 skb->destructor = sock_wfree; 2783 skb_set_hash_from_sk(skb, sk); 2784 /* 2785 * We used to take a refcount on sk, but following operation 2786 * is enough to guarantee sk_free() won't free this sock until 2787 * all in-flight packets are completed 2788 */ 2789 __refcount_add(skb->truesize, &sk->sk_wmem_alloc, &old_wmem); 2790 2791 /* (old_wmem == SK_WMEM_ALLOC_BIAS) if no other TX packet for this socket 2792 * is in a host queue (qdisc, NIC queue). 2793 * Set skb->ooo_okay so that netdev_pick_tx() can choose a TX queue 2794 * based on XPS for better performance. 2795 * Otherwise clear ooo_okay to not risk Out Of Order delivery. 2796 */ 2797 skb->ooo_okay = (old_wmem == SK_WMEM_ALLOC_BIAS); 2798 } 2799 EXPORT_SYMBOL(skb_set_owner_w); 2800 2801 static bool can_skb_orphan_partial(const struct sk_buff *skb) 2802 { 2803 /* Drivers depend on in-order delivery for crypto offload, 2804 * partial orphan breaks out-of-order-OK logic. 2805 */ 2806 if (skb_is_decrypted(skb)) 2807 return false; 2808 2809 return (skb->destructor == sock_wfree || 2810 (IS_ENABLED(CONFIG_INET) && skb->destructor == tcp_wfree)); 2811 } 2812 2813 /* This helper is used by netem, as it can hold packets in its 2814 * delay queue. We want to allow the owner socket to send more 2815 * packets, as if they were already TX completed by a typical driver. 2816 * But we also want to keep skb->sk set because some packet schedulers 2817 * rely on it (sch_fq for example). 2818 */ 2819 void skb_orphan_partial(struct sk_buff *skb) 2820 { 2821 if (skb_is_tcp_pure_ack(skb)) 2822 return; 2823 2824 if (can_skb_orphan_partial(skb) && skb_set_owner_sk_safe(skb, skb->sk)) 2825 return; 2826 2827 skb_orphan(skb); 2828 } 2829 EXPORT_SYMBOL(skb_orphan_partial); 2830 2831 /* 2832 * Read buffer destructor automatically called from kfree_skb. 2833 */ 2834 void sock_rfree(struct sk_buff *skb) 2835 { 2836 struct sock *sk = skb->sk; 2837 unsigned int len = skb->truesize; 2838 2839 atomic_sub(len, &sk->sk_rmem_alloc); 2840 sk_mem_uncharge(sk, len); 2841 } 2842 EXPORT_SYMBOL(sock_rfree); 2843 2844 /* 2845 * Buffer destructor for skbs that are not used directly in read or write 2846 * path, e.g. for error handler skbs. Automatically called from kfree_skb. 2847 */ 2848 void sock_efree(struct sk_buff *skb) 2849 { 2850 sock_put(skb->sk); 2851 } 2852 EXPORT_SYMBOL(sock_efree); 2853 2854 /* Buffer destructor for prefetch/receive path where reference count may 2855 * not be held, e.g. for listen sockets. 2856 */ 2857 #ifdef CONFIG_INET 2858 void sock_pfree(struct sk_buff *skb) 2859 { 2860 struct sock *sk = skb->sk; 2861 2862 if (!sk_is_refcounted(sk)) 2863 return; 2864 2865 if (sk->sk_state == TCP_NEW_SYN_RECV && inet_reqsk(sk)->syncookie) { 2866 inet_reqsk(sk)->rsk_listener = NULL; 2867 reqsk_free(inet_reqsk(sk)); 2868 return; 2869 } 2870 2871 sock_gen_put(sk); 2872 } 2873 EXPORT_SYMBOL(sock_pfree); 2874 #endif /* CONFIG_INET */ 2875 2876 /* 2877 * Allocate a skb from the socket's send buffer. 2878 */ 2879 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force, 2880 gfp_t priority) 2881 { 2882 if (force || 2883 refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) { 2884 struct sk_buff *skb = alloc_skb(size, priority); 2885 2886 if (skb) { 2887 skb_set_owner_w(skb, sk); 2888 return skb; 2889 } 2890 } 2891 return NULL; 2892 } 2893 EXPORT_SYMBOL(sock_wmalloc); 2894 2895 static void sock_ofree(struct sk_buff *skb) 2896 { 2897 struct sock *sk = skb->sk; 2898 2899 atomic_sub(skb->truesize, &sk->sk_omem_alloc); 2900 } 2901 2902 struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size, 2903 gfp_t priority) 2904 { 2905 struct sk_buff *skb; 2906 2907 /* small safe race: SKB_TRUESIZE may differ from final skb->truesize */ 2908 if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) > 2909 READ_ONCE(sock_net(sk)->core.sysctl_optmem_max)) 2910 return NULL; 2911 2912 skb = alloc_skb(size, priority); 2913 if (!skb) 2914 return NULL; 2915 2916 atomic_add(skb->truesize, &sk->sk_omem_alloc); 2917 skb->sk = sk; 2918 skb->destructor = sock_ofree; 2919 return skb; 2920 } 2921 2922 /* 2923 * Allocate a memory block from the socket's option memory buffer. 2924 */ 2925 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority) 2926 { 2927 int optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max); 2928 2929 if ((unsigned int)size <= optmem_max && 2930 atomic_read(&sk->sk_omem_alloc) + size < optmem_max) { 2931 void *mem; 2932 /* First do the add, to avoid the race if kmalloc 2933 * might sleep. 2934 */ 2935 atomic_add(size, &sk->sk_omem_alloc); 2936 mem = kmalloc(size, priority); 2937 if (mem) 2938 return mem; 2939 atomic_sub(size, &sk->sk_omem_alloc); 2940 } 2941 return NULL; 2942 } 2943 EXPORT_SYMBOL(sock_kmalloc); 2944 2945 /* 2946 * Duplicate the input "src" memory block using the socket's 2947 * option memory buffer. 2948 */ 2949 void *sock_kmemdup(struct sock *sk, const void *src, 2950 int size, gfp_t priority) 2951 { 2952 void *mem; 2953 2954 mem = sock_kmalloc(sk, size, priority); 2955 if (mem) 2956 memcpy(mem, src, size); 2957 return mem; 2958 } 2959 EXPORT_SYMBOL(sock_kmemdup); 2960 2961 /* Free an option memory block. Note, we actually want the inline 2962 * here as this allows gcc to detect the nullify and fold away the 2963 * condition entirely. 2964 */ 2965 static inline void __sock_kfree_s(struct sock *sk, void *mem, int size, 2966 const bool nullify) 2967 { 2968 if (WARN_ON_ONCE(!mem)) 2969 return; 2970 if (nullify) 2971 kfree_sensitive(mem); 2972 else 2973 kfree(mem); 2974 atomic_sub(size, &sk->sk_omem_alloc); 2975 } 2976 2977 void sock_kfree_s(struct sock *sk, void *mem, int size) 2978 { 2979 __sock_kfree_s(sk, mem, size, false); 2980 } 2981 EXPORT_SYMBOL(sock_kfree_s); 2982 2983 void sock_kzfree_s(struct sock *sk, void *mem, int size) 2984 { 2985 __sock_kfree_s(sk, mem, size, true); 2986 } 2987 EXPORT_SYMBOL(sock_kzfree_s); 2988 2989 /** 2990 * sk_set_nospace - tell the transport a writer is waiting for space 2991 * @sk: socket 2992 * 2993 * Must be called before the final check of the available send space, 2994 * so that the transport can not miss the request and forget to call 2995 * sk->sk_write_space() once space is available again. 2996 */ 2997 void sk_set_nospace(struct sock *sk) 2998 { 2999 struct socket *sock = sk->sk_socket; 3000 3001 if (!sock) 3002 return; 3003 /* Set SOCK_NOSPACE before tp->tcp_nospace (paired with 3004 * sk_clear_nospace() clearing tp->tcp_nospace before SOCK_NOSPACE) 3005 * so a concurrent clear cannot leave SOCK_NOSPACE set with 3006 * tp->tcp_nospace cleared. 3007 */ 3008 set_bit(SOCK_NOSPACE, &sock->flags); 3009 tcp_set_nospace(sk); 3010 } 3011 EXPORT_SYMBOL(sk_set_nospace); 3012 3013 /** 3014 * sk_clear_nospace - tell the transport no writer is waiting for space 3015 * @sk: socket 3016 * 3017 * Called from ->sk_write_space() handlers, once send space has been 3018 * made available to writers. 3019 */ 3020 void sk_clear_nospace(struct sock *sk) 3021 { 3022 struct socket *sock = sk->sk_socket; 3023 3024 if (!sock) 3025 return; 3026 tcp_clear_nospace(sk); 3027 clear_bit(SOCK_NOSPACE, &sock->flags); 3028 } 3029 EXPORT_SYMBOL(sk_clear_nospace); 3030 3031 /* It is almost wait_for_tcp_memory minus release_sock/lock_sock. 3032 I think, these locks should be removed for datagram sockets. 3033 */ 3034 static long sock_wait_for_wmem(struct sock *sk, long timeo) 3035 { 3036 DEFINE_WAIT(wait); 3037 3038 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 3039 for (;;) { 3040 if (!timeo) 3041 break; 3042 if (signal_pending(current)) 3043 break; 3044 sk_set_nospace(sk); 3045 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); 3046 if (refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) 3047 break; 3048 if (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN) 3049 break; 3050 if (READ_ONCE(sk->sk_err)) 3051 break; 3052 timeo = schedule_timeout(timeo); 3053 } 3054 finish_wait(sk_sleep(sk), &wait); 3055 return timeo; 3056 } 3057 3058 3059 /* 3060 * Generic send/receive buffer handlers 3061 */ 3062 3063 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len, 3064 unsigned long data_len, int noblock, 3065 int *errcode, int max_page_order) 3066 { 3067 struct sk_buff *skb; 3068 long timeo; 3069 int err; 3070 3071 timeo = sock_sndtimeo(sk, noblock); 3072 for (;;) { 3073 err = sock_error(sk); 3074 if (err != 0) 3075 goto failure; 3076 3077 err = -EPIPE; 3078 if (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN) 3079 goto failure; 3080 3081 if (sk_wmem_alloc_get(sk) < READ_ONCE(sk->sk_sndbuf)) 3082 break; 3083 3084 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 3085 sk_set_nospace(sk); 3086 err = -EAGAIN; 3087 if (!timeo) 3088 goto failure; 3089 if (signal_pending(current)) 3090 goto interrupted; 3091 timeo = sock_wait_for_wmem(sk, timeo); 3092 } 3093 skb = alloc_skb_with_frags(header_len, data_len, max_page_order, 3094 errcode, sk->sk_allocation); 3095 if (skb) 3096 skb_set_owner_w(skb, sk); 3097 return skb; 3098 3099 interrupted: 3100 err = sock_intr_errno(timeo); 3101 failure: 3102 *errcode = err; 3103 return NULL; 3104 } 3105 EXPORT_SYMBOL(sock_alloc_send_pskb); 3106 3107 int __sock_cmsg_send(struct sock *sk, struct cmsghdr *cmsg, 3108 struct sockcm_cookie *sockc) 3109 { 3110 u32 tsflags; 3111 3112 BUILD_BUG_ON(SOF_TIMESTAMPING_LAST == (1 << 31)); 3113 3114 switch (cmsg->cmsg_type) { 3115 case SO_MARK: 3116 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && 3117 !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 3118 return -EPERM; 3119 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3120 return -EINVAL; 3121 sockc->mark = *(u32 *)CMSG_DATA(cmsg); 3122 break; 3123 case SO_TIMESTAMPING_OLD: 3124 case SO_TIMESTAMPING_NEW: 3125 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3126 return -EINVAL; 3127 3128 tsflags = *(u32 *)CMSG_DATA(cmsg); 3129 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK) 3130 return -EINVAL; 3131 3132 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK; 3133 sockc->tsflags |= tsflags; 3134 break; 3135 case SCM_TXTIME: 3136 { 3137 ktime_t tmin; 3138 u64 txtime; 3139 3140 if (!sock_flag(sk, SOCK_TXTIME)) 3141 return -EINVAL; 3142 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64))) 3143 return -EINVAL; 3144 3145 txtime = get_unaligned((u64 *)CMSG_DATA(cmsg)); 3146 3147 /* Allow sending without a delivery time: zero special case */ 3148 if (!txtime) { 3149 sockc->transmit_time = 0; 3150 break; 3151 } 3152 3153 switch (sk->sk_clockid) { 3154 case CLOCK_MONOTONIC: 3155 tmin = 1; 3156 break; 3157 case CLOCK_REALTIME: 3158 tmin = max(ktime_mono_to_real(0), 1); 3159 break; 3160 case CLOCK_TAI: 3161 tmin = max(ktime_mono_to_any(0, TK_OFFS_TAI), 1); 3162 break; 3163 default: 3164 tmin = 1; 3165 WARN_ON_ONCE(1); 3166 break; 3167 } 3168 3169 sockc->transmit_time = max_t(ktime_t, txtime, tmin); 3170 break; 3171 } 3172 case SCM_TS_OPT_ID: 3173 if (sk_is_tcp(sk)) 3174 return -EINVAL; 3175 tsflags = READ_ONCE(sk->sk_tsflags); 3176 if (!(tsflags & SOF_TIMESTAMPING_OPT_ID)) 3177 return -EINVAL; 3178 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3179 return -EINVAL; 3180 sockc->ts_opt_id = *(u32 *)CMSG_DATA(cmsg); 3181 sockc->tsflags |= SOCKCM_FLAG_TS_OPT_ID; 3182 break; 3183 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */ 3184 case SCM_RIGHTS: 3185 case SCM_CREDENTIALS: 3186 break; 3187 case SO_PRIORITY: 3188 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3189 return -EINVAL; 3190 if (!sk_set_prio_allowed(sk, *(u32 *)CMSG_DATA(cmsg))) 3191 return -EPERM; 3192 sockc->priority = *(u32 *)CMSG_DATA(cmsg); 3193 break; 3194 case SCM_DEVMEM_DMABUF: 3195 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3196 return -EINVAL; 3197 sockc->dmabuf_id = *(u32 *)CMSG_DATA(cmsg); 3198 break; 3199 default: 3200 return -EINVAL; 3201 } 3202 return 0; 3203 } 3204 EXPORT_SYMBOL(__sock_cmsg_send); 3205 3206 int sock_cmsg_send(struct sock *sk, struct msghdr *msg, 3207 struct sockcm_cookie *sockc) 3208 { 3209 struct cmsghdr *cmsg; 3210 int ret; 3211 3212 for_each_cmsghdr(cmsg, msg) { 3213 if (!CMSG_OK(msg, cmsg)) 3214 return -EINVAL; 3215 if (cmsg->cmsg_level != SOL_SOCKET) 3216 continue; 3217 ret = __sock_cmsg_send(sk, cmsg, sockc); 3218 if (ret) 3219 return ret; 3220 } 3221 return 0; 3222 } 3223 EXPORT_SYMBOL(sock_cmsg_send); 3224 3225 static void sk_enter_memory_pressure(struct sock *sk) 3226 { 3227 if (!sk->sk_prot->enter_memory_pressure) 3228 return; 3229 3230 sk->sk_prot->enter_memory_pressure(sk); 3231 } 3232 3233 static void sk_leave_memory_pressure(struct sock *sk) 3234 { 3235 if (sk->sk_prot->leave_memory_pressure) { 3236 INDIRECT_CALL_INET_1(sk->sk_prot->leave_memory_pressure, 3237 tcp_leave_memory_pressure, sk); 3238 } else { 3239 unsigned long *memory_pressure = sk->sk_prot->memory_pressure; 3240 3241 if (memory_pressure && READ_ONCE(*memory_pressure)) 3242 WRITE_ONCE(*memory_pressure, 0); 3243 } 3244 } 3245 3246 DEFINE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key); 3247 3248 /** 3249 * skb_page_frag_refill - check that a page_frag contains enough room 3250 * @sz: minimum size of the fragment we want to get 3251 * @pfrag: pointer to page_frag 3252 * @gfp: priority for memory allocation 3253 * 3254 * Note: While this allocator tries to use high order pages, there is 3255 * no guarantee that allocations succeed. Therefore, @sz MUST be 3256 * less or equal than PAGE_SIZE. 3257 */ 3258 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp) 3259 { 3260 if (pfrag->page) { 3261 if (page_ref_count(pfrag->page) == 1) { 3262 pfrag->offset = 0; 3263 return true; 3264 } 3265 if (pfrag->offset + sz <= pfrag->size) 3266 return true; 3267 put_page(pfrag->page); 3268 } 3269 3270 pfrag->offset = 0; 3271 if (SKB_FRAG_PAGE_ORDER && 3272 !static_branch_unlikely(&net_high_order_alloc_disable_key)) { 3273 /* Avoid direct reclaim but allow kswapd to wake */ 3274 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) | 3275 __GFP_COMP | __GFP_NOWARN | 3276 __GFP_NORETRY, 3277 SKB_FRAG_PAGE_ORDER); 3278 if (likely(pfrag->page)) { 3279 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER; 3280 return true; 3281 } 3282 } 3283 pfrag->page = alloc_page(gfp); 3284 if (likely(pfrag->page)) { 3285 pfrag->size = PAGE_SIZE; 3286 return true; 3287 } 3288 return false; 3289 } 3290 EXPORT_SYMBOL(skb_page_frag_refill); 3291 3292 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 3293 { 3294 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation))) 3295 return true; 3296 3297 if (!sk->sk_bypass_prot_mem) 3298 sk_enter_memory_pressure(sk); 3299 3300 sk_stream_moderate_sndbuf(sk); 3301 3302 return false; 3303 } 3304 EXPORT_SYMBOL(sk_page_frag_refill); 3305 3306 static void __lock_sock(struct sock *sk) 3307 __releases(&sk->sk_lock.slock) 3308 __acquires(&sk->sk_lock.slock) 3309 { 3310 DEFINE_WAIT(wait); 3311 3312 for (;;) { 3313 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait, 3314 TASK_UNINTERRUPTIBLE); 3315 spin_unlock_bh(&sk->sk_lock.slock); 3316 schedule(); 3317 spin_lock_bh(&sk->sk_lock.slock); 3318 if (!sock_owned_by_user(sk)) 3319 break; 3320 } 3321 finish_wait(&sk->sk_lock.wq, &wait); 3322 } 3323 3324 void __release_sock(struct sock *sk) 3325 __releases(&sk->sk_lock.slock) 3326 __acquires(&sk->sk_lock.slock) 3327 { 3328 struct sk_buff *skb, *next; 3329 int nb = 0; 3330 3331 while ((skb = sk->sk_backlog.head) != NULL) { 3332 sk->sk_backlog.head = sk->sk_backlog.tail = NULL; 3333 3334 spin_unlock_bh(&sk->sk_lock.slock); 3335 3336 do { 3337 next = skb->next; 3338 prefetch(next); 3339 DEBUG_NET_WARN_ON_ONCE(skb_dst_is_noref(skb)); 3340 skb_mark_not_on_list(skb); 3341 sk_backlog_rcv(sk, skb); 3342 3343 skb = next; 3344 3345 if (!(++nb & 15)) 3346 cond_resched(); 3347 } while (skb); 3348 3349 spin_lock_bh(&sk->sk_lock.slock); 3350 } 3351 3352 /* 3353 * Doing the zeroing here guarantee we can not loop forever 3354 * while a wild producer attempts to flood us. 3355 */ 3356 sk->sk_backlog.len = 0; 3357 } 3358 3359 void __sk_flush_backlog(struct sock *sk) 3360 { 3361 spin_lock_bh(&sk->sk_lock.slock); 3362 __release_sock(sk); 3363 3364 if (sk->sk_prot->release_cb) 3365 INDIRECT_CALL_INET_1(sk->sk_prot->release_cb, 3366 tcp_release_cb, sk); 3367 3368 spin_unlock_bh(&sk->sk_lock.slock); 3369 } 3370 EXPORT_SYMBOL_GPL(__sk_flush_backlog); 3371 3372 /** 3373 * sk_wait_data - wait for data to arrive at sk_receive_queue 3374 * @sk: sock to wait on 3375 * @timeo: for how long 3376 * @skb: last skb seen on sk_receive_queue 3377 * 3378 * Now socket state including sk->sk_err is changed only under lock, 3379 * hence we may omit checks after joining wait queue. 3380 * We check receive queue before schedule() only as optimization; 3381 * it is very likely that release_sock() added new data. 3382 */ 3383 int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb) 3384 { 3385 DEFINE_WAIT_FUNC(wait, woken_wake_function); 3386 int rc; 3387 3388 add_wait_queue(sk_sleep(sk), &wait); 3389 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 3390 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait); 3391 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 3392 remove_wait_queue(sk_sleep(sk), &wait); 3393 return rc; 3394 } 3395 EXPORT_SYMBOL(sk_wait_data); 3396 3397 /** 3398 * __sk_mem_raise_allocated - increase memory_allocated 3399 * @sk: socket 3400 * @size: memory size to allocate 3401 * @amt: pages to allocate 3402 * @kind: allocation type 3403 * 3404 * Similar to __sk_mem_schedule(), but does not update sk_forward_alloc. 3405 * 3406 * Unlike the globally shared limits among the sockets under same protocol, 3407 * consuming the budget of a memcg won't have direct effect on other ones. 3408 * So be optimistic about memcg's tolerance, and leave the callers to decide 3409 * whether or not to raise allocated through sk_under_memory_pressure() or 3410 * its variants. 3411 */ 3412 int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind) 3413 { 3414 bool memcg_enabled = false, charged = false; 3415 struct proto *prot = sk->sk_prot; 3416 long allocated = 0; 3417 3418 if (!sk->sk_bypass_prot_mem) { 3419 sk_memory_allocated_add(sk, amt); 3420 allocated = sk_memory_allocated(sk); 3421 } 3422 3423 if (mem_cgroup_sk_enabled(sk)) { 3424 memcg_enabled = true; 3425 charged = mem_cgroup_sk_charge(sk, amt, gfp_memcg_charge()); 3426 if (!charged) 3427 goto suppress_allocation; 3428 } 3429 3430 if (sk->sk_bypass_prot_mem) 3431 return 1; 3432 3433 /* Under limit. */ 3434 if (allocated <= sk_prot_mem_limits(sk, 0)) { 3435 sk_leave_memory_pressure(sk); 3436 return 1; 3437 } 3438 3439 /* Under pressure. */ 3440 if (allocated > sk_prot_mem_limits(sk, 1)) 3441 sk_enter_memory_pressure(sk); 3442 3443 /* Over hard limit. */ 3444 if (allocated > sk_prot_mem_limits(sk, 2)) 3445 goto suppress_allocation; 3446 3447 /* Guarantee minimum buffer size under pressure (either global 3448 * or memcg) to make sure features described in RFC 7323 (TCP 3449 * Extensions for High Performance) work properly. 3450 * 3451 * This rule does NOT stand when exceeds global or memcg's hard 3452 * limit, or else a DoS attack can be taken place by spawning 3453 * lots of sockets whose usage are under minimum buffer size. 3454 */ 3455 if (kind == SK_MEM_RECV) { 3456 if (atomic_read(&sk->sk_rmem_alloc) < sk_get_rmem0(sk, prot)) 3457 return 1; 3458 3459 } else { /* SK_MEM_SEND */ 3460 int wmem0 = sk_get_wmem0(sk, prot); 3461 3462 if (sk->sk_type == SOCK_STREAM) { 3463 if (sk->sk_wmem_queued < wmem0) 3464 return 1; 3465 } else if (refcount_read(&sk->sk_wmem_alloc) < wmem0) { 3466 return 1; 3467 } 3468 } 3469 3470 if (sk_has_memory_pressure(sk)) { 3471 u64 alloc; 3472 3473 /* The following 'average' heuristic is within the 3474 * scope of global accounting, so it only makes 3475 * sense for global memory pressure. 3476 */ 3477 if (!sk_under_global_memory_pressure(sk)) 3478 return 1; 3479 3480 /* Try to be fair among all the sockets under global 3481 * pressure by allowing the ones that below average 3482 * usage to raise. 3483 */ 3484 alloc = sk_sockets_allocated_read_positive(sk); 3485 if (sk_prot_mem_limits(sk, 2) > alloc * 3486 sk_mem_pages(sk->sk_wmem_queued + 3487 atomic_read(&sk->sk_rmem_alloc) + 3488 sk->sk_forward_alloc)) 3489 return 1; 3490 } 3491 3492 suppress_allocation: 3493 3494 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) { 3495 sk_stream_moderate_sndbuf(sk); 3496 3497 /* Fail only if socket is _under_ its sndbuf. 3498 * In this case we cannot block, so that we have to fail. 3499 */ 3500 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf) { 3501 /* Force charge with __GFP_NOFAIL */ 3502 if (memcg_enabled && !charged) 3503 mem_cgroup_sk_charge(sk, amt, 3504 gfp_memcg_charge() | __GFP_NOFAIL); 3505 return 1; 3506 } 3507 } 3508 3509 trace_sock_exceed_buf_limit(sk, prot, allocated, kind); 3510 3511 if (!sk->sk_bypass_prot_mem) 3512 sk_memory_allocated_sub(sk, amt); 3513 3514 if (charged) 3515 mem_cgroup_sk_uncharge(sk, amt); 3516 3517 return 0; 3518 } 3519 3520 /** 3521 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated 3522 * @sk: socket 3523 * @size: memory size to allocate 3524 * @kind: allocation type 3525 * 3526 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means 3527 * rmem allocation. This function assumes that protocols which have 3528 * memory_pressure use sk_wmem_queued as write buffer accounting. 3529 */ 3530 int __sk_mem_schedule(struct sock *sk, int size, int kind) 3531 { 3532 int ret, amt = sk_mem_pages(size); 3533 3534 sk_forward_alloc_add(sk, amt << PAGE_SHIFT); 3535 ret = __sk_mem_raise_allocated(sk, size, amt, kind); 3536 if (!ret) 3537 sk_forward_alloc_add(sk, -(amt << PAGE_SHIFT)); 3538 return ret; 3539 } 3540 EXPORT_SYMBOL(__sk_mem_schedule); 3541 3542 /** 3543 * __sk_mem_reduce_allocated - reclaim memory_allocated 3544 * @sk: socket 3545 * @amount: number of quanta 3546 * 3547 * Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc 3548 */ 3549 void __sk_mem_reduce_allocated(struct sock *sk, int amount) 3550 { 3551 if (mem_cgroup_sk_enabled(sk)) 3552 mem_cgroup_sk_uncharge(sk, amount); 3553 3554 if (sk->sk_bypass_prot_mem) 3555 return; 3556 3557 sk_memory_allocated_sub(sk, amount); 3558 3559 if (sk_under_global_memory_pressure(sk) && 3560 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0))) 3561 sk_leave_memory_pressure(sk); 3562 } 3563 3564 /** 3565 * __sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated 3566 * @sk: socket 3567 * @amount: number of bytes (rounded down to a PAGE_SIZE multiple) 3568 */ 3569 void __sk_mem_reclaim(struct sock *sk, int amount) 3570 { 3571 amount >>= PAGE_SHIFT; 3572 sk_forward_alloc_add(sk, -(amount << PAGE_SHIFT)); 3573 __sk_mem_reduce_allocated(sk, amount); 3574 } 3575 EXPORT_SYMBOL(__sk_mem_reclaim); 3576 3577 void __sk_charge(struct sock *sk, gfp_t gfp) 3578 { 3579 int amt; 3580 3581 gfp |= __GFP_NOFAIL; 3582 if (mem_cgroup_from_sk(sk)) { 3583 /* The socket has not been accepted yet, no need 3584 * to look at newsk->sk_wmem_queued. 3585 */ 3586 amt = sk_mem_pages(sk->sk_forward_alloc + 3587 atomic_read(&sk->sk_rmem_alloc)); 3588 if (amt) 3589 mem_cgroup_sk_charge(sk, amt, gfp); 3590 } 3591 3592 kmem_cache_charge(sk, gfp); 3593 } 3594 3595 int sk_set_peek_off(struct sock *sk, int val) 3596 { 3597 WRITE_ONCE(sk->sk_peek_off, val); 3598 return 0; 3599 } 3600 EXPORT_SYMBOL_GPL(sk_set_peek_off); 3601 3602 /* 3603 * Set of default routines for initialising struct proto_ops when 3604 * the protocol does not support a particular function. In certain 3605 * cases where it makes no sense for a protocol to have a "do nothing" 3606 * function, some default processing is provided. 3607 */ 3608 3609 int sock_no_bind(struct socket *sock, struct sockaddr_unsized *saddr, int len) 3610 { 3611 return -EOPNOTSUPP; 3612 } 3613 EXPORT_SYMBOL(sock_no_bind); 3614 3615 int sock_no_connect(struct socket *sock, struct sockaddr_unsized *saddr, 3616 int len, int flags) 3617 { 3618 return -EOPNOTSUPP; 3619 } 3620 EXPORT_SYMBOL(sock_no_connect); 3621 3622 int sock_no_socketpair(struct socket *sock1, struct socket *sock2) 3623 { 3624 return -EOPNOTSUPP; 3625 } 3626 EXPORT_SYMBOL(sock_no_socketpair); 3627 3628 int sock_no_accept(struct socket *sock, struct socket *newsock, 3629 struct proto_accept_arg *arg) 3630 { 3631 return -EOPNOTSUPP; 3632 } 3633 EXPORT_SYMBOL(sock_no_accept); 3634 3635 int sock_no_getname(struct socket *sock, struct sockaddr *saddr, 3636 int peer) 3637 { 3638 return -EOPNOTSUPP; 3639 } 3640 EXPORT_SYMBOL(sock_no_getname); 3641 3642 int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) 3643 { 3644 return -EOPNOTSUPP; 3645 } 3646 EXPORT_SYMBOL(sock_no_ioctl); 3647 3648 int sock_no_listen(struct socket *sock, int backlog) 3649 { 3650 return -EOPNOTSUPP; 3651 } 3652 EXPORT_SYMBOL(sock_no_listen); 3653 3654 int sock_no_shutdown(struct socket *sock, int how) 3655 { 3656 return -EOPNOTSUPP; 3657 } 3658 EXPORT_SYMBOL(sock_no_shutdown); 3659 3660 int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len) 3661 { 3662 return -EOPNOTSUPP; 3663 } 3664 EXPORT_SYMBOL(sock_no_sendmsg); 3665 3666 int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len) 3667 { 3668 return -EOPNOTSUPP; 3669 } 3670 EXPORT_SYMBOL(sock_no_sendmsg_locked); 3671 3672 int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len, 3673 int flags) 3674 { 3675 return -EOPNOTSUPP; 3676 } 3677 EXPORT_SYMBOL(sock_no_recvmsg); 3678 3679 int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma) 3680 { 3681 /* Mirror missing mmap method error code */ 3682 return -ENODEV; 3683 } 3684 EXPORT_SYMBOL(sock_no_mmap); 3685 3686 /* 3687 * When a file is received (via SCM_RIGHTS, etc), we must bump the 3688 * various sock-based usage counts. 3689 */ 3690 void __receive_sock(struct file *file) 3691 { 3692 struct socket *sock; 3693 3694 sock = sock_from_file(file); 3695 if (sock) { 3696 sock_update_netprioidx(&sock->sk->sk_cgrp_data); 3697 sock_update_classid(&sock->sk->sk_cgrp_data); 3698 } 3699 } 3700 3701 /* 3702 * Default Socket Callbacks 3703 */ 3704 3705 static void sock_def_wakeup(struct sock *sk) 3706 { 3707 struct socket_wq *wq; 3708 3709 rcu_read_lock(); 3710 wq = rcu_dereference(sk->sk_wq); 3711 if (skwq_has_sleeper(wq)) 3712 wake_up_interruptible_all(&wq->wait); 3713 rcu_read_unlock(); 3714 } 3715 3716 static void sock_def_error_report(struct sock *sk) 3717 { 3718 struct socket_wq *wq; 3719 3720 rcu_read_lock(); 3721 wq = rcu_dereference(sk->sk_wq); 3722 if (skwq_has_sleeper(wq)) 3723 wake_up_interruptible_poll(&wq->wait, EPOLLERR); 3724 sk_wake_async_rcu(sk, SOCK_WAKE_IO, POLL_ERR); 3725 rcu_read_unlock(); 3726 } 3727 3728 void sock_def_readable(struct sock *sk) 3729 { 3730 struct socket_wq *wq; 3731 3732 trace_sk_data_ready(sk); 3733 3734 rcu_read_lock(); 3735 wq = rcu_dereference(sk->sk_wq); 3736 if (skwq_has_sleeper(wq)) 3737 wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI | 3738 EPOLLRDNORM | EPOLLRDBAND); 3739 sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_IN); 3740 rcu_read_unlock(); 3741 } 3742 3743 static void sock_def_write_space(struct sock *sk) 3744 { 3745 struct socket_wq *wq; 3746 3747 rcu_read_lock(); 3748 3749 /* Do not wake up a writer until he can make "significant" 3750 * progress. --DaveM 3751 */ 3752 if (sock_writeable(sk)) { 3753 wq = rcu_dereference(sk->sk_wq); 3754 if (skwq_has_sleeper(wq)) 3755 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 3756 EPOLLWRNORM | EPOLLWRBAND); 3757 3758 /* Should agree with poll, otherwise some programs break */ 3759 sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT); 3760 } 3761 3762 rcu_read_unlock(); 3763 } 3764 3765 /* An optimised version of sock_def_write_space(), should only be called 3766 * for SOCK_RCU_FREE sockets under RCU read section and after putting 3767 * ->sk_wmem_alloc. 3768 */ 3769 static void sock_def_write_space_wfree(struct sock *sk, int wmem_alloc) 3770 { 3771 /* Do not wake up a writer until he can make "significant" 3772 * progress. --DaveM 3773 */ 3774 if (__sock_writeable(sk, wmem_alloc)) { 3775 struct socket_wq *wq = rcu_dereference(sk->sk_wq); 3776 3777 /* rely on refcount_sub from sock_wfree() */ 3778 smp_mb__after_atomic(); 3779 if (wq && waitqueue_active(&wq->wait)) 3780 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 3781 EPOLLWRNORM | EPOLLWRBAND); 3782 3783 /* Should agree with poll, otherwise some programs break */ 3784 sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT); 3785 } 3786 } 3787 3788 static void sock_def_destruct(struct sock *sk) 3789 { 3790 } 3791 3792 void sk_send_sigurg(struct sock *sk) 3793 { 3794 if (sk->sk_socket && sk->sk_socket->file) 3795 if (send_sigurg(sk->sk_socket->file)) 3796 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI); 3797 } 3798 EXPORT_SYMBOL(sk_send_sigurg); 3799 3800 void sk_reset_timer(struct sock *sk, struct timer_list* timer, 3801 unsigned long expires) 3802 { 3803 if (!mod_timer(timer, expires)) 3804 sock_hold(sk); 3805 } 3806 EXPORT_SYMBOL(sk_reset_timer); 3807 3808 void sk_stop_timer(struct sock *sk, struct timer_list* timer) 3809 { 3810 if (timer_delete(timer)) 3811 __sock_put(sk); 3812 } 3813 EXPORT_SYMBOL(sk_stop_timer); 3814 3815 void sk_stop_timer_sync(struct sock *sk, struct timer_list *timer) 3816 { 3817 if (timer_delete_sync(timer)) 3818 __sock_put(sk); 3819 } 3820 EXPORT_SYMBOL(sk_stop_timer_sync); 3821 3822 void sock_init_data_uid(struct socket *sock, struct sock *sk, kuid_t uid) 3823 { 3824 sk_init_common(sk); 3825 sk->sk_send_head = NULL; 3826 3827 timer_setup(&sk->sk_timer, NULL, 0); 3828 3829 sk->sk_allocation = GFP_KERNEL; 3830 sk->sk_rcvbuf = READ_ONCE(sysctl_rmem_default); 3831 sk->sk_sndbuf = READ_ONCE(sysctl_wmem_default); 3832 sk->sk_state = TCP_CLOSE; 3833 sk->sk_use_task_frag = true; 3834 sk_set_socket(sk, sock); 3835 3836 sock_set_flag(sk, SOCK_ZAPPED); 3837 3838 if (sock) { 3839 sk->sk_type = sock->type; 3840 RCU_INIT_POINTER(sk->sk_wq, &sock->wq); 3841 sock->sk = sk; 3842 } else { 3843 RCU_INIT_POINTER(sk->sk_wq, NULL); 3844 } 3845 sk->sk_uid = uid; 3846 3847 sk->sk_state_change = sock_def_wakeup; 3848 sk->sk_data_ready = sock_def_readable; 3849 sk->sk_write_space = sock_def_write_space; 3850 sk->sk_error_report = sock_def_error_report; 3851 sk->sk_destruct = sock_def_destruct; 3852 3853 sk->sk_frag.page = NULL; 3854 sk->sk_frag.offset = 0; 3855 sk->sk_peek_off = -1; 3856 3857 memset(sk->sk_peer_pid, 0, sizeof(sk->sk_peer_pid)); 3858 sk->sk_peer_cred = NULL; 3859 spin_lock_init(&sk->sk_peer_lock); 3860 3861 sk->sk_write_pending = 0; 3862 sk->sk_rcvlowat = 1; 3863 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT; 3864 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT; 3865 3866 sk->sk_stamp = SK_DEFAULT_STAMP; 3867 #if BITS_PER_LONG==32 3868 seqlock_init(&sk->sk_stamp_seq); 3869 #endif 3870 atomic_set(&sk->sk_zckey, 0); 3871 3872 #ifdef CONFIG_NET_RX_BUSY_POLL 3873 sk->sk_napi_id = 0; 3874 sk->sk_ll_usec = READ_ONCE(sysctl_net_busy_read); 3875 #endif 3876 3877 sk->sk_max_pacing_rate = ~0UL; 3878 sk->sk_pacing_rate = ~0UL; 3879 WRITE_ONCE(sk->sk_pacing_shift, 10); 3880 sk->sk_incoming_cpu = -1; 3881 3882 sk_rx_queue_clear(sk); 3883 /* 3884 * Before updating sk_refcnt, we must commit prior changes to memory 3885 * (Documentation/RCU/rculist_nulls.rst for details) 3886 */ 3887 smp_wmb(); 3888 refcount_set(&sk->sk_refcnt, 1); 3889 sk_drops_reset(sk); 3890 } 3891 EXPORT_SYMBOL(sock_init_data_uid); 3892 3893 void sock_init_data(struct socket *sock, struct sock *sk) 3894 { 3895 kuid_t uid = sock ? 3896 SOCK_INODE(sock)->i_uid : 3897 make_kuid(sock_net(sk)->user_ns, 0); 3898 3899 sock_init_data_uid(sock, sk, uid); 3900 } 3901 EXPORT_SYMBOL(sock_init_data); 3902 3903 void noinline lock_sock_nested(struct sock *sk, int subclass) 3904 { 3905 /* The sk_lock has mutex_lock() semantics here. */ 3906 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_); 3907 3908 might_sleep(); 3909 #ifdef CONFIG_64BIT 3910 if (sizeof(struct slock_owned) == sizeof(long)) { 3911 socket_lock_t tmp = { 3912 .slock = __SPIN_LOCK_UNLOCKED(tmp.slock), 3913 .owned = 1, 3914 }; 3915 socket_lock_t old = { 3916 .slock = __SPIN_LOCK_UNLOCKED(old.slock), 3917 .owned = 0, 3918 }; 3919 3920 if (likely(try_cmpxchg(&sk->sk_lock.combined, 3921 &old.combined, tmp.combined))) 3922 return; 3923 } 3924 #endif 3925 spin_lock_bh(&sk->sk_lock.slock); 3926 if (unlikely(sock_owned_by_user_nocheck(sk))) 3927 __lock_sock(sk); 3928 sk->sk_lock.owned = 1; 3929 spin_unlock_bh(&sk->sk_lock.slock); 3930 } 3931 EXPORT_SYMBOL(lock_sock_nested); 3932 3933 void release_sock(struct sock *sk) 3934 { 3935 spin_lock_bh(&sk->sk_lock.slock); 3936 3937 if (unlikely(sk->sk_backlog.tail)) 3938 __release_sock(sk); 3939 3940 if (sk->sk_prot->release_cb) { 3941 if (!tcp_release_cb_cond(sk)) 3942 sk->sk_prot->release_cb(sk); 3943 } 3944 sock_release_ownership(sk); 3945 if (unlikely(waitqueue_active(&sk->sk_lock.wq))) 3946 wake_up(&sk->sk_lock.wq); 3947 3948 spin_unlock_bh(&sk->sk_lock.slock); 3949 } 3950 EXPORT_SYMBOL(release_sock); 3951 3952 bool __lock_sock_fast(struct sock *sk) __acquires(&sk->sk_lock.slock) 3953 { 3954 might_sleep(); 3955 spin_lock_bh(&sk->sk_lock.slock); 3956 3957 if (likely(!sock_owned_by_user_nocheck(sk))) { 3958 /* 3959 * Fast path return with bottom halves disabled and 3960 * sock::sk_lock.slock held. 3961 * 3962 * The 'mutex' is not contended and holding 3963 * sock::sk_lock.slock prevents all other lockers to 3964 * proceed so the corresponding unlock_sock_fast() can 3965 * avoid the slow path of release_sock() completely and 3966 * just release slock. 3967 * 3968 * From a semantical POV this is equivalent to 'acquiring' 3969 * the 'mutex', hence the corresponding lockdep 3970 * mutex_release() has to happen in the fast path of 3971 * unlock_sock_fast(). 3972 */ 3973 return false; 3974 } 3975 3976 __lock_sock(sk); 3977 sk->sk_lock.owned = 1; 3978 __acquire(&sk->sk_lock.slock); 3979 spin_unlock_bh(&sk->sk_lock.slock); 3980 return true; 3981 } 3982 EXPORT_SYMBOL(__lock_sock_fast); 3983 3984 int sock_gettstamp(struct socket *sock, void __user *userstamp, 3985 bool timeval, bool time32) 3986 { 3987 struct sock *sk = sock->sk; 3988 struct timespec64 ts; 3989 3990 /* sk->sk_flags must only be changed under the socket lock, 3991 * because sock_set_flag() uses non atomic operations. 3992 */ 3993 if (!sock_flag(sk, SOCK_TIMESTAMP)) { 3994 lock_sock(sk); 3995 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 3996 release_sock(sk); 3997 } 3998 ts = ktime_to_timespec64(sock_read_timestamp(sk)); 3999 if (ts.tv_sec == -1) 4000 return -ENOENT; 4001 if (ts.tv_sec == 0) { 4002 ktime_t kt = ktime_get_real(); 4003 sock_write_timestamp(sk, kt); 4004 ts = ktime_to_timespec64(kt); 4005 } 4006 4007 if (timeval) 4008 ts.tv_nsec /= 1000; 4009 4010 #ifdef CONFIG_COMPAT_32BIT_TIME 4011 if (time32) 4012 return put_old_timespec32(&ts, userstamp); 4013 #endif 4014 #ifdef CONFIG_SPARC64 4015 /* beware of padding in sparc64 timeval */ 4016 if (timeval && !in_compat_syscall()) { 4017 struct __kernel_old_timeval __user tv = { 4018 .tv_sec = ts.tv_sec, 4019 .tv_usec = ts.tv_nsec, 4020 }; 4021 if (copy_to_user(userstamp, &tv, sizeof(tv))) 4022 return -EFAULT; 4023 return 0; 4024 } 4025 #endif 4026 return put_timespec64(&ts, userstamp); 4027 } 4028 EXPORT_SYMBOL(sock_gettstamp); 4029 4030 void sock_enable_timestamp(struct sock *sk, enum sock_flags flag) 4031 { 4032 if (!sock_flag(sk, flag)) { 4033 unsigned long previous_flags = sk->sk_flags; 4034 4035 sock_set_flag(sk, flag); 4036 /* 4037 * we just set one of the two flags which require net 4038 * time stamping, but time stamping might have been on 4039 * already because of the other one 4040 */ 4041 if (sock_needs_netstamp(sk) && 4042 !(previous_flags & SK_FLAGS_TIMESTAMP)) 4043 net_enable_timestamp(); 4044 } 4045 } 4046 4047 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, 4048 int level, int type) 4049 { 4050 struct sock_extended_err ee; 4051 struct sk_buff *skb; 4052 int copied, err; 4053 4054 err = -EAGAIN; 4055 skb = sock_dequeue_err_skb(sk); 4056 if (skb == NULL) 4057 goto out; 4058 4059 copied = skb->len; 4060 if (copied > len) { 4061 msg->msg_flags |= MSG_TRUNC; 4062 copied = len; 4063 } 4064 err = skb_copy_datagram_msg(skb, 0, msg, copied); 4065 if (err) 4066 goto out_free_skb; 4067 4068 sock_recv_timestamp(msg, sk, skb); 4069 4070 /* We must use a bounce buffer for CONFIG_HARDENED_USERCOPY=y */ 4071 ee = SKB_EXT_ERR(skb)->ee; 4072 put_cmsg(msg, level, type, sizeof(ee), &ee); 4073 4074 msg->msg_flags |= MSG_ERRQUEUE; 4075 err = copied; 4076 4077 out_free_skb: 4078 kfree_skb(skb); 4079 out: 4080 return err; 4081 } 4082 EXPORT_SYMBOL(sock_recv_errqueue); 4083 4084 /* 4085 * Get a socket option on an socket. 4086 * 4087 * FIX: POSIX 1003.1g is very ambiguous here. It states that 4088 * asynchronous errors should be reported by getsockopt. We assume 4089 * this means if you specify SO_ERROR (otherwise what is the point of it). 4090 */ 4091 int sock_common_getsockopt(struct socket *sock, int level, int optname, 4092 char __user *optval, int __user *optlen) 4093 { 4094 struct sock *sk = sock->sk; 4095 4096 return READ_ONCE(sk->sk_prot)->getsockopt(sk, level, optname, optval, optlen); 4097 } 4098 EXPORT_SYMBOL(sock_common_getsockopt); 4099 4100 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, 4101 int flags) 4102 { 4103 struct sock *sk = sock->sk; 4104 4105 return sk->sk_prot->recvmsg(sk, msg, size, flags); 4106 } 4107 EXPORT_SYMBOL(sock_common_recvmsg); 4108 4109 /* 4110 * Set socket options on an inet socket. 4111 */ 4112 int sock_common_setsockopt(struct socket *sock, int level, int optname, 4113 sockptr_t optval, unsigned int optlen) 4114 { 4115 struct sock *sk = sock->sk; 4116 4117 return READ_ONCE(sk->sk_prot)->setsockopt(sk, level, optname, optval, optlen); 4118 } 4119 EXPORT_SYMBOL(sock_common_setsockopt); 4120 4121 void sk_common_release(struct sock *sk) 4122 { 4123 if (sk->sk_prot->destroy) 4124 sk->sk_prot->destroy(sk); 4125 4126 /* 4127 * Observation: when sk_common_release is called, processes have 4128 * no access to socket. But net still has. 4129 * Step one, detach it from networking: 4130 * 4131 * A. Remove from hash tables. 4132 */ 4133 4134 sk->sk_prot->unhash(sk); 4135 4136 /* 4137 * In this point socket cannot receive new packets, but it is possible 4138 * that some packets are in flight because some CPU runs receiver and 4139 * did hash table lookup before we unhashed socket. They will achieve 4140 * receive queue and will be purged by socket destructor. 4141 * 4142 * Also we still have packets pending on receive queue and probably, 4143 * our own packets waiting in device queues. sock_destroy will drain 4144 * receive queue, but transmitted packets will delay socket destruction 4145 * until the last reference will be released. 4146 */ 4147 4148 sock_orphan(sk); 4149 4150 xfrm_sk_free_policy(sk); 4151 4152 sock_put(sk); 4153 } 4154 EXPORT_SYMBOL(sk_common_release); 4155 4156 void sk_get_meminfo(const struct sock *sk, u32 *mem) 4157 { 4158 memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS); 4159 4160 mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk); 4161 mem[SK_MEMINFO_RCVBUF] = READ_ONCE(sk->sk_rcvbuf); 4162 mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk); 4163 mem[SK_MEMINFO_SNDBUF] = READ_ONCE(sk->sk_sndbuf); 4164 mem[SK_MEMINFO_FWD_ALLOC] = READ_ONCE(sk->sk_forward_alloc); 4165 mem[SK_MEMINFO_WMEM_QUEUED] = READ_ONCE(sk->sk_wmem_queued); 4166 mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc); 4167 mem[SK_MEMINFO_BACKLOG] = READ_ONCE(sk->sk_backlog.len); 4168 mem[SK_MEMINFO_DROPS] = sk_drops_read(sk); 4169 } 4170 4171 #ifdef CONFIG_PROC_FS 4172 static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR); 4173 4174 int sock_prot_inuse_get(struct net *net, struct proto *prot) 4175 { 4176 int cpu, idx = prot->inuse_idx; 4177 int res = 0; 4178 4179 for_each_possible_cpu(cpu) 4180 res += per_cpu_ptr(net->core.prot_inuse, cpu)->val[idx]; 4181 4182 return res >= 0 ? res : 0; 4183 } 4184 EXPORT_SYMBOL_GPL(sock_prot_inuse_get); 4185 4186 int sock_inuse_get(struct net *net) 4187 { 4188 int cpu, res = 0; 4189 4190 for_each_possible_cpu(cpu) 4191 res += per_cpu_ptr(net->core.prot_inuse, cpu)->all; 4192 4193 return res; 4194 } 4195 4196 EXPORT_SYMBOL_GPL(sock_inuse_get); 4197 4198 static int __net_init sock_inuse_init_net(struct net *net) 4199 { 4200 net->core.prot_inuse = alloc_percpu(struct prot_inuse); 4201 if (net->core.prot_inuse == NULL) 4202 return -ENOMEM; 4203 return 0; 4204 } 4205 4206 static void __net_exit sock_inuse_exit_net(struct net *net) 4207 { 4208 free_percpu(net->core.prot_inuse); 4209 } 4210 4211 static struct pernet_operations net_inuse_ops = { 4212 .init = sock_inuse_init_net, 4213 .exit = sock_inuse_exit_net, 4214 }; 4215 4216 static __init int net_inuse_init(void) 4217 { 4218 if (register_pernet_subsys(&net_inuse_ops)) 4219 panic("Cannot initialize net inuse counters"); 4220 4221 return 0; 4222 } 4223 4224 core_initcall(net_inuse_init); 4225 4226 static int assign_proto_idx(struct proto *prot) 4227 { 4228 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR); 4229 4230 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR)) { 4231 pr_err("PROTO_INUSE_NR exhausted\n"); 4232 return -ENOSPC; 4233 } 4234 4235 set_bit(prot->inuse_idx, proto_inuse_idx); 4236 return 0; 4237 } 4238 4239 static void release_proto_idx(struct proto *prot) 4240 { 4241 if (prot->inuse_idx != PROTO_INUSE_NR) 4242 clear_bit(prot->inuse_idx, proto_inuse_idx); 4243 } 4244 #else 4245 static inline int assign_proto_idx(struct proto *prot) 4246 { 4247 return 0; 4248 } 4249 4250 static inline void release_proto_idx(struct proto *prot) 4251 { 4252 } 4253 4254 #endif 4255 4256 static void tw_prot_cleanup(struct timewait_sock_ops *twsk_prot) 4257 { 4258 if (!twsk_prot) 4259 return; 4260 kfree(twsk_prot->twsk_slab_name); 4261 twsk_prot->twsk_slab_name = NULL; 4262 kmem_cache_destroy(twsk_prot->twsk_slab); 4263 twsk_prot->twsk_slab = NULL; 4264 } 4265 4266 static int tw_prot_init(const struct proto *prot) 4267 { 4268 struct timewait_sock_ops *twsk_prot = prot->twsk_prot; 4269 4270 if (!twsk_prot) 4271 return 0; 4272 4273 twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", 4274 prot->name); 4275 if (!twsk_prot->twsk_slab_name) 4276 return -ENOMEM; 4277 4278 twsk_prot->twsk_slab = 4279 kmem_cache_create(twsk_prot->twsk_slab_name, 4280 twsk_prot->twsk_obj_size, 0, 4281 SLAB_ACCOUNT | prot->slab_flags, 4282 NULL); 4283 if (!twsk_prot->twsk_slab) { 4284 pr_crit("%s: Can't create timewait sock SLAB cache!\n", 4285 prot->name); 4286 return -ENOMEM; 4287 } 4288 4289 return 0; 4290 } 4291 4292 static void req_prot_cleanup(struct request_sock_ops *rsk_prot) 4293 { 4294 if (!rsk_prot) 4295 return; 4296 kfree(rsk_prot->slab_name); 4297 rsk_prot->slab_name = NULL; 4298 kmem_cache_destroy(rsk_prot->slab); 4299 rsk_prot->slab = NULL; 4300 } 4301 4302 static int req_prot_init(const struct proto *prot) 4303 { 4304 struct request_sock_ops *rsk_prot = prot->rsk_prot; 4305 4306 if (!rsk_prot) 4307 return 0; 4308 4309 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s", 4310 prot->name); 4311 if (!rsk_prot->slab_name) 4312 return -ENOMEM; 4313 4314 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name, 4315 rsk_prot->obj_size, 0, 4316 SLAB_ACCOUNT | prot->slab_flags, 4317 NULL); 4318 4319 if (!rsk_prot->slab) { 4320 pr_crit("%s: Can't create request sock SLAB cache!\n", 4321 prot->name); 4322 return -ENOMEM; 4323 } 4324 return 0; 4325 } 4326 4327 int proto_register(struct proto *prot, int alloc_slab) 4328 { 4329 int ret = -ENOBUFS; 4330 4331 if (prot->memory_allocated && !prot->sysctl_mem) { 4332 pr_err("%s: missing sysctl_mem\n", prot->name); 4333 return -EINVAL; 4334 } 4335 if (prot->memory_allocated && !prot->per_cpu_fw_alloc) { 4336 pr_err("%s: missing per_cpu_fw_alloc\n", prot->name); 4337 return -EINVAL; 4338 } 4339 if (alloc_slab) { 4340 struct kmem_cache_args args = { 4341 .useroffset = prot->useroffset, 4342 .usersize = prot->usersize, 4343 .freeptr_offset = prot->freeptr_offset, 4344 .use_freeptr_offset = !!prot->freeptr_offset, 4345 }; 4346 4347 prot->slab = kmem_cache_create(prot->name, prot->obj_size, 4348 &args, 4349 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT | 4350 prot->slab_flags); 4351 if (prot->slab == NULL) { 4352 pr_crit("%s: Can't create sock SLAB cache!\n", 4353 prot->name); 4354 goto out; 4355 } 4356 4357 if (req_prot_init(prot)) 4358 goto out_free_request_sock_slab; 4359 4360 if (tw_prot_init(prot)) 4361 goto out_free_timewait_sock_slab; 4362 } 4363 4364 mutex_lock(&proto_list_mutex); 4365 ret = assign_proto_idx(prot); 4366 if (ret) { 4367 mutex_unlock(&proto_list_mutex); 4368 goto out_free_timewait_sock_slab; 4369 } 4370 list_add(&prot->node, &proto_list); 4371 mutex_unlock(&proto_list_mutex); 4372 return ret; 4373 4374 out_free_timewait_sock_slab: 4375 if (alloc_slab) 4376 tw_prot_cleanup(prot->twsk_prot); 4377 out_free_request_sock_slab: 4378 if (alloc_slab) { 4379 req_prot_cleanup(prot->rsk_prot); 4380 4381 kmem_cache_destroy(prot->slab); 4382 prot->slab = NULL; 4383 } 4384 out: 4385 return ret; 4386 } 4387 EXPORT_SYMBOL(proto_register); 4388 4389 void proto_unregister(struct proto *prot) 4390 { 4391 mutex_lock(&proto_list_mutex); 4392 release_proto_idx(prot); 4393 list_del(&prot->node); 4394 mutex_unlock(&proto_list_mutex); 4395 4396 kmem_cache_destroy(prot->slab); 4397 prot->slab = NULL; 4398 4399 req_prot_cleanup(prot->rsk_prot); 4400 tw_prot_cleanup(prot->twsk_prot); 4401 } 4402 EXPORT_SYMBOL(proto_unregister); 4403 4404 int sock_load_diag_module(int family, int protocol) 4405 { 4406 if (!protocol) { 4407 if (!sock_is_registered(family)) 4408 return -ENOENT; 4409 4410 return request_module("net-pf-%d-proto-%d-type-%d", PF_NETLINK, 4411 NETLINK_SOCK_DIAG, family); 4412 } 4413 4414 #ifdef CONFIG_INET 4415 if (family == AF_INET && 4416 protocol != IPPROTO_RAW && 4417 protocol < MAX_INET_PROTOS && 4418 !rcu_access_pointer(inet_protos[protocol])) 4419 return -ENOENT; 4420 #endif 4421 4422 return request_module("net-pf-%d-proto-%d-type-%d-%d", PF_NETLINK, 4423 NETLINK_SOCK_DIAG, family, protocol); 4424 } 4425 EXPORT_SYMBOL(sock_load_diag_module); 4426 4427 #ifdef CONFIG_PROC_FS 4428 static void *proto_seq_start(struct seq_file *seq, loff_t *pos) 4429 __acquires(proto_list_mutex) 4430 { 4431 mutex_lock(&proto_list_mutex); 4432 return seq_list_start_head(&proto_list, *pos); 4433 } 4434 4435 static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos) 4436 { 4437 return seq_list_next(v, &proto_list, pos); 4438 } 4439 4440 static void proto_seq_stop(struct seq_file *seq, void *v) 4441 __releases(proto_list_mutex) 4442 { 4443 mutex_unlock(&proto_list_mutex); 4444 } 4445 4446 static char proto_method_implemented(const void *method) 4447 { 4448 return method == NULL ? 'n' : 'y'; 4449 } 4450 static long sock_prot_memory_allocated(struct proto *proto) 4451 { 4452 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L; 4453 } 4454 4455 static const char *sock_prot_memory_pressure(struct proto *proto) 4456 { 4457 return proto->memory_pressure != NULL ? 4458 proto_memory_pressure(proto) ? "yes" : "no" : "NI"; 4459 } 4460 4461 static void proto_seq_printf(struct seq_file *seq, struct proto *proto) 4462 { 4463 4464 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s " 4465 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n", 4466 proto->name, 4467 proto->obj_size, 4468 sock_prot_inuse_get(seq_file_net(seq), proto), 4469 sock_prot_memory_allocated(proto), 4470 sock_prot_memory_pressure(proto), 4471 proto->max_header, 4472 proto->slab == NULL ? "no" : "yes", 4473 module_name(proto->owner), 4474 proto_method_implemented(proto->close), 4475 proto_method_implemented(proto->connect), 4476 proto_method_implemented(proto->disconnect), 4477 proto_method_implemented(proto->accept), 4478 proto_method_implemented(proto->ioctl), 4479 proto_method_implemented(proto->init), 4480 proto_method_implemented(proto->destroy), 4481 proto_method_implemented(proto->shutdown), 4482 proto_method_implemented(proto->setsockopt), 4483 proto_method_implemented(proto->getsockopt), 4484 proto_method_implemented(proto->sendmsg), 4485 proto_method_implemented(proto->recvmsg), 4486 proto_method_implemented(proto->bind), 4487 proto_method_implemented(proto->backlog_rcv), 4488 proto_method_implemented(proto->hash), 4489 proto_method_implemented(proto->unhash), 4490 proto_method_implemented(proto->get_port), 4491 proto_method_implemented(proto->enter_memory_pressure)); 4492 } 4493 4494 static int proto_seq_show(struct seq_file *seq, void *v) 4495 { 4496 if (v == &proto_list) 4497 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s", 4498 "protocol", 4499 "size", 4500 "sockets", 4501 "memory", 4502 "press", 4503 "maxhdr", 4504 "slab", 4505 "module", 4506 "cl co di ac io in de sh ss gs se re bi br ha uh gp em\n"); 4507 else 4508 proto_seq_printf(seq, list_entry(v, struct proto, node)); 4509 return 0; 4510 } 4511 4512 static const struct seq_operations proto_seq_ops = { 4513 .start = proto_seq_start, 4514 .next = proto_seq_next, 4515 .stop = proto_seq_stop, 4516 .show = proto_seq_show, 4517 }; 4518 4519 static __net_init int proto_init_net(struct net *net) 4520 { 4521 if (!proc_create_net("protocols", 0444, net->proc_net, &proto_seq_ops, 4522 sizeof(struct seq_net_private))) 4523 return -ENOMEM; 4524 4525 return 0; 4526 } 4527 4528 static __net_exit void proto_exit_net(struct net *net) 4529 { 4530 remove_proc_entry("protocols", net->proc_net); 4531 } 4532 4533 4534 static __net_initdata struct pernet_operations proto_net_ops = { 4535 .init = proto_init_net, 4536 .exit = proto_exit_net, 4537 }; 4538 4539 static int __init proto_init(void) 4540 { 4541 return register_pernet_subsys(&proto_net_ops); 4542 } 4543 4544 subsys_initcall(proto_init); 4545 4546 #endif /* PROC_FS */ 4547 4548 #ifdef CONFIG_NET_RX_BUSY_POLL 4549 bool sk_busy_loop_end(void *p, unsigned long start_time) 4550 { 4551 struct sock *sk = p; 4552 4553 if (!skb_queue_empty_lockless(&sk->sk_receive_queue)) 4554 return true; 4555 4556 if (sk_is_udp(sk) && 4557 !skb_queue_empty_lockless(&udp_sk(sk)->reader_queue)) 4558 return true; 4559 4560 return sk_busy_loop_timeout(sk, start_time); 4561 } 4562 EXPORT_SYMBOL(sk_busy_loop_end); 4563 #endif /* CONFIG_NET_RX_BUSY_POLL */ 4564 4565 int sock_bind_add(struct sock *sk, struct sockaddr_unsized *addr, int addr_len) 4566 { 4567 if (!sk->sk_prot->bind_add) 4568 return -EOPNOTSUPP; 4569 return sk->sk_prot->bind_add(sk, addr, addr_len); 4570 } 4571 EXPORT_SYMBOL(sock_bind_add); 4572 4573 /* Copy 'size' bytes from userspace and return `size` back to userspace */ 4574 int sock_ioctl_inout(struct sock *sk, unsigned int cmd, 4575 void __user *arg, void *karg, size_t size) 4576 { 4577 int ret; 4578 4579 if (copy_from_user(karg, arg, size)) 4580 return -EFAULT; 4581 4582 ret = READ_ONCE(sk->sk_prot)->ioctl(sk, cmd, karg); 4583 if (ret) 4584 return ret; 4585 4586 if (copy_to_user(arg, karg, size)) 4587 return -EFAULT; 4588 4589 return 0; 4590 } 4591 EXPORT_SYMBOL(sock_ioctl_inout); 4592 4593 /* This is the most common ioctl prep function, where the result (4 bytes) is 4594 * copied back to userspace if the ioctl() returns successfully. No input is 4595 * copied from userspace as input argument. 4596 */ 4597 static int sock_ioctl_out(struct sock *sk, unsigned int cmd, void __user *arg) 4598 { 4599 int ret, karg = 0; 4600 4601 ret = READ_ONCE(sk->sk_prot)->ioctl(sk, cmd, &karg); 4602 if (ret) 4603 return ret; 4604 4605 return put_user(karg, (int __user *)arg); 4606 } 4607 4608 /* A wrapper around sock ioctls, which copies the data from userspace 4609 * (depending on the protocol/ioctl), and copies back the result to userspace. 4610 * The main motivation for this function is to pass kernel memory to the 4611 * protocol ioctl callbacks, instead of userspace memory. 4612 */ 4613 int sk_ioctl(struct sock *sk, unsigned int cmd, void __user *arg) 4614 { 4615 int rc = 1; 4616 4617 if (sk->sk_type == SOCK_RAW && sk->sk_family == AF_INET) 4618 rc = ipmr_sk_ioctl(sk, cmd, arg); 4619 else if (sk->sk_type == SOCK_RAW && sk->sk_family == AF_INET6) 4620 rc = ip6mr_sk_ioctl(sk, cmd, arg); 4621 else if (sk_is_phonet(sk)) 4622 rc = phonet_sk_ioctl(sk, cmd, arg); 4623 4624 /* If ioctl was processed, returns its value */ 4625 if (rc <= 0) 4626 return rc; 4627 4628 /* Otherwise call the default handler */ 4629 return sock_ioctl_out(sk, cmd, arg); 4630 } 4631 EXPORT_SYMBOL(sk_ioctl); 4632 4633 static int __init sock_struct_check(void) 4634 { 4635 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_drops); 4636 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_peek_off); 4637 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_error_queue); 4638 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_receive_queue); 4639 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_backlog); 4640 4641 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rx_dst); 4642 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rx_dst_ifindex); 4643 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rx_dst_cookie); 4644 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rcvbuf); 4645 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_filter); 4646 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_wq); 4647 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_data_ready); 4648 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rcvtimeo); 4649 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rcvlowat); 4650 4651 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rxtx, sk_err); 4652 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rxtx, sk_socket); 4653 #ifdef CONFIG_MEMCG 4654 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rxtx, sk_memcg); 4655 #endif 4656 4657 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_lock); 4658 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_reserved_mem); 4659 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_forward_alloc); 4660 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_tsflags); 4661 4662 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_omem_alloc); 4663 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_omem_alloc); 4664 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_err_soft); 4665 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_wmem_queued); 4666 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_wmem_alloc); 4667 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_tsq_flags); 4668 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_send_head); 4669 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_write_queue); 4670 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_write_pending); 4671 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_frag); 4672 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_timer); 4673 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_pacing_rate); 4674 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_zckey); 4675 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_tskey); 4676 4677 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_dst_pending_confirm); 4678 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_pacing_status); 4679 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_max_pacing_rate); 4680 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_sndtimeo); 4681 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_priority); 4682 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_mark); 4683 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_uid); 4684 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_protocol); 4685 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_dst_cache); 4686 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_route_caps); 4687 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_gso_type); 4688 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_gso_max_size); 4689 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_allocation); 4690 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_txhash); 4691 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_sndbuf); 4692 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_gso_max_segs); 4693 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_pacing_shift); 4694 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_use_task_frag); 4695 return 0; 4696 } 4697 4698 core_initcall(sock_struct_check); 4699