1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * Generic socket support routines. Memory allocators, socket lock/release 8 * handler for protocols to use and generic option handler. 9 * 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Florian La Roche, <flla@stud.uni-sb.de> 13 * Alan Cox, <A.Cox@swansea.ac.uk> 14 * 15 * Fixes: 16 * Alan Cox : Numerous verify_area() problems 17 * Alan Cox : Connecting on a connecting socket 18 * now returns an error for tcp. 19 * Alan Cox : sock->protocol is set correctly. 20 * and is not sometimes left as 0. 21 * Alan Cox : connect handles icmp errors on a 22 * connect properly. Unfortunately there 23 * is a restart syscall nasty there. I 24 * can't match BSD without hacking the C 25 * library. Ideas urgently sought! 26 * Alan Cox : Disallow bind() to addresses that are 27 * not ours - especially broadcast ones!! 28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost) 29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets, 30 * instead they leave that for the DESTROY timer. 31 * Alan Cox : Clean up error flag in accept 32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer 33 * was buggy. Put a remove_sock() in the handler 34 * for memory when we hit 0. Also altered the timer 35 * code. The ACK stuff can wait and needs major 36 * TCP layer surgery. 37 * Alan Cox : Fixed TCP ack bug, removed remove sock 38 * and fixed timer/inet_bh race. 39 * Alan Cox : Added zapped flag for TCP 40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code 41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb 42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources 43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing. 44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so... 45 * Rick Sladkey : Relaxed UDP rules for matching packets. 46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support 47 * Pauline Middelink : identd support 48 * Alan Cox : Fixed connect() taking signals I think. 49 * Alan Cox : SO_LINGER supported 50 * Alan Cox : Error reporting fixes 51 * Anonymous : inet_create tidied up (sk->reuse setting) 52 * Alan Cox : inet sockets don't set sk->type! 53 * Alan Cox : Split socket option code 54 * Alan Cox : Callbacks 55 * Alan Cox : Nagle flag for Charles & Johannes stuff 56 * Alex : Removed restriction on inet fioctl 57 * Alan Cox : Splitting INET from NET core 58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt() 59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code 60 * Alan Cox : Split IP from generic code 61 * Alan Cox : New kfree_skbmem() 62 * Alan Cox : Make SO_DEBUG superuser only. 63 * Alan Cox : Allow anyone to clear SO_DEBUG 64 * (compatibility fix) 65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput. 66 * Alan Cox : Allocator for a socket is settable. 67 * Alan Cox : SO_ERROR includes soft errors. 68 * Alan Cox : Allow NULL arguments on some SO_ opts 69 * Alan Cox : Generic socket allocation to make hooks 70 * easier (suggested by Craig Metz). 71 * Michael Pall : SO_ERROR returns positive errno again 72 * Steve Whitehouse: Added default destructor to free 73 * protocol private data. 74 * Steve Whitehouse: Added various other default routines 75 * common to several socket families. 76 * Chris Evans : Call suser() check last on F_SETOWN 77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER. 78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s() 79 * Andi Kleen : Fix write_space callback 80 * Chris Evans : Security fixes - signedness again 81 * Arnaldo C. Melo : cleanups, use skb_queue_purge 82 * 83 * To Fix: 84 */ 85 86 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 87 88 #include <linux/unaligned.h> 89 #include <linux/capability.h> 90 #include <linux/errno.h> 91 #include <linux/errqueue.h> 92 #include <linux/types.h> 93 #include <linux/socket.h> 94 #include <linux/in.h> 95 #include <linux/kernel.h> 96 #include <linux/module.h> 97 #include <linux/proc_fs.h> 98 #include <linux/seq_file.h> 99 #include <linux/sched.h> 100 #include <linux/sched/mm.h> 101 #include <linux/timer.h> 102 #include <linux/string.h> 103 #include <linux/sockios.h> 104 #include <linux/net.h> 105 #include <linux/mm.h> 106 #include <linux/slab.h> 107 #include <linux/interrupt.h> 108 #include <linux/poll.h> 109 #include <linux/tcp.h> 110 #include <linux/udp.h> 111 #include <linux/init.h> 112 #include <linux/highmem.h> 113 #include <linux/user_namespace.h> 114 #include <linux/static_key.h> 115 #include <linux/memcontrol.h> 116 #include <linux/prefetch.h> 117 #include <linux/compat.h> 118 #include <linux/mroute.h> 119 #include <linux/mroute6.h> 120 #include <linux/icmpv6.h> 121 122 #include <linux/uaccess.h> 123 124 #include <linux/netdevice.h> 125 #include <net/protocol.h> 126 #include <linux/skbuff.h> 127 #include <linux/skbuff_ref.h> 128 #include <net/net_namespace.h> 129 #include <net/request_sock.h> 130 #include <net/sock.h> 131 #include <net/proto_memory.h> 132 #include <net/page_pool/types.h> 133 #include <linux/net_tstamp.h> 134 #include <net/xfrm.h> 135 #include <linux/ipsec.h> 136 #include <net/cls_cgroup.h> 137 #include <net/netprio_cgroup.h> 138 #include <linux/sock_diag.h> 139 140 #include <linux/filter.h> 141 #include <net/sock_reuseport.h> 142 #include <net/bpf_sk_storage.h> 143 144 #include <trace/events/sock.h> 145 146 #include <net/psp.h> 147 #include <net/tcp.h> 148 #include <net/busy_poll.h> 149 #include <net/phonet/phonet.h> 150 151 #include <linux/ethtool.h> 152 153 #include <uapi/linux/pidfd.h> 154 155 #include "dev.h" 156 157 static DEFINE_MUTEX(proto_list_mutex); 158 static LIST_HEAD(proto_list); 159 160 static void sock_def_write_space_wfree(struct sock *sk, int wmem_alloc); 161 static void sock_def_write_space(struct sock *sk); 162 163 /** 164 * sk_ns_capable - General socket capability test 165 * @sk: Socket to use a capability on or through 166 * @user_ns: The user namespace of the capability to use 167 * @cap: The capability to use 168 * 169 * Test to see if the opener of the socket had when the socket was 170 * created and the current process has the capability @cap in the user 171 * namespace @user_ns. 172 */ 173 bool sk_ns_capable(const struct sock *sk, 174 struct user_namespace *user_ns, int cap) 175 { 176 return file_ns_capable(sk->sk_socket->file, user_ns, cap) && 177 ns_capable(user_ns, cap); 178 } 179 EXPORT_SYMBOL(sk_ns_capable); 180 181 /** 182 * sk_capable - Socket global capability test 183 * @sk: Socket to use a capability on or through 184 * @cap: The global capability to use 185 * 186 * Test to see if the opener of the socket had when the socket was 187 * created and the current process has the capability @cap in all user 188 * namespaces. 189 */ 190 bool sk_capable(const struct sock *sk, int cap) 191 { 192 return sk_ns_capable(sk, &init_user_ns, cap); 193 } 194 EXPORT_SYMBOL(sk_capable); 195 196 /** 197 * sk_net_capable - Network namespace socket capability test 198 * @sk: Socket to use a capability on or through 199 * @cap: The capability to use 200 * 201 * Test to see if the opener of the socket had when the socket was created 202 * and the current process has the capability @cap over the network namespace 203 * the socket is a member of. 204 */ 205 bool sk_net_capable(const struct sock *sk, int cap) 206 { 207 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap); 208 } 209 EXPORT_SYMBOL(sk_net_capable); 210 211 /* 212 * Each address family might have different locking rules, so we have 213 * one slock key per address family and separate keys for internal and 214 * userspace sockets. 215 */ 216 static struct lock_class_key af_family_keys[AF_MAX]; 217 static struct lock_class_key af_family_kern_keys[AF_MAX]; 218 static struct lock_class_key af_family_slock_keys[AF_MAX]; 219 static struct lock_class_key af_family_kern_slock_keys[AF_MAX]; 220 221 /* 222 * Make lock validator output more readable. (we pre-construct these 223 * strings build-time, so that runtime initialization of socket 224 * locks is fast): 225 */ 226 227 #define _sock_locks(x) \ 228 x "AF_UNSPEC", x "AF_UNIX" , x "AF_INET" , \ 229 x "AF_AX25" , x "AF_IPX" , x "AF_APPLETALK", \ 230 x "AF_NETROM", x "AF_BRIDGE" , x "AF_ATMPVC" , \ 231 x "AF_X25" , x "AF_INET6" , x "AF_ROSE" , \ 232 x "AF_DECnet", x "AF_NETBEUI" , x "AF_SECURITY" , \ 233 x "AF_KEY" , x "AF_NETLINK" , x "AF_PACKET" , \ 234 x "AF_ASH" , x "AF_ECONET" , x "AF_ATMSVC" , \ 235 x "AF_RDS" , x "AF_SNA" , x "AF_IRDA" , \ 236 x "AF_PPPOX" , x "AF_WANPIPE" , x "AF_LLC" , \ 237 x "27" , x "28" , x "AF_CAN" , \ 238 x "AF_TIPC" , x "AF_BLUETOOTH", x "IUCV" , \ 239 x "AF_RXRPC" , x "AF_ISDN" , x "AF_PHONET" , \ 240 x "AF_IEEE802154", x "AF_CAIF" , x "AF_ALG" , \ 241 x "AF_NFC" , x "AF_VSOCK" , x "AF_KCM" , \ 242 x "AF_QIPCRTR", x "AF_SMC" , x "AF_XDP" , \ 243 x "AF_MCTP" , \ 244 x "AF_MAX" 245 246 static const char *const af_family_key_strings[AF_MAX+1] = { 247 _sock_locks("sk_lock-") 248 }; 249 static const char *const af_family_slock_key_strings[AF_MAX+1] = { 250 _sock_locks("slock-") 251 }; 252 static const char *const af_family_clock_key_strings[AF_MAX+1] = { 253 _sock_locks("clock-") 254 }; 255 256 static const char *const af_family_kern_key_strings[AF_MAX+1] = { 257 _sock_locks("k-sk_lock-") 258 }; 259 static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = { 260 _sock_locks("k-slock-") 261 }; 262 static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = { 263 _sock_locks("k-clock-") 264 }; 265 static const char *const af_family_rlock_key_strings[AF_MAX+1] = { 266 _sock_locks("rlock-") 267 }; 268 static const char *const af_family_wlock_key_strings[AF_MAX+1] = { 269 _sock_locks("wlock-") 270 }; 271 static const char *const af_family_elock_key_strings[AF_MAX+1] = { 272 _sock_locks("elock-") 273 }; 274 275 /* 276 * sk_callback_lock and sk queues locking rules are per-address-family, 277 * so split the lock classes by using a per-AF key: 278 */ 279 static struct lock_class_key af_callback_keys[AF_MAX]; 280 static struct lock_class_key af_rlock_keys[AF_MAX]; 281 static struct lock_class_key af_wlock_keys[AF_MAX]; 282 static struct lock_class_key af_elock_keys[AF_MAX]; 283 static struct lock_class_key af_kern_callback_keys[AF_MAX]; 284 285 /* Run time adjustable parameters. */ 286 __u32 sysctl_wmem_max __read_mostly = 4 << 20; 287 EXPORT_SYMBOL(sysctl_wmem_max); 288 __u32 sysctl_rmem_max __read_mostly = 4 << 20; 289 EXPORT_SYMBOL(sysctl_rmem_max); 290 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_DEFAULT; 291 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_DEFAULT; 292 293 DEFINE_STATIC_KEY_FALSE(memalloc_socks_key); 294 EXPORT_SYMBOL_GPL(memalloc_socks_key); 295 296 /** 297 * sk_set_memalloc - sets %SOCK_MEMALLOC 298 * @sk: socket to set it on 299 * 300 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves. 301 * It's the responsibility of the admin to adjust min_free_kbytes 302 * to meet the requirements 303 */ 304 void sk_set_memalloc(struct sock *sk) 305 { 306 sock_set_flag(sk, SOCK_MEMALLOC); 307 sk->sk_allocation |= __GFP_MEMALLOC; 308 static_branch_inc(&memalloc_socks_key); 309 } 310 EXPORT_SYMBOL_GPL(sk_set_memalloc); 311 312 void sk_clear_memalloc(struct sock *sk) 313 { 314 sock_reset_flag(sk, SOCK_MEMALLOC); 315 sk->sk_allocation &= ~__GFP_MEMALLOC; 316 static_branch_dec(&memalloc_socks_key); 317 318 /* 319 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward 320 * progress of swapping. SOCK_MEMALLOC may be cleared while 321 * it has rmem allocations due to the last swapfile being deactivated 322 * but there is a risk that the socket is unusable due to exceeding 323 * the rmem limits. Reclaim the reserves and obey rmem limits again. 324 */ 325 sk_mem_reclaim(sk); 326 } 327 EXPORT_SYMBOL_GPL(sk_clear_memalloc); 328 329 int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb) 330 { 331 int ret; 332 unsigned int noreclaim_flag; 333 334 /* these should have been dropped before queueing */ 335 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC)); 336 337 noreclaim_flag = memalloc_noreclaim_save(); 338 ret = INDIRECT_CALL_INET(sk->sk_backlog_rcv, 339 tcp_v6_do_rcv, 340 tcp_v4_do_rcv, 341 sk, skb); 342 memalloc_noreclaim_restore(noreclaim_flag); 343 344 return ret; 345 } 346 EXPORT_SYMBOL(__sk_backlog_rcv); 347 348 void sk_error_report(struct sock *sk) 349 { 350 sk->sk_error_report(sk); 351 352 switch (sk->sk_family) { 353 case AF_INET: 354 fallthrough; 355 case AF_INET6: 356 trace_inet_sk_error_report(sk); 357 break; 358 default: 359 break; 360 } 361 } 362 EXPORT_SYMBOL(sk_error_report); 363 364 int sock_get_timeout(long timeo, void *optval, bool old_timeval) 365 { 366 struct __kernel_sock_timeval tv; 367 368 if (timeo == MAX_SCHEDULE_TIMEOUT) { 369 tv.tv_sec = 0; 370 tv.tv_usec = 0; 371 } else { 372 tv.tv_sec = timeo / HZ; 373 tv.tv_usec = ((timeo % HZ) * USEC_PER_SEC) / HZ; 374 } 375 376 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) { 377 struct old_timeval32 tv32 = { tv.tv_sec, tv.tv_usec }; 378 *(struct old_timeval32 *)optval = tv32; 379 return sizeof(tv32); 380 } 381 382 if (old_timeval) { 383 struct __kernel_old_timeval old_tv; 384 old_tv.tv_sec = tv.tv_sec; 385 old_tv.tv_usec = tv.tv_usec; 386 *(struct __kernel_old_timeval *)optval = old_tv; 387 return sizeof(old_tv); 388 } 389 390 *(struct __kernel_sock_timeval *)optval = tv; 391 return sizeof(tv); 392 } 393 EXPORT_SYMBOL(sock_get_timeout); 394 395 int sock_copy_user_timeval(struct __kernel_sock_timeval *tv, 396 sockptr_t optval, int optlen, bool old_timeval) 397 { 398 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) { 399 struct old_timeval32 tv32; 400 401 if (optlen < sizeof(tv32)) 402 return -EINVAL; 403 404 if (copy_from_sockptr(&tv32, optval, sizeof(tv32))) 405 return -EFAULT; 406 tv->tv_sec = tv32.tv_sec; 407 tv->tv_usec = tv32.tv_usec; 408 } else if (old_timeval) { 409 struct __kernel_old_timeval old_tv; 410 411 if (optlen < sizeof(old_tv)) 412 return -EINVAL; 413 if (copy_from_sockptr(&old_tv, optval, sizeof(old_tv))) 414 return -EFAULT; 415 tv->tv_sec = old_tv.tv_sec; 416 tv->tv_usec = old_tv.tv_usec; 417 } else { 418 if (optlen < sizeof(*tv)) 419 return -EINVAL; 420 if (copy_from_sockptr(tv, optval, sizeof(*tv))) 421 return -EFAULT; 422 } 423 424 return 0; 425 } 426 EXPORT_SYMBOL(sock_copy_user_timeval); 427 428 static int sock_set_timeout(long *timeo_p, sockptr_t optval, int optlen, 429 bool old_timeval) 430 { 431 struct __kernel_sock_timeval tv; 432 int err = sock_copy_user_timeval(&tv, optval, optlen, old_timeval); 433 long val; 434 435 if (err) 436 return err; 437 438 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC) 439 return -EDOM; 440 441 if (tv.tv_sec < 0) { 442 static int warned __read_mostly; 443 444 WRITE_ONCE(*timeo_p, 0); 445 if (warned < 10 && net_ratelimit()) { 446 warned++; 447 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n", 448 __func__, current->comm, task_pid_nr(current)); 449 } 450 return 0; 451 } 452 val = MAX_SCHEDULE_TIMEOUT; 453 if ((tv.tv_sec || tv.tv_usec) && 454 (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT / HZ - 1))) 455 val = tv.tv_sec * HZ + DIV_ROUND_UP((unsigned long)tv.tv_usec, 456 USEC_PER_SEC / HZ); 457 WRITE_ONCE(*timeo_p, val); 458 return 0; 459 } 460 461 static bool sk_set_prio_allowed(const struct sock *sk, int val) 462 { 463 return ((val >= TC_PRIO_BESTEFFORT && val <= TC_PRIO_INTERACTIVE) || 464 sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) || 465 sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)); 466 } 467 468 static bool sock_needs_netstamp(const struct sock *sk) 469 { 470 switch (sk->sk_family) { 471 case AF_UNSPEC: 472 case AF_UNIX: 473 return false; 474 default: 475 return true; 476 } 477 } 478 479 static void sock_disable_timestamp(struct sock *sk, unsigned long flags) 480 { 481 if (sk->sk_flags & flags) { 482 sk->sk_flags &= ~flags; 483 if (sock_needs_netstamp(sk) && 484 !(sk->sk_flags & SK_FLAGS_TIMESTAMP)) 485 net_disable_timestamp(); 486 } 487 } 488 489 490 int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 491 { 492 unsigned long flags; 493 struct sk_buff_head *list = &sk->sk_receive_queue; 494 495 if (atomic_read(&sk->sk_rmem_alloc) >= READ_ONCE(sk->sk_rcvbuf)) { 496 sk_drops_inc(sk); 497 trace_sock_rcvqueue_full(sk, skb); 498 return -ENOMEM; 499 } 500 501 if (!sk_rmem_schedule(sk, skb, skb->truesize)) { 502 sk_drops_inc(sk); 503 return -ENOBUFS; 504 } 505 506 skb->dev = NULL; 507 skb_set_owner_r(skb, sk); 508 509 /* we escape from rcu protected region, make sure we dont leak 510 * a norefcounted dst 511 */ 512 skb_dst_force(skb); 513 514 spin_lock_irqsave(&list->lock, flags); 515 sock_skb_set_dropcount(sk, skb); 516 __skb_queue_tail(list, skb); 517 spin_unlock_irqrestore(&list->lock, flags); 518 519 if (!sock_flag(sk, SOCK_DEAD)) 520 sk->sk_data_ready(sk); 521 return 0; 522 } 523 EXPORT_SYMBOL(__sock_queue_rcv_skb); 524 525 enum skb_drop_reason 526 sock_queue_rcv_skb_reason(struct sock *sk, struct sk_buff *skb) 527 { 528 enum skb_drop_reason drop_reason; 529 int err; 530 531 drop_reason = sk_filter_reason(sk, skb); 532 if (drop_reason) 533 return drop_reason; 534 535 err = __sock_queue_rcv_skb(sk, skb); 536 switch (err) { 537 case -ENOMEM: 538 return SKB_DROP_REASON_SOCKET_RCVBUFF; 539 case -ENOBUFS: 540 return SKB_DROP_REASON_PROTO_MEM; 541 } 542 return SKB_NOT_DROPPED_YET; 543 } 544 EXPORT_SYMBOL(sock_queue_rcv_skb_reason); 545 546 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, 547 const int nested, unsigned int trim_cap, bool refcounted) 548 { 549 enum skb_drop_reason reason; 550 int rc = NET_RX_SUCCESS; 551 int err; 552 553 reason = sk_filter_trim_cap(sk, skb, trim_cap); 554 if (reason) 555 goto discard_and_relse; 556 557 skb->dev = NULL; 558 559 if (sk_rcvqueues_full(sk, READ_ONCE(sk->sk_rcvbuf))) { 560 sk_drops_inc(sk); 561 reason = SKB_DROP_REASON_SOCKET_RCVBUFF; 562 goto discard_and_relse; 563 } 564 if (nested) 565 bh_lock_sock_nested(sk); 566 else 567 bh_lock_sock(sk); 568 if (!sock_owned_by_user(sk)) { 569 /* 570 * trylock + unlock semantics: 571 */ 572 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_); 573 574 rc = sk_backlog_rcv(sk, skb); 575 576 mutex_release(&sk->sk_lock.dep_map, _RET_IP_); 577 } else if ((err = sk_add_backlog(sk, skb, READ_ONCE(sk->sk_rcvbuf)))) { 578 bh_unlock_sock(sk); 579 if (err == -ENOMEM) 580 reason = SKB_DROP_REASON_PFMEMALLOC; 581 if (err == -ENOBUFS) 582 reason = SKB_DROP_REASON_SOCKET_BACKLOG; 583 sk_drops_inc(sk); 584 goto discard_and_relse; 585 } 586 587 bh_unlock_sock(sk); 588 out: 589 if (refcounted) 590 sock_put(sk); 591 return rc; 592 discard_and_relse: 593 sk_skb_reason_drop(sk, skb, reason); 594 goto out; 595 } 596 EXPORT_SYMBOL(__sk_receive_skb); 597 598 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ip6_dst_check(struct dst_entry *, 599 u32)); 600 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *, 601 u32)); 602 struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie) 603 { 604 struct dst_entry *dst = __sk_dst_get(sk); 605 606 if (dst && READ_ONCE(dst->obsolete) && 607 INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check, 608 dst, cookie) == NULL) { 609 sk_tx_queue_clear(sk); 610 WRITE_ONCE(sk->sk_dst_pending_confirm, 0); 611 RCU_INIT_POINTER(sk->sk_dst_cache, NULL); 612 dst_release(dst); 613 return NULL; 614 } 615 616 return dst; 617 } 618 EXPORT_SYMBOL(__sk_dst_check); 619 620 struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie) 621 { 622 struct dst_entry *dst = sk_dst_get(sk); 623 624 if (dst && READ_ONCE(dst->obsolete) && 625 INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check, 626 dst, cookie) == NULL) { 627 sk_dst_reset(sk); 628 dst_release(dst); 629 return NULL; 630 } 631 632 return dst; 633 } 634 EXPORT_SYMBOL(sk_dst_check); 635 636 static int sock_bindtoindex_locked(struct sock *sk, int ifindex) 637 { 638 int ret = -ENOPROTOOPT; 639 #ifdef CONFIG_NETDEVICES 640 struct net *net = sock_net(sk); 641 642 /* Sorry... */ 643 ret = -EPERM; 644 if (sk->sk_bound_dev_if && !ns_capable(net->user_ns, CAP_NET_RAW)) 645 goto out; 646 647 ret = -EINVAL; 648 if (ifindex < 0) 649 goto out; 650 651 /* Paired with all READ_ONCE() done locklessly. */ 652 WRITE_ONCE(sk->sk_bound_dev_if, ifindex); 653 654 if (sk->sk_prot->rehash) 655 sk->sk_prot->rehash(sk); 656 sk_dst_reset(sk); 657 658 ret = 0; 659 660 out: 661 #endif 662 663 return ret; 664 } 665 666 int sock_bindtoindex(struct sock *sk, int ifindex, bool lock_sk) 667 { 668 int ret; 669 670 if (lock_sk) 671 lock_sock(sk); 672 ret = sock_bindtoindex_locked(sk, ifindex); 673 if (lock_sk) 674 release_sock(sk); 675 676 return ret; 677 } 678 EXPORT_SYMBOL(sock_bindtoindex); 679 680 static int sock_setbindtodevice(struct sock *sk, sockptr_t optval, int optlen) 681 { 682 int ret = -ENOPROTOOPT; 683 #ifdef CONFIG_NETDEVICES 684 struct net *net = sock_net(sk); 685 char devname[IFNAMSIZ]; 686 int index; 687 688 ret = -EINVAL; 689 if (optlen < 0) 690 goto out; 691 692 /* Bind this socket to a particular device like "eth0", 693 * as specified in the passed interface name. If the 694 * name is "" or the option length is zero the socket 695 * is not bound. 696 */ 697 if (optlen > IFNAMSIZ - 1) 698 optlen = IFNAMSIZ - 1; 699 memset(devname, 0, sizeof(devname)); 700 701 ret = -EFAULT; 702 if (copy_from_sockptr(devname, optval, optlen)) 703 goto out; 704 705 index = 0; 706 if (devname[0] != '\0') { 707 struct net_device *dev; 708 709 rcu_read_lock(); 710 dev = dev_get_by_name_rcu(net, devname); 711 if (dev) 712 index = dev->ifindex; 713 rcu_read_unlock(); 714 ret = -ENODEV; 715 if (!dev) 716 goto out; 717 } 718 719 sockopt_lock_sock(sk); 720 ret = sock_bindtoindex_locked(sk, index); 721 sockopt_release_sock(sk); 722 out: 723 #endif 724 725 return ret; 726 } 727 728 static int sock_getbindtodevice(struct sock *sk, sockptr_t optval, 729 sockptr_t optlen, int len) 730 { 731 int ret = -ENOPROTOOPT; 732 #ifdef CONFIG_NETDEVICES 733 int bound_dev_if = READ_ONCE(sk->sk_bound_dev_if); 734 struct net *net = sock_net(sk); 735 char devname[IFNAMSIZ]; 736 737 if (bound_dev_if == 0) { 738 len = 0; 739 goto zero; 740 } 741 742 ret = -EINVAL; 743 if (len < IFNAMSIZ) 744 goto out; 745 746 ret = netdev_get_name(net, devname, bound_dev_if); 747 if (ret) 748 goto out; 749 750 len = strlen(devname) + 1; 751 752 ret = -EFAULT; 753 if (copy_to_sockptr(optval, devname, len)) 754 goto out; 755 756 zero: 757 ret = -EFAULT; 758 if (copy_to_sockptr(optlen, &len, sizeof(int))) 759 goto out; 760 761 ret = 0; 762 763 out: 764 #endif 765 766 return ret; 767 } 768 769 bool sk_mc_loop(const struct sock *sk) 770 { 771 if (dev_recursion_level()) 772 return false; 773 sk = sk_const_to_full_sk(sk); 774 if (!sk) 775 return true; 776 777 switch (READ_ONCE(sk->sk_family)) { 778 case AF_INET: 779 return inet_test_bit(MC_LOOP, sk); 780 #if IS_ENABLED(CONFIG_IPV6) 781 case AF_INET6: 782 return inet6_test_bit(MC6_LOOP, sk); 783 #endif 784 } 785 return true; 786 } 787 EXPORT_SYMBOL(sk_mc_loop); 788 789 void sock_set_reuseaddr(struct sock *sk) 790 { 791 lock_sock(sk); 792 sk->sk_reuse = SK_CAN_REUSE; 793 release_sock(sk); 794 } 795 EXPORT_SYMBOL(sock_set_reuseaddr); 796 797 void sock_set_reuseport(struct sock *sk) 798 { 799 lock_sock(sk); 800 sk->sk_reuseport = true; 801 release_sock(sk); 802 } 803 EXPORT_SYMBOL(sock_set_reuseport); 804 805 void sock_no_linger(struct sock *sk) 806 { 807 lock_sock(sk); 808 WRITE_ONCE(sk->sk_lingertime, 0); 809 sock_set_flag(sk, SOCK_LINGER); 810 release_sock(sk); 811 } 812 EXPORT_SYMBOL(sock_no_linger); 813 814 void sock_set_priority(struct sock *sk, u32 priority) 815 { 816 WRITE_ONCE(sk->sk_priority, priority); 817 } 818 EXPORT_SYMBOL(sock_set_priority); 819 820 void sock_set_sndtimeo(struct sock *sk, s64 secs) 821 { 822 if (secs && secs < MAX_SCHEDULE_TIMEOUT / HZ - 1) 823 WRITE_ONCE(sk->sk_sndtimeo, secs * HZ); 824 else 825 WRITE_ONCE(sk->sk_sndtimeo, MAX_SCHEDULE_TIMEOUT); 826 } 827 EXPORT_SYMBOL(sock_set_sndtimeo); 828 829 static void __sock_set_timestamps(struct sock *sk, bool val, bool new, bool ns) 830 { 831 sock_valbool_flag(sk, SOCK_RCVTSTAMP, val); 832 sock_valbool_flag(sk, SOCK_RCVTSTAMPNS, val && ns); 833 if (val) { 834 sock_valbool_flag(sk, SOCK_TSTAMP_NEW, new); 835 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 836 } 837 } 838 839 void sock_set_timestamp(struct sock *sk, int optname, bool valbool) 840 { 841 switch (optname) { 842 case SO_TIMESTAMP_OLD: 843 __sock_set_timestamps(sk, valbool, false, false); 844 break; 845 case SO_TIMESTAMP_NEW: 846 __sock_set_timestamps(sk, valbool, true, false); 847 break; 848 case SO_TIMESTAMPNS_OLD: 849 __sock_set_timestamps(sk, valbool, false, true); 850 break; 851 case SO_TIMESTAMPNS_NEW: 852 __sock_set_timestamps(sk, valbool, true, true); 853 break; 854 } 855 } 856 857 static int sock_timestamping_bind_phc(struct sock *sk, int phc_index) 858 { 859 struct net *net = sock_net(sk); 860 struct net_device *dev = NULL; 861 bool match = false; 862 int *vclock_index; 863 int i, num; 864 865 if (sk->sk_bound_dev_if) 866 dev = dev_get_by_index(net, sk->sk_bound_dev_if); 867 868 if (!dev) { 869 pr_err("%s: sock not bind to device\n", __func__); 870 return -EOPNOTSUPP; 871 } 872 873 num = ethtool_get_phc_vclocks(dev, &vclock_index); 874 dev_put(dev); 875 876 for (i = 0; i < num; i++) { 877 if (*(vclock_index + i) == phc_index) { 878 match = true; 879 break; 880 } 881 } 882 883 if (num > 0) 884 kfree(vclock_index); 885 886 if (!match) 887 return -EINVAL; 888 889 WRITE_ONCE(sk->sk_bind_phc, phc_index); 890 891 return 0; 892 } 893 894 int sock_set_timestamping(struct sock *sk, int optname, 895 struct so_timestamping timestamping) 896 { 897 int val = timestamping.flags; 898 int ret; 899 900 if (val & ~SOF_TIMESTAMPING_MASK) 901 return -EINVAL; 902 903 if (val & SOF_TIMESTAMPING_OPT_ID_TCP && 904 !(val & SOF_TIMESTAMPING_OPT_ID)) 905 return -EINVAL; 906 907 if (val & SOF_TIMESTAMPING_OPT_ID && 908 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) { 909 if (sk_is_tcp(sk)) { 910 if ((1 << sk->sk_state) & 911 (TCPF_CLOSE | TCPF_LISTEN)) 912 return -EINVAL; 913 if (val & SOF_TIMESTAMPING_OPT_ID_TCP) 914 atomic_set(&sk->sk_tskey, tcp_sk(sk)->write_seq); 915 else 916 atomic_set(&sk->sk_tskey, tcp_sk(sk)->snd_una); 917 } else { 918 atomic_set(&sk->sk_tskey, 0); 919 } 920 } 921 922 if (val & SOF_TIMESTAMPING_OPT_STATS && 923 !(val & SOF_TIMESTAMPING_OPT_TSONLY)) 924 return -EINVAL; 925 926 if (val & SOF_TIMESTAMPING_BIND_PHC) { 927 ret = sock_timestamping_bind_phc(sk, timestamping.bind_phc); 928 if (ret) 929 return ret; 930 } 931 932 WRITE_ONCE(sk->sk_tsflags, val); 933 sock_valbool_flag(sk, SOCK_TSTAMP_NEW, optname == SO_TIMESTAMPING_NEW); 934 sock_valbool_flag(sk, SOCK_TIMESTAMPING_ANY, !!(val & TSFLAGS_ANY)); 935 936 if (val & SOF_TIMESTAMPING_RX_SOFTWARE) 937 sock_enable_timestamp(sk, 938 SOCK_TIMESTAMPING_RX_SOFTWARE); 939 else 940 sock_disable_timestamp(sk, 941 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE)); 942 return 0; 943 } 944 945 #if defined(CONFIG_CGROUP_BPF) 946 void bpf_skops_tx_timestamping(struct sock *sk, struct sk_buff *skb, int op) 947 { 948 struct bpf_sock_ops_kern sock_ops; 949 950 memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); 951 sock_ops.op = op; 952 sock_ops.is_fullsock = 1; 953 sock_ops.sk = sk; 954 bpf_skops_init_skb(&sock_ops, skb, 0); 955 __cgroup_bpf_run_filter_sock_ops(sk, &sock_ops, CGROUP_SOCK_OPS); 956 } 957 #endif 958 959 void sock_set_keepalive(struct sock *sk) 960 { 961 lock_sock(sk); 962 if (sk->sk_prot->keepalive) 963 sk->sk_prot->keepalive(sk, true); 964 sock_valbool_flag(sk, SOCK_KEEPOPEN, true); 965 release_sock(sk); 966 } 967 EXPORT_SYMBOL(sock_set_keepalive); 968 969 static void __sock_set_rcvbuf(struct sock *sk, int val) 970 { 971 struct socket *sock = sk->sk_socket; 972 973 /* Ensure val * 2 fits into an int, to prevent max_t() from treating it 974 * as a negative value. 975 */ 976 val = min_t(int, val, INT_MAX / 2); 977 sk->sk_userlocks |= SOCK_RCVBUF_LOCK; 978 979 /* We double it on the way in to account for "struct sk_buff" etc. 980 * overhead. Applications assume that the SO_RCVBUF setting they make 981 * will allow that much actual data to be received on that socket. 982 * 983 * Applications are unaware that "struct sk_buff" and other overheads 984 * allocate from the receive buffer during socket buffer allocation. 985 * 986 * And after considering the possible alternatives, returning the value 987 * we actually used in getsockopt is the most desirable behavior. 988 */ 989 WRITE_ONCE(sk->sk_rcvbuf, max_t(int, val * 2, SOCK_MIN_RCVBUF)); 990 991 if (sock) { 992 const struct proto_ops *ops = READ_ONCE(sock->ops); 993 994 if (ops->set_rcvbuf) 995 ops->set_rcvbuf(sk, sk->sk_rcvbuf); 996 } 997 } 998 999 void sock_set_rcvbuf(struct sock *sk, int val) 1000 { 1001 lock_sock(sk); 1002 __sock_set_rcvbuf(sk, val); 1003 release_sock(sk); 1004 } 1005 EXPORT_SYMBOL(sock_set_rcvbuf); 1006 1007 static void __sock_set_mark(struct sock *sk, u32 val) 1008 { 1009 if (val != sk->sk_mark) { 1010 WRITE_ONCE(sk->sk_mark, val); 1011 sk_dst_reset(sk); 1012 } 1013 } 1014 1015 void sock_set_mark(struct sock *sk, u32 val) 1016 { 1017 lock_sock(sk); 1018 __sock_set_mark(sk, val); 1019 release_sock(sk); 1020 } 1021 EXPORT_SYMBOL(sock_set_mark); 1022 1023 static void sock_release_reserved_memory(struct sock *sk, int bytes) 1024 { 1025 /* Round down bytes to multiple of pages */ 1026 bytes = round_down(bytes, PAGE_SIZE); 1027 1028 WARN_ON(bytes > sk->sk_reserved_mem); 1029 WRITE_ONCE(sk->sk_reserved_mem, sk->sk_reserved_mem - bytes); 1030 sk_mem_reclaim(sk); 1031 } 1032 1033 static int sock_reserve_memory(struct sock *sk, int bytes) 1034 { 1035 long allocated; 1036 bool charged; 1037 int pages; 1038 1039 if (!mem_cgroup_sk_enabled(sk) || !sk_is_tcp(sk)) 1040 return -EOPNOTSUPP; 1041 1042 if (!bytes) 1043 return 0; 1044 1045 pages = sk_mem_pages(bytes); 1046 1047 /* pre-charge to memcg */ 1048 charged = mem_cgroup_sk_charge(sk, pages, 1049 GFP_KERNEL | __GFP_RETRY_MAYFAIL); 1050 if (!charged) 1051 return -ENOMEM; 1052 1053 if (sk->sk_bypass_prot_mem) 1054 goto success; 1055 1056 /* pre-charge to forward_alloc */ 1057 sk_memory_allocated_add(sk, pages); 1058 allocated = sk_memory_allocated(sk); 1059 1060 /* If the system goes into memory pressure with this 1061 * precharge, give up and return error. 1062 */ 1063 if (allocated > sk_prot_mem_limits(sk, 1)) { 1064 sk_memory_allocated_sub(sk, pages); 1065 mem_cgroup_sk_uncharge(sk, pages); 1066 return -ENOMEM; 1067 } 1068 1069 success: 1070 sk_forward_alloc_add(sk, pages << PAGE_SHIFT); 1071 1072 WRITE_ONCE(sk->sk_reserved_mem, 1073 sk->sk_reserved_mem + (pages << PAGE_SHIFT)); 1074 1075 return 0; 1076 } 1077 1078 #ifdef CONFIG_PAGE_POOL 1079 1080 /* This is the number of tokens and frags that the user can SO_DEVMEM_DONTNEED 1081 * in 1 syscall. The limit exists to limit the amount of memory the kernel 1082 * allocates to copy these tokens, and to prevent looping over the frags for 1083 * too long. 1084 */ 1085 #define MAX_DONTNEED_TOKENS 128 1086 #define MAX_DONTNEED_FRAGS 1024 1087 1088 static noinline_for_stack int 1089 sock_devmem_dontneed(struct sock *sk, sockptr_t optval, unsigned int optlen) 1090 { 1091 unsigned int num_tokens, i, j, netmem_num = 0; 1092 struct dmabuf_token *tokens; 1093 int ret = 0, num_frags = 0; 1094 netmem_ref netmems[16]; 1095 1096 if (!sk_is_tcp(sk)) 1097 return -EBADF; 1098 1099 if (optlen % sizeof(*tokens) || 1100 optlen > sizeof(*tokens) * MAX_DONTNEED_TOKENS) 1101 return -EINVAL; 1102 1103 num_tokens = optlen / sizeof(*tokens); 1104 tokens = kvmalloc_objs(*tokens, num_tokens); 1105 if (!tokens) 1106 return -ENOMEM; 1107 1108 if (copy_from_sockptr(tokens, optval, optlen)) { 1109 kvfree(tokens); 1110 return -EFAULT; 1111 } 1112 1113 xa_lock_bh(&sk->sk_user_frags); 1114 for (i = 0; i < num_tokens; i++) { 1115 for (j = 0; j < tokens[i].token_count; j++) { 1116 if (++num_frags > MAX_DONTNEED_FRAGS) 1117 goto frag_limit_reached; 1118 1119 netmem_ref netmem = (__force netmem_ref)__xa_erase( 1120 &sk->sk_user_frags, tokens[i].token_start + j); 1121 1122 if (!netmem || WARN_ON_ONCE(!netmem_is_net_iov(netmem))) 1123 continue; 1124 1125 netmems[netmem_num++] = netmem; 1126 if (netmem_num == ARRAY_SIZE(netmems)) { 1127 xa_unlock_bh(&sk->sk_user_frags); 1128 page_pool_put_netmem_bulk(netmems, 1129 ARRAY_SIZE(netmems)); 1130 netmem_num = 0; 1131 xa_lock_bh(&sk->sk_user_frags); 1132 } 1133 ret++; 1134 } 1135 } 1136 1137 frag_limit_reached: 1138 xa_unlock_bh(&sk->sk_user_frags); 1139 page_pool_put_netmem_bulk(netmems, netmem_num); 1140 1141 kvfree(tokens); 1142 return ret; 1143 } 1144 #endif 1145 1146 void sockopt_lock_sock(struct sock *sk) 1147 { 1148 /* When current->bpf_ctx is set, the setsockopt is called from 1149 * a bpf prog. bpf has ensured the sk lock has been 1150 * acquired before calling setsockopt(). 1151 */ 1152 if (has_current_bpf_ctx()) 1153 return; 1154 1155 lock_sock(sk); 1156 } 1157 EXPORT_SYMBOL(sockopt_lock_sock); 1158 1159 void sockopt_release_sock(struct sock *sk) 1160 { 1161 if (has_current_bpf_ctx()) 1162 return; 1163 1164 release_sock(sk); 1165 } 1166 EXPORT_SYMBOL(sockopt_release_sock); 1167 1168 bool sockopt_ns_capable(struct user_namespace *ns, int cap) 1169 { 1170 return has_current_bpf_ctx() || ns_capable(ns, cap); 1171 } 1172 EXPORT_SYMBOL(sockopt_ns_capable); 1173 1174 bool sockopt_capable(int cap) 1175 { 1176 return has_current_bpf_ctx() || capable(cap); 1177 } 1178 EXPORT_SYMBOL(sockopt_capable); 1179 1180 static int sockopt_validate_clockid(__kernel_clockid_t value) 1181 { 1182 switch (value) { 1183 case CLOCK_REALTIME: 1184 case CLOCK_MONOTONIC: 1185 case CLOCK_TAI: 1186 return 0; 1187 } 1188 return -EINVAL; 1189 } 1190 1191 /* 1192 * This is meant for all protocols to use and covers goings on 1193 * at the socket level. Everything here is generic. 1194 */ 1195 1196 int sk_setsockopt(struct sock *sk, int level, int optname, 1197 sockptr_t optval, unsigned int optlen) 1198 { 1199 struct so_timestamping timestamping; 1200 struct socket *sock = sk->sk_socket; 1201 struct sock_txtime sk_txtime; 1202 int val; 1203 int valbool; 1204 struct linger ling; 1205 int ret = 0; 1206 1207 /* 1208 * Options without arguments 1209 */ 1210 1211 if (optname == SO_BINDTODEVICE) 1212 return sock_setbindtodevice(sk, optval, optlen); 1213 1214 if (optlen < sizeof(int)) 1215 return -EINVAL; 1216 1217 if (copy_from_sockptr(&val, optval, sizeof(val))) 1218 return -EFAULT; 1219 1220 valbool = val ? 1 : 0; 1221 1222 /* handle options which do not require locking the socket. */ 1223 switch (optname) { 1224 case SO_PRIORITY: 1225 if (sk_set_prio_allowed(sk, val)) { 1226 sock_set_priority(sk, val); 1227 return 0; 1228 } 1229 return -EPERM; 1230 case SO_TYPE: 1231 case SO_PROTOCOL: 1232 case SO_DOMAIN: 1233 case SO_ERROR: 1234 return -ENOPROTOOPT; 1235 #ifdef CONFIG_NET_RX_BUSY_POLL 1236 case SO_BUSY_POLL: 1237 if (val < 0) 1238 return -EINVAL; 1239 WRITE_ONCE(sk->sk_ll_usec, val); 1240 return 0; 1241 case SO_PREFER_BUSY_POLL: 1242 if (valbool && !sockopt_capable(CAP_NET_ADMIN)) 1243 return -EPERM; 1244 WRITE_ONCE(sk->sk_prefer_busy_poll, valbool); 1245 return 0; 1246 case SO_BUSY_POLL_BUDGET: 1247 if (val > READ_ONCE(sk->sk_busy_poll_budget) && 1248 !sockopt_capable(CAP_NET_ADMIN)) 1249 return -EPERM; 1250 if (val < 0 || val > U16_MAX) 1251 return -EINVAL; 1252 WRITE_ONCE(sk->sk_busy_poll_budget, val); 1253 return 0; 1254 #endif 1255 case SO_MAX_PACING_RATE: 1256 { 1257 unsigned long ulval = (val == ~0U) ? ~0UL : (unsigned int)val; 1258 unsigned long pacing_rate; 1259 1260 if (sizeof(ulval) != sizeof(val) && 1261 optlen >= sizeof(ulval) && 1262 copy_from_sockptr(&ulval, optval, sizeof(ulval))) { 1263 return -EFAULT; 1264 } 1265 if (ulval != ~0UL) 1266 cmpxchg(&sk->sk_pacing_status, 1267 SK_PACING_NONE, 1268 SK_PACING_NEEDED); 1269 /* Pairs with READ_ONCE() from sk_getsockopt() */ 1270 WRITE_ONCE(sk->sk_max_pacing_rate, ulval); 1271 pacing_rate = READ_ONCE(sk->sk_pacing_rate); 1272 if (ulval < pacing_rate) 1273 WRITE_ONCE(sk->sk_pacing_rate, ulval); 1274 return 0; 1275 } 1276 case SO_TXREHASH: 1277 if (!sk_is_tcp(sk)) 1278 return -EOPNOTSUPP; 1279 if (val < -1 || val > 1) 1280 return -EINVAL; 1281 if ((u8)val == SOCK_TXREHASH_DEFAULT) 1282 val = READ_ONCE(sock_net(sk)->core.sysctl_txrehash); 1283 /* Paired with READ_ONCE() in tcp_rtx_synack() 1284 * and sk_getsockopt(). 1285 */ 1286 WRITE_ONCE(sk->sk_txrehash, (u8)val); 1287 return 0; 1288 case SO_PEEK_OFF: 1289 { 1290 int (*set_peek_off)(struct sock *sk, int val); 1291 1292 set_peek_off = READ_ONCE(sock->ops)->set_peek_off; 1293 if (set_peek_off) 1294 ret = set_peek_off(sk, val); 1295 else 1296 ret = -EOPNOTSUPP; 1297 return ret; 1298 } 1299 #ifdef CONFIG_PAGE_POOL 1300 case SO_DEVMEM_DONTNEED: 1301 return sock_devmem_dontneed(sk, optval, optlen); 1302 #endif 1303 case SO_SNDTIMEO_OLD: 1304 case SO_SNDTIMEO_NEW: 1305 return sock_set_timeout(&sk->sk_sndtimeo, optval, 1306 optlen, optname == SO_SNDTIMEO_OLD); 1307 case SO_RCVTIMEO_OLD: 1308 case SO_RCVTIMEO_NEW: 1309 return sock_set_timeout(&sk->sk_rcvtimeo, optval, 1310 optlen, optname == SO_RCVTIMEO_OLD); 1311 } 1312 1313 sockopt_lock_sock(sk); 1314 1315 switch (optname) { 1316 case SO_DEBUG: 1317 if (val && !sockopt_capable(CAP_NET_ADMIN)) 1318 ret = -EACCES; 1319 else 1320 sock_valbool_flag(sk, SOCK_DBG, valbool); 1321 break; 1322 case SO_REUSEADDR: 1323 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE); 1324 break; 1325 case SO_REUSEPORT: 1326 if (valbool && !sk_is_inet(sk)) 1327 ret = -EOPNOTSUPP; 1328 else 1329 sk->sk_reuseport = valbool; 1330 break; 1331 case SO_DONTROUTE: 1332 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool); 1333 sk_dst_reset(sk); 1334 break; 1335 case SO_BROADCAST: 1336 sock_valbool_flag(sk, SOCK_BROADCAST, valbool); 1337 break; 1338 case SO_SNDBUF: 1339 /* Don't error on this BSD doesn't and if you think 1340 * about it this is right. Otherwise apps have to 1341 * play 'guess the biggest size' games. RCVBUF/SNDBUF 1342 * are treated in BSD as hints 1343 */ 1344 val = min_t(u32, val, READ_ONCE(sysctl_wmem_max)); 1345 set_sndbuf: 1346 /* Ensure val * 2 fits into an int, to prevent max_t() 1347 * from treating it as a negative value. 1348 */ 1349 val = min_t(int, val, INT_MAX / 2); 1350 sk->sk_userlocks |= SOCK_SNDBUF_LOCK; 1351 WRITE_ONCE(sk->sk_sndbuf, 1352 max_t(int, val * 2, SOCK_MIN_SNDBUF)); 1353 /* Wake up sending tasks if we upped the value. */ 1354 sk->sk_write_space(sk); 1355 break; 1356 1357 case SO_SNDBUFFORCE: 1358 if (!sockopt_capable(CAP_NET_ADMIN)) { 1359 ret = -EPERM; 1360 break; 1361 } 1362 1363 /* No negative values (to prevent underflow, as val will be 1364 * multiplied by 2). 1365 */ 1366 if (val < 0) 1367 val = 0; 1368 goto set_sndbuf; 1369 1370 case SO_RCVBUF: 1371 /* Don't error on this BSD doesn't and if you think 1372 * about it this is right. Otherwise apps have to 1373 * play 'guess the biggest size' games. RCVBUF/SNDBUF 1374 * are treated in BSD as hints 1375 */ 1376 __sock_set_rcvbuf(sk, min_t(u32, val, READ_ONCE(sysctl_rmem_max))); 1377 break; 1378 1379 case SO_RCVBUFFORCE: 1380 if (!sockopt_capable(CAP_NET_ADMIN)) { 1381 ret = -EPERM; 1382 break; 1383 } 1384 1385 /* No negative values (to prevent underflow, as val will be 1386 * multiplied by 2). 1387 */ 1388 __sock_set_rcvbuf(sk, max(val, 0)); 1389 break; 1390 1391 case SO_KEEPALIVE: 1392 if (sk->sk_prot->keepalive) 1393 sk->sk_prot->keepalive(sk, valbool); 1394 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool); 1395 break; 1396 1397 case SO_OOBINLINE: 1398 sock_valbool_flag(sk, SOCK_URGINLINE, valbool); 1399 break; 1400 1401 case SO_NO_CHECK: 1402 sk->sk_no_check_tx = valbool; 1403 break; 1404 1405 case SO_LINGER: 1406 if (optlen < sizeof(ling)) { 1407 ret = -EINVAL; /* 1003.1g */ 1408 break; 1409 } 1410 if (copy_from_sockptr(&ling, optval, sizeof(ling))) { 1411 ret = -EFAULT; 1412 break; 1413 } 1414 if (!ling.l_onoff) { 1415 sock_reset_flag(sk, SOCK_LINGER); 1416 } else { 1417 unsigned long t_sec = ling.l_linger; 1418 1419 if (t_sec >= MAX_SCHEDULE_TIMEOUT / HZ) 1420 WRITE_ONCE(sk->sk_lingertime, MAX_SCHEDULE_TIMEOUT); 1421 else 1422 WRITE_ONCE(sk->sk_lingertime, t_sec * HZ); 1423 sock_set_flag(sk, SOCK_LINGER); 1424 } 1425 break; 1426 1427 case SO_BSDCOMPAT: 1428 break; 1429 1430 case SO_TIMESTAMP_OLD: 1431 case SO_TIMESTAMP_NEW: 1432 case SO_TIMESTAMPNS_OLD: 1433 case SO_TIMESTAMPNS_NEW: 1434 sock_set_timestamp(sk, optname, valbool); 1435 break; 1436 1437 case SO_TIMESTAMPING_NEW: 1438 case SO_TIMESTAMPING_OLD: 1439 if (optlen == sizeof(timestamping)) { 1440 if (copy_from_sockptr(×tamping, optval, 1441 sizeof(timestamping))) { 1442 ret = -EFAULT; 1443 break; 1444 } 1445 } else { 1446 memset(×tamping, 0, sizeof(timestamping)); 1447 timestamping.flags = val; 1448 } 1449 ret = sock_set_timestamping(sk, optname, timestamping); 1450 break; 1451 1452 case SO_RCVLOWAT: 1453 { 1454 int (*set_rcvlowat)(struct sock *sk, int val) = NULL; 1455 1456 if (val < 0) 1457 val = INT_MAX; 1458 if (sock) 1459 set_rcvlowat = READ_ONCE(sock->ops)->set_rcvlowat; 1460 if (set_rcvlowat) 1461 ret = set_rcvlowat(sk, val); 1462 else 1463 WRITE_ONCE(sk->sk_rcvlowat, val ? : 1); 1464 break; 1465 } 1466 case SO_ATTACH_FILTER: { 1467 struct sock_fprog fprog; 1468 1469 if (sk_is_tcp(sk) && 1470 !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1471 ret = -EPERM; 1472 break; 1473 } 1474 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen); 1475 if (!ret) 1476 ret = sk_attach_filter(&fprog, sk); 1477 break; 1478 } 1479 case SO_ATTACH_BPF: 1480 ret = -EINVAL; 1481 if (optlen == sizeof(u32)) { 1482 u32 ufd; 1483 1484 ret = -EFAULT; 1485 if (copy_from_sockptr(&ufd, optval, sizeof(ufd))) 1486 break; 1487 1488 ret = sk_attach_bpf(ufd, sk); 1489 } 1490 break; 1491 1492 case SO_ATTACH_REUSEPORT_CBPF: { 1493 struct sock_fprog fprog; 1494 1495 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen); 1496 if (!ret) 1497 ret = sk_reuseport_attach_filter(&fprog, sk); 1498 break; 1499 } 1500 case SO_ATTACH_REUSEPORT_EBPF: 1501 ret = -EINVAL; 1502 if (optlen == sizeof(u32)) { 1503 u32 ufd; 1504 1505 ret = -EFAULT; 1506 if (copy_from_sockptr(&ufd, optval, sizeof(ufd))) 1507 break; 1508 1509 ret = sk_reuseport_attach_bpf(ufd, sk); 1510 } 1511 break; 1512 1513 case SO_DETACH_REUSEPORT_BPF: 1514 ret = reuseport_detach_prog(sk); 1515 break; 1516 1517 case SO_DETACH_FILTER: 1518 ret = sk_detach_filter(sk); 1519 break; 1520 1521 case SO_LOCK_FILTER: 1522 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool) 1523 ret = -EPERM; 1524 else 1525 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool); 1526 break; 1527 1528 case SO_MARK: 1529 if (!sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && 1530 !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1531 ret = -EPERM; 1532 break; 1533 } 1534 1535 __sock_set_mark(sk, val); 1536 break; 1537 case SO_RCVMARK: 1538 sock_valbool_flag(sk, SOCK_RCVMARK, valbool); 1539 break; 1540 1541 case SO_RCVPRIORITY: 1542 sock_valbool_flag(sk, SOCK_RCVPRIORITY, valbool); 1543 break; 1544 1545 case SO_RXQ_OVFL: 1546 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool); 1547 break; 1548 1549 case SO_WIFI_STATUS: 1550 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool); 1551 break; 1552 1553 case SO_NOFCS: 1554 sock_valbool_flag(sk, SOCK_NOFCS, valbool); 1555 break; 1556 1557 case SO_SELECT_ERR_QUEUE: 1558 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool); 1559 break; 1560 1561 case SO_PASSCRED: 1562 if (sk_may_scm_recv(sk)) 1563 sk->sk_scm_credentials = valbool; 1564 else 1565 ret = -EOPNOTSUPP; 1566 break; 1567 1568 case SO_PASSSEC: 1569 if (IS_ENABLED(CONFIG_SECURITY_NETWORK) && sk_may_scm_recv(sk)) 1570 sk->sk_scm_security = valbool; 1571 else 1572 ret = -EOPNOTSUPP; 1573 break; 1574 1575 case SO_PASSPIDFD: 1576 if (sk_is_unix(sk)) 1577 sk->sk_scm_pidfd = valbool; 1578 else 1579 ret = -EOPNOTSUPP; 1580 break; 1581 1582 case SO_PASSPIDFD_THREAD: 1583 if (sk_is_unix(sk)) 1584 sk->sk_scm_pidfd_thread = valbool; 1585 else 1586 ret = -EOPNOTSUPP; 1587 break; 1588 1589 case SO_PASSRIGHTS: 1590 if (sk_is_unix(sk)) 1591 sk->sk_scm_rights = valbool; 1592 else 1593 ret = -EOPNOTSUPP; 1594 break; 1595 1596 case SO_INCOMING_CPU: 1597 reuseport_update_incoming_cpu(sk, val); 1598 break; 1599 1600 case SO_CNX_ADVICE: 1601 if (val == 1) 1602 dst_negative_advice(sk); 1603 break; 1604 1605 case SO_ZEROCOPY: 1606 if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) { 1607 if (!(sk_is_tcp(sk) || 1608 (sk->sk_type == SOCK_DGRAM && 1609 sk->sk_protocol == IPPROTO_UDP))) 1610 ret = -EOPNOTSUPP; 1611 } else if (sk->sk_family != PF_RDS) { 1612 ret = -EOPNOTSUPP; 1613 } 1614 if (!ret) { 1615 if (val < 0 || val > 1) 1616 ret = -EINVAL; 1617 else 1618 sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool); 1619 } 1620 break; 1621 1622 case SO_TXTIME: 1623 if (optlen != sizeof(struct sock_txtime)) { 1624 ret = -EINVAL; 1625 break; 1626 } else if (copy_from_sockptr(&sk_txtime, optval, 1627 sizeof(struct sock_txtime))) { 1628 ret = -EFAULT; 1629 break; 1630 } else if (sk_txtime.flags & ~SOF_TXTIME_FLAGS_MASK) { 1631 ret = -EINVAL; 1632 break; 1633 } 1634 /* CLOCK_MONOTONIC is only used by sch_fq, and this packet 1635 * scheduler has enough safe guards. 1636 */ 1637 if (sk_txtime.clockid != CLOCK_MONOTONIC && 1638 !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1639 ret = -EPERM; 1640 break; 1641 } 1642 1643 ret = sockopt_validate_clockid(sk_txtime.clockid); 1644 if (ret) 1645 break; 1646 1647 sock_valbool_flag(sk, SOCK_TXTIME, true); 1648 sk->sk_clockid = sk_txtime.clockid; 1649 sk->sk_txtime_deadline_mode = 1650 !!(sk_txtime.flags & SOF_TXTIME_DEADLINE_MODE); 1651 sk->sk_txtime_report_errors = 1652 !!(sk_txtime.flags & SOF_TXTIME_REPORT_ERRORS); 1653 break; 1654 1655 case SO_BINDTOIFINDEX: 1656 ret = sock_bindtoindex_locked(sk, val); 1657 break; 1658 1659 case SO_BUF_LOCK: 1660 if (val & ~SOCK_BUF_LOCK_MASK) { 1661 ret = -EINVAL; 1662 break; 1663 } 1664 sk->sk_userlocks = val | (sk->sk_userlocks & 1665 ~SOCK_BUF_LOCK_MASK); 1666 break; 1667 1668 case SO_RESERVE_MEM: 1669 { 1670 int delta; 1671 1672 if (val < 0 || val > SZ_1G) { 1673 ret = -EINVAL; 1674 break; 1675 } 1676 1677 delta = val - sk->sk_reserved_mem; 1678 if (delta < 0) 1679 sock_release_reserved_memory(sk, -delta); 1680 else 1681 ret = sock_reserve_memory(sk, delta); 1682 break; 1683 } 1684 1685 default: 1686 ret = -ENOPROTOOPT; 1687 break; 1688 } 1689 sockopt_release_sock(sk); 1690 return ret; 1691 } 1692 1693 int sock_setsockopt(struct socket *sock, int level, int optname, 1694 sockptr_t optval, unsigned int optlen) 1695 { 1696 return sk_setsockopt(sock->sk, level, optname, 1697 optval, optlen); 1698 } 1699 EXPORT_SYMBOL(sock_setsockopt); 1700 1701 static const struct cred *sk_get_peer_cred(struct sock *sk) 1702 { 1703 const struct cred *cred; 1704 1705 spin_lock(&sk->sk_peer_lock); 1706 cred = get_cred(sk->sk_peer_cred); 1707 spin_unlock(&sk->sk_peer_lock); 1708 1709 return cred; 1710 } 1711 1712 static void cred_to_ucred(struct pid *pid, const struct cred *cred, 1713 struct ucred *ucred) 1714 { 1715 ucred->pid = pid_vnr(pid); 1716 ucred->uid = ucred->gid = -1; 1717 if (cred) { 1718 struct user_namespace *current_ns = current_user_ns(); 1719 1720 ucred->uid = from_kuid_munged(current_ns, cred->euid); 1721 ucred->gid = from_kgid_munged(current_ns, cred->egid); 1722 } 1723 } 1724 1725 static int groups_to_user(sockptr_t dst, const struct group_info *src) 1726 { 1727 struct user_namespace *user_ns = current_user_ns(); 1728 int i; 1729 1730 for (i = 0; i < src->ngroups; i++) { 1731 gid_t gid = from_kgid_munged(user_ns, src->gid[i]); 1732 1733 if (copy_to_sockptr_offset(dst, i * sizeof(gid), &gid, sizeof(gid))) 1734 return -EFAULT; 1735 } 1736 1737 return 0; 1738 } 1739 1740 /* Hand out a pidfd for @type of the socket's peer via SO_PEERPIDFD*. */ 1741 static int sk_getsockopt_peerpidfd(struct sock *sk, enum pid_type type, 1742 sockptr_t optval, sockptr_t optlen, int len) 1743 { 1744 struct file *pidfd_file = NULL; 1745 unsigned int flags = 0; 1746 struct pid *peer_pid; 1747 int pidfd; 1748 1749 if (len > sizeof(pidfd)) 1750 len = sizeof(pidfd); 1751 1752 spin_lock(&sk->sk_peer_lock); 1753 peer_pid = get_pid(sk->sk_peer_pid[type]); 1754 spin_unlock(&sk->sk_peer_lock); 1755 1756 if (!peer_pid) 1757 return -ENODATA; 1758 1759 /* The use of PIDFD_STALE requires stashing of struct pid on pidfs 1760 * with pidfs_register_pid() and only AF_UNIX is prepared for this. 1761 */ 1762 if (sk_is_unix(sk)) 1763 flags |= PIDFD_STALE; 1764 if (type == PIDTYPE_PID) 1765 flags |= PIDFD_THREAD; 1766 1767 pidfd = pidfd_prepare(peer_pid, flags, &pidfd_file); 1768 put_pid(peer_pid); 1769 if (pidfd < 0) 1770 return pidfd; 1771 1772 if (copy_to_sockptr(optval, &pidfd, len) || 1773 copy_to_sockptr(optlen, &len, sizeof(int))) { 1774 put_unused_fd(pidfd); 1775 fput(pidfd_file); 1776 1777 return -EFAULT; 1778 } 1779 1780 fd_install(pidfd, pidfd_file); 1781 return 0; 1782 } 1783 1784 int sk_getsockopt(struct sock *sk, int level, int optname, 1785 sockptr_t optval, sockptr_t optlen) 1786 { 1787 struct socket *sock = sk->sk_socket; 1788 1789 union { 1790 int val; 1791 u64 val64; 1792 unsigned long ulval; 1793 struct linger ling; 1794 struct old_timeval32 tm32; 1795 struct __kernel_old_timeval tm; 1796 struct __kernel_sock_timeval stm; 1797 struct sock_txtime txtime; 1798 struct so_timestamping timestamping; 1799 } v; 1800 1801 int lv = sizeof(int); 1802 int len; 1803 1804 if (copy_from_sockptr(&len, optlen, sizeof(int))) 1805 return -EFAULT; 1806 if (len < 0) 1807 return -EINVAL; 1808 1809 memset(&v, 0, sizeof(v)); 1810 1811 switch (optname) { 1812 case SO_DEBUG: 1813 v.val = sock_flag(sk, SOCK_DBG); 1814 break; 1815 1816 case SO_DONTROUTE: 1817 v.val = sock_flag(sk, SOCK_LOCALROUTE); 1818 break; 1819 1820 case SO_BROADCAST: 1821 v.val = sock_flag(sk, SOCK_BROADCAST); 1822 break; 1823 1824 case SO_SNDBUF: 1825 v.val = READ_ONCE(sk->sk_sndbuf); 1826 break; 1827 1828 case SO_RCVBUF: 1829 v.val = READ_ONCE(sk->sk_rcvbuf); 1830 break; 1831 1832 case SO_REUSEADDR: 1833 v.val = sk->sk_reuse; 1834 break; 1835 1836 case SO_REUSEPORT: 1837 v.val = sk->sk_reuseport; 1838 break; 1839 1840 case SO_KEEPALIVE: 1841 v.val = sock_flag(sk, SOCK_KEEPOPEN); 1842 break; 1843 1844 case SO_TYPE: 1845 v.val = sk->sk_type; 1846 break; 1847 1848 case SO_PROTOCOL: 1849 v.val = sk->sk_protocol; 1850 break; 1851 1852 case SO_DOMAIN: 1853 v.val = sk->sk_family; 1854 break; 1855 1856 case SO_ERROR: 1857 v.val = -sock_error(sk); 1858 if (v.val == 0) 1859 v.val = xchg(&sk->sk_err_soft, 0); 1860 break; 1861 1862 case SO_OOBINLINE: 1863 v.val = sock_flag(sk, SOCK_URGINLINE); 1864 break; 1865 1866 case SO_NO_CHECK: 1867 v.val = sk->sk_no_check_tx; 1868 break; 1869 1870 case SO_PRIORITY: 1871 v.val = READ_ONCE(sk->sk_priority); 1872 break; 1873 1874 case SO_LINGER: 1875 lv = sizeof(v.ling); 1876 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER); 1877 v.ling.l_linger = READ_ONCE(sk->sk_lingertime) / HZ; 1878 break; 1879 1880 case SO_BSDCOMPAT: 1881 break; 1882 1883 case SO_TIMESTAMP_OLD: 1884 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && 1885 !sock_flag(sk, SOCK_TSTAMP_NEW) && 1886 !sock_flag(sk, SOCK_RCVTSTAMPNS); 1887 break; 1888 1889 case SO_TIMESTAMPNS_OLD: 1890 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && !sock_flag(sk, SOCK_TSTAMP_NEW); 1891 break; 1892 1893 case SO_TIMESTAMP_NEW: 1894 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && sock_flag(sk, SOCK_TSTAMP_NEW); 1895 break; 1896 1897 case SO_TIMESTAMPNS_NEW: 1898 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && sock_flag(sk, SOCK_TSTAMP_NEW); 1899 break; 1900 1901 case SO_TIMESTAMPING_OLD: 1902 case SO_TIMESTAMPING_NEW: 1903 lv = sizeof(v.timestamping); 1904 /* For the later-added case SO_TIMESTAMPING_NEW: Be strict about only 1905 * returning the flags when they were set through the same option. 1906 * Don't change the beviour for the old case SO_TIMESTAMPING_OLD. 1907 */ 1908 if (optname == SO_TIMESTAMPING_OLD || sock_flag(sk, SOCK_TSTAMP_NEW)) { 1909 v.timestamping.flags = READ_ONCE(sk->sk_tsflags); 1910 v.timestamping.bind_phc = READ_ONCE(sk->sk_bind_phc); 1911 } 1912 break; 1913 1914 case SO_RCVTIMEO_OLD: 1915 case SO_RCVTIMEO_NEW: 1916 lv = sock_get_timeout(READ_ONCE(sk->sk_rcvtimeo), &v, 1917 SO_RCVTIMEO_OLD == optname); 1918 break; 1919 1920 case SO_SNDTIMEO_OLD: 1921 case SO_SNDTIMEO_NEW: 1922 lv = sock_get_timeout(READ_ONCE(sk->sk_sndtimeo), &v, 1923 SO_SNDTIMEO_OLD == optname); 1924 break; 1925 1926 case SO_RCVLOWAT: 1927 v.val = READ_ONCE(sk->sk_rcvlowat); 1928 break; 1929 1930 case SO_SNDLOWAT: 1931 v.val = 1; 1932 break; 1933 1934 case SO_PASSCRED: 1935 if (!sk_may_scm_recv(sk)) 1936 return -EOPNOTSUPP; 1937 1938 v.val = sk->sk_scm_credentials; 1939 break; 1940 1941 case SO_PASSPIDFD: 1942 if (!sk_is_unix(sk)) 1943 return -EOPNOTSUPP; 1944 1945 v.val = sk->sk_scm_pidfd; 1946 break; 1947 1948 case SO_PASSPIDFD_THREAD: 1949 if (!sk_is_unix(sk)) 1950 return -EOPNOTSUPP; 1951 1952 v.val = sk->sk_scm_pidfd_thread; 1953 break; 1954 1955 case SO_PASSRIGHTS: 1956 if (!sk_is_unix(sk)) 1957 return -EOPNOTSUPP; 1958 1959 v.val = sk->sk_scm_rights; 1960 break; 1961 1962 case SO_PEERCRED: 1963 { 1964 struct ucred peercred; 1965 if (len > sizeof(peercred)) 1966 len = sizeof(peercred); 1967 1968 spin_lock(&sk->sk_peer_lock); 1969 cred_to_ucred(sk->sk_peer_pid[PIDTYPE_TGID], sk->sk_peer_cred, 1970 &peercred); 1971 spin_unlock(&sk->sk_peer_lock); 1972 1973 if (copy_to_sockptr(optval, &peercred, len)) 1974 return -EFAULT; 1975 goto lenout; 1976 } 1977 1978 case SO_PEERPIDFD: 1979 return sk_getsockopt_peerpidfd(sk, PIDTYPE_TGID, optval, optlen, len); 1980 1981 case SO_PEERPIDFD_THREAD: 1982 /* Only AF_UNIX records the peer's connecting thread. */ 1983 if (!sk_is_unix(sk)) 1984 return -EOPNOTSUPP; 1985 1986 return sk_getsockopt_peerpidfd(sk, PIDTYPE_PID, optval, optlen, len); 1987 1988 case SO_PEERGROUPS: 1989 { 1990 const struct cred *cred; 1991 int ret, n; 1992 1993 cred = sk_get_peer_cred(sk); 1994 if (!cred) 1995 return -ENODATA; 1996 1997 n = cred->group_info->ngroups; 1998 if (len < n * sizeof(gid_t)) { 1999 len = n * sizeof(gid_t); 2000 put_cred(cred); 2001 return copy_to_sockptr(optlen, &len, sizeof(int)) ? -EFAULT : -ERANGE; 2002 } 2003 len = n * sizeof(gid_t); 2004 2005 ret = groups_to_user(optval, cred->group_info); 2006 put_cred(cred); 2007 if (ret) 2008 return ret; 2009 goto lenout; 2010 } 2011 2012 case SO_PEERNAME: 2013 { 2014 struct sockaddr_storage address; 2015 2016 lv = READ_ONCE(sock->ops)->getname(sock, (struct sockaddr *)&address, 2); 2017 if (lv < 0) 2018 return -ENOTCONN; 2019 if (lv < len) 2020 return -EINVAL; 2021 if (copy_to_sockptr(optval, &address, len)) 2022 return -EFAULT; 2023 goto lenout; 2024 } 2025 2026 /* Dubious BSD thing... Probably nobody even uses it, but 2027 * the UNIX standard wants it for whatever reason... -DaveM 2028 */ 2029 case SO_ACCEPTCONN: 2030 v.val = sk->sk_state == TCP_LISTEN; 2031 break; 2032 2033 case SO_PASSSEC: 2034 if (!IS_ENABLED(CONFIG_SECURITY_NETWORK) || !sk_may_scm_recv(sk)) 2035 return -EOPNOTSUPP; 2036 2037 v.val = sk->sk_scm_security; 2038 break; 2039 2040 case SO_PEERSEC: 2041 return security_socket_getpeersec_stream(sock, 2042 optval, optlen, len); 2043 2044 case SO_MARK: 2045 v.val = READ_ONCE(sk->sk_mark); 2046 break; 2047 2048 case SO_RCVMARK: 2049 v.val = sock_flag(sk, SOCK_RCVMARK); 2050 break; 2051 2052 case SO_RCVPRIORITY: 2053 v.val = sock_flag(sk, SOCK_RCVPRIORITY); 2054 break; 2055 2056 case SO_RXQ_OVFL: 2057 v.val = sock_flag(sk, SOCK_RXQ_OVFL); 2058 break; 2059 2060 case SO_WIFI_STATUS: 2061 v.val = sock_flag(sk, SOCK_WIFI_STATUS); 2062 break; 2063 2064 case SO_PEEK_OFF: 2065 if (!READ_ONCE(sock->ops)->set_peek_off) 2066 return -EOPNOTSUPP; 2067 2068 v.val = READ_ONCE(sk->sk_peek_off); 2069 break; 2070 case SO_NOFCS: 2071 v.val = sock_flag(sk, SOCK_NOFCS); 2072 break; 2073 2074 case SO_BINDTODEVICE: 2075 return sock_getbindtodevice(sk, optval, optlen, len); 2076 2077 case SO_GET_FILTER: 2078 len = sk_get_filter(sk, optval, len); 2079 if (len < 0) 2080 return len; 2081 2082 goto lenout; 2083 2084 case SO_LOCK_FILTER: 2085 v.val = sock_flag(sk, SOCK_FILTER_LOCKED); 2086 break; 2087 2088 case SO_BPF_EXTENSIONS: 2089 v.val = bpf_tell_extensions(); 2090 break; 2091 2092 case SO_SELECT_ERR_QUEUE: 2093 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE); 2094 break; 2095 2096 #ifdef CONFIG_NET_RX_BUSY_POLL 2097 case SO_BUSY_POLL: 2098 v.val = READ_ONCE(sk->sk_ll_usec); 2099 break; 2100 case SO_PREFER_BUSY_POLL: 2101 v.val = READ_ONCE(sk->sk_prefer_busy_poll); 2102 break; 2103 #endif 2104 2105 case SO_MAX_PACING_RATE: 2106 /* The READ_ONCE() pair with the WRITE_ONCE() in sk_setsockopt() */ 2107 if (sizeof(v.ulval) != sizeof(v.val) && len >= sizeof(v.ulval)) { 2108 lv = sizeof(v.ulval); 2109 v.ulval = READ_ONCE(sk->sk_max_pacing_rate); 2110 } else { 2111 /* 32bit version */ 2112 v.val = min_t(unsigned long, ~0U, 2113 READ_ONCE(sk->sk_max_pacing_rate)); 2114 } 2115 break; 2116 2117 case SO_INCOMING_CPU: 2118 v.val = READ_ONCE(sk->sk_incoming_cpu); 2119 break; 2120 2121 case SO_MEMINFO: 2122 { 2123 u32 meminfo[SK_MEMINFO_VARS]; 2124 2125 sk_get_meminfo(sk, meminfo); 2126 2127 len = min_t(unsigned int, len, sizeof(meminfo)); 2128 if (copy_to_sockptr(optval, &meminfo, len)) 2129 return -EFAULT; 2130 2131 goto lenout; 2132 } 2133 2134 #ifdef CONFIG_NET_RX_BUSY_POLL 2135 case SO_INCOMING_NAPI_ID: 2136 v.val = READ_ONCE(sk->sk_napi_id); 2137 2138 /* aggregate non-NAPI IDs down to 0 */ 2139 if (!napi_id_valid(v.val)) 2140 v.val = 0; 2141 2142 break; 2143 #endif 2144 2145 case SO_COOKIE: 2146 lv = sizeof(u64); 2147 if (len < lv) 2148 return -EINVAL; 2149 v.val64 = sock_gen_cookie(sk); 2150 break; 2151 2152 case SO_ZEROCOPY: 2153 v.val = sock_flag(sk, SOCK_ZEROCOPY); 2154 break; 2155 2156 case SO_TXTIME: 2157 lv = sizeof(v.txtime); 2158 v.txtime.clockid = sk->sk_clockid; 2159 v.txtime.flags |= sk->sk_txtime_deadline_mode ? 2160 SOF_TXTIME_DEADLINE_MODE : 0; 2161 v.txtime.flags |= sk->sk_txtime_report_errors ? 2162 SOF_TXTIME_REPORT_ERRORS : 0; 2163 break; 2164 2165 case SO_BINDTOIFINDEX: 2166 v.val = READ_ONCE(sk->sk_bound_dev_if); 2167 break; 2168 2169 case SO_NETNS_COOKIE: 2170 lv = sizeof(u64); 2171 if (len != lv) 2172 return -EINVAL; 2173 v.val64 = sock_net(sk)->net_cookie; 2174 break; 2175 2176 case SO_BUF_LOCK: 2177 v.val = sk->sk_userlocks & SOCK_BUF_LOCK_MASK; 2178 break; 2179 2180 case SO_RESERVE_MEM: 2181 v.val = READ_ONCE(sk->sk_reserved_mem); 2182 break; 2183 2184 case SO_TXREHASH: 2185 if (!sk_is_tcp(sk)) 2186 return -EOPNOTSUPP; 2187 2188 /* Paired with WRITE_ONCE() in sk_setsockopt() */ 2189 v.val = READ_ONCE(sk->sk_txrehash); 2190 break; 2191 2192 default: 2193 /* We implement the SO_SNDLOWAT etc to not be settable 2194 * (1003.1g 7). 2195 */ 2196 return -ENOPROTOOPT; 2197 } 2198 2199 if (len > lv) 2200 len = lv; 2201 if (copy_to_sockptr(optval, &v, len)) 2202 return -EFAULT; 2203 lenout: 2204 if (copy_to_sockptr(optlen, &len, sizeof(int))) 2205 return -EFAULT; 2206 return 0; 2207 } 2208 2209 /* 2210 * Initialize an sk_lock. 2211 * 2212 * (We also register the sk_lock with the lock validator.) 2213 */ 2214 static inline void sock_lock_init(struct sock *sk) 2215 { 2216 sk_owner_clear(sk); 2217 2218 if (sk->sk_kern_sock) 2219 sock_lock_init_class_and_name( 2220 sk, 2221 af_family_kern_slock_key_strings[sk->sk_family], 2222 af_family_kern_slock_keys + sk->sk_family, 2223 af_family_kern_key_strings[sk->sk_family], 2224 af_family_kern_keys + sk->sk_family); 2225 else 2226 sock_lock_init_class_and_name( 2227 sk, 2228 af_family_slock_key_strings[sk->sk_family], 2229 af_family_slock_keys + sk->sk_family, 2230 af_family_key_strings[sk->sk_family], 2231 af_family_keys + sk->sk_family); 2232 } 2233 2234 /* 2235 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet, 2236 * even temporarily, because of RCU lookups. sk_node should also be left as is. 2237 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end 2238 */ 2239 static void sock_copy(struct sock *nsk, const struct sock *osk) 2240 { 2241 const struct proto *prot = READ_ONCE(osk->sk_prot); 2242 #ifdef CONFIG_SECURITY_NETWORK 2243 void *sptr = nsk->sk_security; 2244 #endif 2245 2246 /* If we move sk_tx_queue_mapping out of the private section, 2247 * we must check if sk_tx_queue_clear() is called after 2248 * sock_copy() in sk_clone_lock(). 2249 */ 2250 BUILD_BUG_ON(offsetof(struct sock, sk_tx_queue_mapping) < 2251 offsetof(struct sock, sk_dontcopy_begin) || 2252 offsetof(struct sock, sk_tx_queue_mapping) >= 2253 offsetof(struct sock, sk_dontcopy_end)); 2254 2255 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin)); 2256 2257 unsafe_memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end, 2258 prot->obj_size - offsetof(struct sock, sk_dontcopy_end), 2259 /* alloc is larger than struct, see sk_prot_alloc() */); 2260 2261 #ifdef CONFIG_SECURITY_NETWORK 2262 nsk->sk_security = sptr; 2263 security_sk_clone(osk, nsk); 2264 #endif 2265 } 2266 2267 static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority, 2268 int family) 2269 { 2270 struct sock *sk; 2271 struct kmem_cache *slab; 2272 2273 slab = prot->slab; 2274 if (slab != NULL) { 2275 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO); 2276 if (!sk) 2277 return sk; 2278 if (want_init_on_alloc(priority)) 2279 sk_prot_clear_nulls(sk, prot->obj_size); 2280 } else 2281 sk = kmalloc(prot->obj_size, priority); 2282 2283 if (sk != NULL) { 2284 if (security_sk_alloc(sk, family, priority)) 2285 goto out_free; 2286 2287 if (!try_module_get(prot->owner)) 2288 goto out_free_sec; 2289 } 2290 2291 return sk; 2292 2293 out_free_sec: 2294 security_sk_free(sk); 2295 out_free: 2296 if (slab != NULL) 2297 kmem_cache_free(slab, sk); 2298 else 2299 kfree(sk); 2300 return NULL; 2301 } 2302 2303 static void sk_prot_free(struct proto *prot, struct sock *sk) 2304 { 2305 struct kmem_cache *slab; 2306 struct module *owner; 2307 2308 owner = prot->owner; 2309 slab = prot->slab; 2310 2311 cgroup_sk_free(&sk->sk_cgrp_data); 2312 mem_cgroup_sk_free(sk); 2313 security_sk_free(sk); 2314 2315 sk_owner_put(sk); 2316 2317 if (slab != NULL) 2318 kmem_cache_free(slab, sk); 2319 else 2320 kfree(sk); 2321 module_put(owner); 2322 } 2323 2324 /** 2325 * sk_alloc - All socket objects are allocated here 2326 * @net: the applicable net namespace 2327 * @family: protocol family 2328 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 2329 * @prot: struct proto associated with this new sock instance 2330 * @kern: is this to be a kernel socket? 2331 */ 2332 struct sock *sk_alloc(struct net *net, int family, gfp_t priority, 2333 struct proto *prot, int kern) 2334 { 2335 struct sock *sk; 2336 2337 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family); 2338 if (sk) { 2339 sk->sk_family = family; 2340 /* 2341 * See comment in struct sock definition to understand 2342 * why we need sk_prot_creator -acme 2343 */ 2344 sk->sk_prot = sk->sk_prot_creator = prot; 2345 2346 if (READ_ONCE(net->core.sysctl_bypass_prot_mem)) 2347 sk->sk_bypass_prot_mem = 1; 2348 2349 sk->sk_kern_sock = kern; 2350 sock_lock_init(sk); 2351 2352 sk->sk_net_refcnt = kern ? 0 : 1; 2353 if (likely(sk->sk_net_refcnt)) { 2354 get_net_track(net, &sk->ns_tracker, priority); 2355 sock_inuse_add(net, 1); 2356 } else { 2357 net_passive_inc(net); 2358 __netns_tracker_alloc(net, &sk->ns_tracker, 2359 false, priority); 2360 } 2361 2362 sock_net_set(sk, net); 2363 refcount_set(&sk->sk_wmem_alloc, SK_WMEM_ALLOC_BIAS); 2364 2365 mem_cgroup_sk_alloc(sk); 2366 cgroup_sk_alloc(&sk->sk_cgrp_data); 2367 sock_update_classid(&sk->sk_cgrp_data); 2368 sock_update_netprioidx(&sk->sk_cgrp_data); 2369 sk_tx_queue_clear(sk); 2370 } 2371 2372 return sk; 2373 } 2374 EXPORT_SYMBOL(sk_alloc); 2375 2376 /* Sockets having SOCK_RCU_FREE will call this function after one RCU 2377 * grace period. This is the case for UDP sockets and TCP listeners. 2378 */ 2379 static void __sk_destruct(struct rcu_head *head) 2380 { 2381 struct sock *sk = container_of(head, struct sock, sk_rcu); 2382 struct net *net = sock_net(sk); 2383 struct sk_filter *filter; 2384 2385 if (sk->sk_destruct) 2386 sk->sk_destruct(sk); 2387 2388 filter = rcu_dereference_check(sk->sk_filter, 2389 refcount_read(&sk->sk_wmem_alloc) == 0); 2390 if (filter) { 2391 sk_filter_uncharge(sk, filter); 2392 RCU_INIT_POINTER(sk->sk_filter, NULL); 2393 } 2394 2395 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP); 2396 2397 #ifdef CONFIG_BPF_SYSCALL 2398 bpf_sk_storage_free(sk); 2399 #endif 2400 2401 if (atomic_read(&sk->sk_omem_alloc)) 2402 pr_debug("%s: optmem leakage (%d bytes) detected\n", 2403 __func__, atomic_read(&sk->sk_omem_alloc)); 2404 2405 if (sk->sk_frag.page) { 2406 put_page(sk->sk_frag.page); 2407 sk->sk_frag.page = NULL; 2408 } 2409 2410 /* We do not need to acquire sk->sk_peer_lock, we are the last user. */ 2411 put_cred(sk->sk_peer_cred); 2412 put_pids(sk->sk_peer_pid); 2413 2414 if (likely(sk->sk_net_refcnt)) { 2415 put_net_track(net, &sk->ns_tracker); 2416 } else { 2417 __netns_tracker_free(net, &sk->ns_tracker, false); 2418 net_passive_dec(net); 2419 } 2420 sk_prot_free(sk->sk_prot_creator, sk); 2421 } 2422 2423 void sk_net_refcnt_upgrade(struct sock *sk) 2424 { 2425 struct net *net = sock_net(sk); 2426 2427 WARN_ON_ONCE(sk->sk_net_refcnt); 2428 __netns_tracker_free(net, &sk->ns_tracker, false); 2429 net_passive_dec(net); 2430 sk->sk_net_refcnt = 1; 2431 get_net_track(net, &sk->ns_tracker, GFP_KERNEL); 2432 sock_inuse_add(net, 1); 2433 } 2434 EXPORT_SYMBOL_GPL(sk_net_refcnt_upgrade); 2435 2436 void sk_destruct(struct sock *sk) 2437 { 2438 bool use_call_rcu = sock_flag(sk, SOCK_RCU_FREE); 2439 2440 if (rcu_access_pointer(sk->sk_reuseport_cb)) { 2441 reuseport_detach_sock(sk); 2442 use_call_rcu = true; 2443 } 2444 2445 if (use_call_rcu) 2446 call_rcu(&sk->sk_rcu, __sk_destruct); 2447 else 2448 __sk_destruct(&sk->sk_rcu); 2449 } 2450 2451 static void __sk_free(struct sock *sk) 2452 { 2453 if (likely(sk->sk_net_refcnt)) 2454 sock_inuse_add(sock_net(sk), -1); 2455 2456 if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk))) 2457 sock_diag_broadcast_destroy(sk); 2458 else 2459 sk_destruct(sk); 2460 } 2461 2462 void sk_free(struct sock *sk) 2463 { 2464 /* 2465 * We subtract one from sk_wmem_alloc and can know if 2466 * some packets are still in some tx queue. 2467 * If not null, sock_wfree() will call __sk_free(sk) later 2468 */ 2469 if (refcount_dec_and_test(&sk->sk_wmem_alloc)) 2470 __sk_free(sk); 2471 } 2472 EXPORT_SYMBOL(sk_free); 2473 2474 static void sk_init_common(struct sock *sk) 2475 { 2476 skb_queue_head_init(&sk->sk_receive_queue); 2477 skb_queue_head_init(&sk->sk_write_queue); 2478 skb_queue_head_init(&sk->sk_error_queue); 2479 2480 rwlock_init(&sk->sk_callback_lock); 2481 lockdep_set_class_and_name(&sk->sk_receive_queue.lock, 2482 af_rlock_keys + sk->sk_family, 2483 af_family_rlock_key_strings[sk->sk_family]); 2484 lockdep_set_class_and_name(&sk->sk_write_queue.lock, 2485 af_wlock_keys + sk->sk_family, 2486 af_family_wlock_key_strings[sk->sk_family]); 2487 lockdep_set_class_and_name(&sk->sk_error_queue.lock, 2488 af_elock_keys + sk->sk_family, 2489 af_family_elock_key_strings[sk->sk_family]); 2490 if (sk->sk_kern_sock) 2491 lockdep_set_class_and_name(&sk->sk_callback_lock, 2492 af_kern_callback_keys + sk->sk_family, 2493 af_family_kern_clock_key_strings[sk->sk_family]); 2494 else 2495 lockdep_set_class_and_name(&sk->sk_callback_lock, 2496 af_callback_keys + sk->sk_family, 2497 af_family_clock_key_strings[sk->sk_family]); 2498 } 2499 2500 /** 2501 * sk_clone - clone a socket 2502 * @sk: the socket to clone 2503 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 2504 * @lock: if true, lock the cloned sk 2505 * 2506 * If @lock is true, the clone is locked by bh_lock_sock(), and 2507 * caller must unlock socket even in error path by bh_unlock_sock(). 2508 */ 2509 struct sock *sk_clone(const struct sock *sk, const gfp_t priority, 2510 bool lock) 2511 { 2512 struct proto *prot = READ_ONCE(sk->sk_prot); 2513 struct sk_filter *filter; 2514 bool is_charged = true; 2515 struct sock *newsk; 2516 2517 newsk = sk_prot_alloc(prot, priority, sk->sk_family); 2518 if (!newsk) 2519 goto out; 2520 2521 sock_copy(newsk, sk); 2522 2523 newsk->sk_prot_creator = prot; 2524 #ifdef CONFIG_BPF_SYSCALL 2525 RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL); 2526 #endif 2527 #if IS_ENABLED(CONFIG_INET_PSP) 2528 DEBUG_NET_WARN_ON_ONCE(rcu_access_pointer(sk->psp_assoc)); 2529 #endif 2530 2531 /* SANITY */ 2532 if (likely(newsk->sk_net_refcnt)) { 2533 get_net_track(sock_net(newsk), &newsk->ns_tracker, priority); 2534 sock_inuse_add(sock_net(newsk), 1); 2535 } else { 2536 /* Kernel sockets are not elevating the struct net refcount. 2537 * Instead, use a tracker to more easily detect if a layer 2538 * is not properly dismantling its kernel sockets at netns 2539 * destroy time. 2540 */ 2541 net_passive_inc(sock_net(newsk)); 2542 __netns_tracker_alloc(sock_net(newsk), &newsk->ns_tracker, 2543 false, priority); 2544 } 2545 2546 sk_node_init(&newsk->sk_node); 2547 sock_lock_init(newsk); 2548 2549 if (lock) 2550 bh_lock_sock(newsk); 2551 2552 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL; 2553 newsk->sk_backlog.len = 0; 2554 2555 atomic_set(&newsk->sk_rmem_alloc, 0); 2556 2557 refcount_set(&newsk->sk_wmem_alloc, SK_WMEM_ALLOC_BIAS); 2558 2559 atomic_set(&newsk->sk_omem_alloc, 0); 2560 sk_init_common(newsk); 2561 2562 newsk->sk_dst_cache = NULL; 2563 newsk->sk_dst_pending_confirm = 0; 2564 newsk->sk_wmem_queued = 0; 2565 newsk->sk_forward_alloc = 0; 2566 newsk->sk_reserved_mem = 0; 2567 DEBUG_NET_WARN_ON_ONCE(newsk->sk_drop_counters); 2568 sk_drops_reset(newsk); 2569 newsk->sk_send_head = NULL; 2570 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK; 2571 atomic_set(&newsk->sk_zckey, 0); 2572 2573 sock_reset_flag(newsk, SOCK_DONE); 2574 2575 #ifdef CONFIG_MEMCG 2576 /* sk->sk_memcg will be populated at accept() time */ 2577 newsk->sk_memcg = NULL; 2578 #endif 2579 2580 cgroup_sk_clone(&newsk->sk_cgrp_data); 2581 2582 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL); 2583 2584 if (sock_needs_netstamp(sk) && newsk->sk_flags & SK_FLAGS_TIMESTAMP) 2585 net_enable_timestamp(); 2586 2587 rcu_read_lock(); 2588 filter = rcu_dereference(sk->sk_filter); 2589 if (filter != NULL) 2590 /* though it's an empty new sock, the charging may fail 2591 * if sysctl_optmem_max was changed between creation of 2592 * original socket and cloning 2593 */ 2594 is_charged = sk_filter_charge(newsk, filter); 2595 RCU_INIT_POINTER(newsk->sk_filter, filter); 2596 rcu_read_unlock(); 2597 2598 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) { 2599 /* We need to make sure that we don't uncharge the new 2600 * socket if we couldn't charge it in the first place 2601 * as otherwise we uncharge the parent's filter. 2602 */ 2603 if (!is_charged) 2604 RCU_INIT_POINTER(newsk->sk_filter, NULL); 2605 2606 goto free; 2607 } 2608 2609 if (bpf_sk_storage_clone(sk, newsk)) 2610 goto free; 2611 2612 /* Clear sk_user_data if parent had the pointer tagged 2613 * as not suitable for copying when cloning. 2614 */ 2615 if (sk_user_data_is_nocopy(newsk)) 2616 newsk->sk_user_data = NULL; 2617 2618 newsk->sk_err = 0; 2619 newsk->sk_err_soft = 0; 2620 newsk->sk_priority = 0; 2621 newsk->sk_incoming_cpu = raw_smp_processor_id(); 2622 2623 /* Before updating sk_refcnt, we must commit prior changes to memory 2624 * (Documentation/RCU/rculist_nulls.rst for details) 2625 */ 2626 smp_wmb(); 2627 refcount_set(&newsk->sk_refcnt, 2); 2628 2629 sk_set_socket(newsk, NULL); 2630 sk_tx_queue_clear(newsk); 2631 sk_rx_queue_clear(newsk); 2632 RCU_INIT_POINTER(newsk->sk_wq, NULL); 2633 2634 if (newsk->sk_prot->sockets_allocated) 2635 sk_sockets_allocated_inc(newsk); 2636 out: 2637 return newsk; 2638 free: 2639 /* It is still raw copy of parent, so invalidate 2640 * destructor and make plain sk_free() 2641 */ 2642 newsk->sk_destruct = NULL; 2643 if (lock) 2644 bh_unlock_sock(newsk); 2645 sk_free(newsk); 2646 newsk = NULL; 2647 goto out; 2648 } 2649 EXPORT_SYMBOL_GPL(sk_clone); 2650 2651 static u32 sk_dst_gso_max_size(struct sock *sk, const struct net_device *dev) 2652 { 2653 bool is_ipv6 = false; 2654 u32 max_size; 2655 2656 #if IS_ENABLED(CONFIG_IPV6) 2657 is_ipv6 = (sk->sk_family == AF_INET6 && 2658 !ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)); 2659 #endif 2660 /* pairs with the WRITE_ONCE() in netif_set_gso(_ipv4)_max_size() */ 2661 max_size = is_ipv6 ? READ_ONCE(dev->gso_max_size) : 2662 READ_ONCE(dev->gso_ipv4_max_size); 2663 if (max_size > GSO_LEGACY_MAX_SIZE && !sk_is_tcp(sk)) 2664 max_size = GSO_LEGACY_MAX_SIZE; 2665 2666 return max_size - (MAX_TCP_HEADER + 1); 2667 } 2668 2669 void sk_setup_caps(struct sock *sk, struct dst_entry *dst) 2670 { 2671 const struct net_device *dev; 2672 u32 max_segs = 1; 2673 2674 rcu_read_lock(); 2675 dev = dst_dev_rcu(dst); 2676 sk->sk_route_caps = dev->features; 2677 if (sk_is_tcp(sk)) { 2678 struct inet_connection_sock *icsk = inet_csk(sk); 2679 2680 sk->sk_route_caps |= NETIF_F_GSO; 2681 icsk->icsk_ack.dst_quick_ack = dst_metric(dst, RTAX_QUICKACK); 2682 } 2683 if (sk->sk_route_caps & NETIF_F_GSO) 2684 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE; 2685 if (unlikely(sk->sk_gso_disabled)) 2686 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 2687 if (sk_can_gso(sk)) { 2688 if (dst->header_len && !xfrm_dst_offload_ok(dst)) { 2689 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 2690 } else { 2691 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM; 2692 sk->sk_gso_max_size = sk_dst_gso_max_size(sk, dev); 2693 /* pairs with the WRITE_ONCE() in netif_set_gso_max_segs() */ 2694 max_segs = max_t(u32, READ_ONCE(dev->gso_max_segs), 1); 2695 } 2696 } 2697 sk->sk_gso_max_segs = max_segs; 2698 sk_dst_set(sk, dst); 2699 rcu_read_unlock(); 2700 } 2701 EXPORT_SYMBOL_GPL(sk_setup_caps); 2702 2703 bool sk_has_decrypt_user(const struct sock *sk) 2704 { 2705 return psp_sk_assoc(sk) || 2706 (sk_is_inet(sk) && inet_csk_has_ulp(sk)); /* for tls */ 2707 } 2708 2709 /* 2710 * Simple resource managers for sockets. 2711 */ 2712 2713 2714 /* 2715 * Write buffer destructor automatically called from kfree_skb. 2716 */ 2717 void sock_wfree(struct sk_buff *skb) 2718 { 2719 unsigned int len = skb->truesize; 2720 struct sock *sk = skb->sk; 2721 bool free; 2722 int old; 2723 2724 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) { 2725 void (*sk_write_space)(struct sock *sk); 2726 2727 sk_write_space = READ_ONCE(sk->sk_write_space); 2728 2729 if (sock_flag(sk, SOCK_RCU_FREE) && 2730 sk_write_space == sock_def_write_space) { 2731 rcu_read_lock(); 2732 free = __refcount_sub_and_test(len, &sk->sk_wmem_alloc, 2733 &old); 2734 sock_def_write_space_wfree(sk, old - len); 2735 rcu_read_unlock(); 2736 if (unlikely(free)) 2737 __sk_free(sk); 2738 return; 2739 } 2740 2741 /* 2742 * Keep a reference on sk_wmem_alloc, this will be released 2743 * after sk_write_space() call 2744 */ 2745 WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc)); 2746 sk_write_space(sk); 2747 len = 1; 2748 } 2749 /* 2750 * if sk_wmem_alloc reaches 0, we must finish what sk_free() 2751 * could not do because of in-flight packets 2752 */ 2753 if (refcount_sub_and_test(len, &sk->sk_wmem_alloc)) 2754 __sk_free(sk); 2755 } 2756 EXPORT_SYMBOL(sock_wfree); 2757 2758 /* This variant of sock_wfree() is used by TCP, 2759 * since it sets SOCK_USE_WRITE_QUEUE. 2760 */ 2761 #ifdef CONFIG_INET 2762 void __sock_wfree(struct sk_buff *skb) 2763 { 2764 struct sock *sk = skb->sk; 2765 2766 if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)) 2767 __sk_free(sk); 2768 } 2769 EXPORT_SYMBOL_GPL(__sock_wfree); 2770 #endif 2771 2772 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk) 2773 { 2774 int old_wmem; 2775 2776 skb_orphan(skb); 2777 #ifdef CONFIG_INET 2778 if (unlikely(!sk_fullsock(sk))) 2779 return skb_set_owner_edemux(skb, sk); 2780 #endif 2781 skb->sk = sk; 2782 skb->destructor = sock_wfree; 2783 skb_set_hash_from_sk(skb, sk); 2784 /* 2785 * We used to take a refcount on sk, but following operation 2786 * is enough to guarantee sk_free() won't free this sock until 2787 * all in-flight packets are completed 2788 */ 2789 __refcount_add(skb->truesize, &sk->sk_wmem_alloc, &old_wmem); 2790 2791 /* (old_wmem == SK_WMEM_ALLOC_BIAS) if no other TX packet for this socket 2792 * is in a host queue (qdisc, NIC queue). 2793 * Set skb->ooo_okay so that netdev_pick_tx() can choose a TX queue 2794 * based on XPS for better performance. 2795 * Otherwise clear ooo_okay to not risk Out Of Order delivery. 2796 */ 2797 skb->ooo_okay = (old_wmem == SK_WMEM_ALLOC_BIAS); 2798 } 2799 EXPORT_SYMBOL(skb_set_owner_w); 2800 2801 static bool can_skb_orphan_partial(const struct sk_buff *skb) 2802 { 2803 /* Drivers depend on in-order delivery for crypto offload, 2804 * partial orphan breaks out-of-order-OK logic. 2805 */ 2806 if (skb_is_decrypted(skb)) 2807 return false; 2808 2809 return (skb->destructor == sock_wfree || 2810 (IS_ENABLED(CONFIG_INET) && skb->destructor == tcp_wfree)); 2811 } 2812 2813 /* This helper is used by netem, as it can hold packets in its 2814 * delay queue. We want to allow the owner socket to send more 2815 * packets, as if they were already TX completed by a typical driver. 2816 * But we also want to keep skb->sk set because some packet schedulers 2817 * rely on it (sch_fq for example). 2818 */ 2819 void skb_orphan_partial(struct sk_buff *skb) 2820 { 2821 if (skb_is_tcp_pure_ack(skb)) 2822 return; 2823 2824 if (can_skb_orphan_partial(skb) && skb_set_owner_sk_safe(skb, skb->sk)) 2825 return; 2826 2827 skb_orphan(skb); 2828 } 2829 EXPORT_SYMBOL(skb_orphan_partial); 2830 2831 /* 2832 * Read buffer destructor automatically called from kfree_skb. 2833 */ 2834 void sock_rfree(struct sk_buff *skb) 2835 { 2836 struct sock *sk = skb->sk; 2837 unsigned int len = skb->truesize; 2838 2839 atomic_sub(len, &sk->sk_rmem_alloc); 2840 sk_mem_uncharge(sk, len); 2841 } 2842 EXPORT_SYMBOL(sock_rfree); 2843 2844 /* 2845 * Buffer destructor for skbs that are not used directly in read or write 2846 * path, e.g. for error handler skbs. Automatically called from kfree_skb. 2847 */ 2848 void sock_efree(struct sk_buff *skb) 2849 { 2850 sock_put(skb->sk); 2851 } 2852 EXPORT_SYMBOL(sock_efree); 2853 2854 /* Buffer destructor for prefetch/receive path where reference count may 2855 * not be held, e.g. for listen sockets. 2856 */ 2857 #ifdef CONFIG_INET 2858 void sock_pfree(struct sk_buff *skb) 2859 { 2860 struct sock *sk = skb->sk; 2861 2862 if (!sk_is_refcounted(sk)) 2863 return; 2864 2865 if (sk->sk_state == TCP_NEW_SYN_RECV && inet_reqsk(sk)->syncookie) { 2866 inet_reqsk(sk)->rsk_listener = NULL; 2867 reqsk_free(inet_reqsk(sk)); 2868 return; 2869 } 2870 2871 sock_gen_put(sk); 2872 } 2873 EXPORT_SYMBOL(sock_pfree); 2874 #endif /* CONFIG_INET */ 2875 2876 /* 2877 * Allocate a skb from the socket's send buffer. 2878 */ 2879 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force, 2880 gfp_t priority) 2881 { 2882 if (force || 2883 refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) { 2884 struct sk_buff *skb = alloc_skb(size, priority); 2885 2886 if (skb) { 2887 skb_set_owner_w(skb, sk); 2888 return skb; 2889 } 2890 } 2891 return NULL; 2892 } 2893 EXPORT_SYMBOL(sock_wmalloc); 2894 2895 static void sock_ofree(struct sk_buff *skb) 2896 { 2897 struct sock *sk = skb->sk; 2898 2899 atomic_sub(skb->truesize, &sk->sk_omem_alloc); 2900 } 2901 2902 struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size, 2903 gfp_t priority) 2904 { 2905 struct sk_buff *skb; 2906 2907 /* small safe race: SKB_TRUESIZE may differ from final skb->truesize */ 2908 if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) > 2909 READ_ONCE(sock_net(sk)->core.sysctl_optmem_max)) 2910 return NULL; 2911 2912 skb = alloc_skb(size, priority); 2913 if (!skb) 2914 return NULL; 2915 2916 atomic_add(skb->truesize, &sk->sk_omem_alloc); 2917 skb->sk = sk; 2918 skb->destructor = sock_ofree; 2919 return skb; 2920 } 2921 2922 /* 2923 * Allocate a memory block from the socket's option memory buffer. 2924 */ 2925 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority) 2926 { 2927 int optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max); 2928 2929 if ((unsigned int)size <= optmem_max && 2930 atomic_read(&sk->sk_omem_alloc) + size < optmem_max) { 2931 void *mem; 2932 /* First do the add, to avoid the race if kmalloc 2933 * might sleep. 2934 */ 2935 atomic_add(size, &sk->sk_omem_alloc); 2936 mem = kmalloc(size, priority); 2937 if (mem) 2938 return mem; 2939 atomic_sub(size, &sk->sk_omem_alloc); 2940 } 2941 return NULL; 2942 } 2943 EXPORT_SYMBOL(sock_kmalloc); 2944 2945 /* 2946 * Duplicate the input "src" memory block using the socket's 2947 * option memory buffer. 2948 */ 2949 void *sock_kmemdup(struct sock *sk, const void *src, 2950 int size, gfp_t priority) 2951 { 2952 void *mem; 2953 2954 mem = sock_kmalloc(sk, size, priority); 2955 if (mem) 2956 memcpy(mem, src, size); 2957 return mem; 2958 } 2959 EXPORT_SYMBOL(sock_kmemdup); 2960 2961 /* Free an option memory block. Note, we actually want the inline 2962 * here as this allows gcc to detect the nullify and fold away the 2963 * condition entirely. 2964 */ 2965 static inline void __sock_kfree_s(struct sock *sk, void *mem, int size, 2966 const bool nullify) 2967 { 2968 if (WARN_ON_ONCE(!mem)) 2969 return; 2970 if (nullify) 2971 kfree_sensitive(mem); 2972 else 2973 kfree(mem); 2974 atomic_sub(size, &sk->sk_omem_alloc); 2975 } 2976 2977 void sock_kfree_s(struct sock *sk, void *mem, int size) 2978 { 2979 __sock_kfree_s(sk, mem, size, false); 2980 } 2981 EXPORT_SYMBOL(sock_kfree_s); 2982 2983 void sock_kzfree_s(struct sock *sk, void *mem, int size) 2984 { 2985 __sock_kfree_s(sk, mem, size, true); 2986 } 2987 EXPORT_SYMBOL(sock_kzfree_s); 2988 2989 /** 2990 * sk_set_nospace - tell the transport a writer is waiting for space 2991 * @sk: socket 2992 * 2993 * Must be called before the final check of the available send space, 2994 * so that the transport can not miss the request and forget to call 2995 * sk->sk_write_space() once space is available again. 2996 */ 2997 void sk_set_nospace(struct sock *sk) 2998 { 2999 struct socket *sock = sk->sk_socket; 3000 3001 if (!sock) 3002 return; 3003 /* Set SOCK_NOSPACE before tp->tcp_nospace (paired with 3004 * sk_clear_nospace() clearing tp->tcp_nospace before SOCK_NOSPACE) 3005 * so a concurrent clear cannot leave SOCK_NOSPACE set with 3006 * tp->tcp_nospace cleared. 3007 */ 3008 set_bit(SOCK_NOSPACE, &sock->flags); 3009 tcp_set_nospace(sk); 3010 } 3011 EXPORT_SYMBOL(sk_set_nospace); 3012 3013 /** 3014 * sk_clear_nospace - tell the transport no writer is waiting for space 3015 * @sk: socket 3016 * 3017 * Called from ->sk_write_space() handlers, once send space has been 3018 * made available to writers. 3019 */ 3020 void sk_clear_nospace(struct sock *sk) 3021 { 3022 struct socket *sock = sk->sk_socket; 3023 3024 if (!sock) 3025 return; 3026 tcp_clear_nospace(sk); 3027 clear_bit(SOCK_NOSPACE, &sock->flags); 3028 } 3029 EXPORT_SYMBOL(sk_clear_nospace); 3030 3031 /* It is almost wait_for_tcp_memory minus release_sock/lock_sock. 3032 I think, these locks should be removed for datagram sockets. 3033 */ 3034 static long sock_wait_for_wmem(struct sock *sk, long timeo) 3035 { 3036 DEFINE_WAIT(wait); 3037 3038 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 3039 for (;;) { 3040 if (!timeo) 3041 break; 3042 if (signal_pending(current)) 3043 break; 3044 sk_set_nospace(sk); 3045 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); 3046 if (refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) 3047 break; 3048 if (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN) 3049 break; 3050 if (READ_ONCE(sk->sk_err)) 3051 break; 3052 timeo = schedule_timeout(timeo); 3053 } 3054 finish_wait(sk_sleep(sk), &wait); 3055 return timeo; 3056 } 3057 3058 3059 /* 3060 * Generic send/receive buffer handlers 3061 */ 3062 3063 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len, 3064 unsigned long data_len, int noblock, 3065 int *errcode, int max_page_order) 3066 { 3067 struct sk_buff *skb; 3068 long timeo; 3069 int err; 3070 3071 timeo = sock_sndtimeo(sk, noblock); 3072 for (;;) { 3073 err = sock_error(sk); 3074 if (err != 0) 3075 goto failure; 3076 3077 err = -EPIPE; 3078 if (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN) 3079 goto failure; 3080 3081 if (sk_wmem_alloc_get(sk) < READ_ONCE(sk->sk_sndbuf)) 3082 break; 3083 3084 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 3085 sk_set_nospace(sk); 3086 err = -EAGAIN; 3087 if (!timeo) 3088 goto failure; 3089 if (signal_pending(current)) 3090 goto interrupted; 3091 timeo = sock_wait_for_wmem(sk, timeo); 3092 } 3093 skb = alloc_skb_with_frags(header_len, data_len, max_page_order, 3094 errcode, sk->sk_allocation); 3095 if (skb) 3096 skb_set_owner_w(skb, sk); 3097 return skb; 3098 3099 interrupted: 3100 err = sock_intr_errno(timeo); 3101 failure: 3102 *errcode = err; 3103 return NULL; 3104 } 3105 EXPORT_SYMBOL(sock_alloc_send_pskb); 3106 3107 int __sock_cmsg_send(struct sock *sk, struct cmsghdr *cmsg, 3108 struct sockcm_cookie *sockc) 3109 { 3110 u32 tsflags; 3111 3112 BUILD_BUG_ON(SOF_TIMESTAMPING_LAST == (1 << 31)); 3113 3114 switch (cmsg->cmsg_type) { 3115 case SO_MARK: 3116 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && 3117 !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 3118 return -EPERM; 3119 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3120 return -EINVAL; 3121 sockc->mark = *(u32 *)CMSG_DATA(cmsg); 3122 break; 3123 case SO_TIMESTAMPING_OLD: 3124 case SO_TIMESTAMPING_NEW: 3125 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3126 return -EINVAL; 3127 3128 tsflags = *(u32 *)CMSG_DATA(cmsg); 3129 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK) 3130 return -EINVAL; 3131 3132 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK; 3133 sockc->tsflags |= tsflags; 3134 break; 3135 case SCM_TXTIME: 3136 { 3137 ktime_t tmin; 3138 u64 txtime; 3139 3140 if (!sock_flag(sk, SOCK_TXTIME)) 3141 return -EINVAL; 3142 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64))) 3143 return -EINVAL; 3144 3145 txtime = get_unaligned((u64 *)CMSG_DATA(cmsg)); 3146 3147 /* Allow sending without a delivery time: zero special case */ 3148 if (!txtime) { 3149 sockc->transmit_time = 0; 3150 break; 3151 } 3152 3153 switch (sk->sk_clockid) { 3154 case CLOCK_MONOTONIC: 3155 tmin = 1; 3156 break; 3157 case CLOCK_REALTIME: 3158 tmin = max(ktime_mono_to_real(0), 1); 3159 break; 3160 case CLOCK_TAI: 3161 tmin = max(ktime_mono_to_any(0, TK_OFFS_TAI), 1); 3162 break; 3163 default: 3164 tmin = 1; 3165 WARN_ON_ONCE(1); 3166 break; 3167 } 3168 3169 sockc->transmit_time = max_t(ktime_t, txtime, tmin); 3170 break; 3171 } 3172 case SCM_TS_OPT_ID: 3173 if (sk_is_tcp(sk)) 3174 return -EINVAL; 3175 tsflags = READ_ONCE(sk->sk_tsflags); 3176 if (!(tsflags & SOF_TIMESTAMPING_OPT_ID)) 3177 return -EINVAL; 3178 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3179 return -EINVAL; 3180 sockc->ts_opt_id = *(u32 *)CMSG_DATA(cmsg); 3181 sockc->tsflags |= SOCKCM_FLAG_TS_OPT_ID; 3182 break; 3183 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */ 3184 case SCM_RIGHTS: 3185 case SCM_CREDENTIALS: 3186 break; 3187 case SO_PRIORITY: 3188 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3189 return -EINVAL; 3190 if (!sk_set_prio_allowed(sk, *(u32 *)CMSG_DATA(cmsg))) 3191 return -EPERM; 3192 sockc->priority = *(u32 *)CMSG_DATA(cmsg); 3193 break; 3194 case SCM_DEVMEM_DMABUF: 3195 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 3196 return -EINVAL; 3197 sockc->dmabuf_id = *(u32 *)CMSG_DATA(cmsg); 3198 break; 3199 default: 3200 return -EINVAL; 3201 } 3202 return 0; 3203 } 3204 EXPORT_SYMBOL(__sock_cmsg_send); 3205 3206 int sock_cmsg_send(struct sock *sk, struct msghdr *msg, 3207 struct sockcm_cookie *sockc) 3208 { 3209 struct cmsghdr *cmsg; 3210 int ret; 3211 3212 for_each_cmsghdr(cmsg, msg) { 3213 if (!CMSG_OK(msg, cmsg)) 3214 return -EINVAL; 3215 if (cmsg->cmsg_level != SOL_SOCKET) 3216 continue; 3217 ret = __sock_cmsg_send(sk, cmsg, sockc); 3218 if (ret) 3219 return ret; 3220 } 3221 return 0; 3222 } 3223 EXPORT_SYMBOL(sock_cmsg_send); 3224 3225 static void sk_enter_memory_pressure(struct sock *sk) 3226 { 3227 if (!sk->sk_prot->enter_memory_pressure) 3228 return; 3229 3230 sk->sk_prot->enter_memory_pressure(sk); 3231 } 3232 3233 static void sk_leave_memory_pressure(struct sock *sk) 3234 { 3235 if (sk->sk_prot->leave_memory_pressure) { 3236 INDIRECT_CALL_INET_1(sk->sk_prot->leave_memory_pressure, 3237 tcp_leave_memory_pressure, sk); 3238 } else { 3239 unsigned long *memory_pressure = sk->sk_prot->memory_pressure; 3240 3241 if (memory_pressure && READ_ONCE(*memory_pressure)) 3242 WRITE_ONCE(*memory_pressure, 0); 3243 } 3244 } 3245 3246 DEFINE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key); 3247 3248 /** 3249 * skb_page_frag_refill - check that a page_frag contains enough room 3250 * @sz: minimum size of the fragment we want to get 3251 * @pfrag: pointer to page_frag 3252 * @gfp: priority for memory allocation 3253 * 3254 * Note: While this allocator tries to use high order pages, there is 3255 * no guarantee that allocations succeed. Therefore, @sz MUST be 3256 * less or equal than PAGE_SIZE. 3257 */ 3258 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp) 3259 { 3260 if (pfrag->page) { 3261 if (page_ref_count(pfrag->page) == 1) { 3262 pfrag->offset = 0; 3263 return true; 3264 } 3265 if (pfrag->offset + sz <= pfrag->size) 3266 return true; 3267 put_page(pfrag->page); 3268 } 3269 3270 pfrag->offset = 0; 3271 if (SKB_FRAG_PAGE_ORDER && 3272 !static_branch_unlikely(&net_high_order_alloc_disable_key)) { 3273 /* Avoid direct reclaim but allow kswapd to wake */ 3274 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) | 3275 __GFP_COMP | __GFP_NOWARN | 3276 __GFP_NORETRY, 3277 SKB_FRAG_PAGE_ORDER); 3278 if (likely(pfrag->page)) { 3279 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER; 3280 return true; 3281 } 3282 } 3283 pfrag->page = alloc_page(gfp); 3284 if (likely(pfrag->page)) { 3285 pfrag->size = PAGE_SIZE; 3286 return true; 3287 } 3288 return false; 3289 } 3290 EXPORT_SYMBOL(skb_page_frag_refill); 3291 3292 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 3293 { 3294 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation))) 3295 return true; 3296 3297 if (!sk->sk_bypass_prot_mem) 3298 sk_enter_memory_pressure(sk); 3299 3300 sk_stream_moderate_sndbuf(sk); 3301 3302 return false; 3303 } 3304 EXPORT_SYMBOL(sk_page_frag_refill); 3305 3306 static void __lock_sock(struct sock *sk) 3307 __releases(&sk->sk_lock.slock) 3308 __acquires(&sk->sk_lock.slock) 3309 { 3310 DEFINE_WAIT(wait); 3311 3312 for (;;) { 3313 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait, 3314 TASK_UNINTERRUPTIBLE); 3315 spin_unlock_bh(&sk->sk_lock.slock); 3316 schedule(); 3317 spin_lock_bh(&sk->sk_lock.slock); 3318 if (!sock_owned_by_user(sk)) 3319 break; 3320 } 3321 finish_wait(&sk->sk_lock.wq, &wait); 3322 } 3323 3324 void __release_sock(struct sock *sk) 3325 __releases(&sk->sk_lock.slock) 3326 __acquires(&sk->sk_lock.slock) 3327 { 3328 struct sk_buff *skb, *next; 3329 int nb = 0; 3330 3331 while ((skb = sk->sk_backlog.head) != NULL) { 3332 sk->sk_backlog.head = sk->sk_backlog.tail = NULL; 3333 3334 spin_unlock_bh(&sk->sk_lock.slock); 3335 3336 while (1) { 3337 next = skb->next; 3338 prefetch(next); 3339 DEBUG_NET_WARN_ON_ONCE(skb_dst_is_noref(skb)); 3340 skb_mark_not_on_list(skb); 3341 sk_backlog_rcv(sk, skb); 3342 3343 skb = next; 3344 if (!skb) 3345 break; 3346 3347 if (!(++nb & 15)) 3348 cond_resched(); 3349 } 3350 3351 spin_lock_bh(&sk->sk_lock.slock); 3352 } 3353 3354 /* 3355 * Doing the zeroing here guarantee we can not loop forever 3356 * while a wild producer attempts to flood us. 3357 */ 3358 sk->sk_backlog.len = 0; 3359 } 3360 3361 void __sk_flush_backlog(struct sock *sk) 3362 { 3363 spin_lock_bh(&sk->sk_lock.slock); 3364 __release_sock(sk); 3365 3366 if (sk->sk_prot->release_cb) 3367 INDIRECT_CALL_INET_1(sk->sk_prot->release_cb, 3368 tcp_release_cb, sk); 3369 3370 spin_unlock_bh(&sk->sk_lock.slock); 3371 } 3372 EXPORT_SYMBOL_GPL(__sk_flush_backlog); 3373 3374 /** 3375 * sk_wait_data - wait for data to arrive at sk_receive_queue 3376 * @sk: sock to wait on 3377 * @timeo: for how long 3378 * @skb: last skb seen on sk_receive_queue 3379 * 3380 * Now socket state including sk->sk_err is changed only under lock, 3381 * hence we may omit checks after joining wait queue. 3382 * We check receive queue before schedule() only as optimization; 3383 * it is very likely that release_sock() added new data. 3384 */ 3385 int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb) 3386 { 3387 DEFINE_WAIT_FUNC(wait, woken_wake_function); 3388 int rc; 3389 3390 add_wait_queue(sk_sleep(sk), &wait); 3391 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 3392 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait); 3393 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 3394 remove_wait_queue(sk_sleep(sk), &wait); 3395 return rc; 3396 } 3397 EXPORT_SYMBOL(sk_wait_data); 3398 3399 /** 3400 * __sk_mem_raise_allocated - increase memory_allocated 3401 * @sk: socket 3402 * @size: memory size to allocate 3403 * @amt: pages to allocate 3404 * @kind: allocation type 3405 * 3406 * Similar to __sk_mem_schedule(), but does not update sk_forward_alloc. 3407 * 3408 * Unlike the globally shared limits among the sockets under same protocol, 3409 * consuming the budget of a memcg won't have direct effect on other ones. 3410 * So be optimistic about memcg's tolerance, and leave the callers to decide 3411 * whether or not to raise allocated through sk_under_memory_pressure() or 3412 * its variants. 3413 */ 3414 int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind) 3415 { 3416 bool memcg_enabled = false, charged = false; 3417 struct proto *prot = sk->sk_prot; 3418 long allocated = 0; 3419 3420 if (!sk->sk_bypass_prot_mem) { 3421 sk_memory_allocated_add(sk, amt); 3422 allocated = sk_memory_allocated(sk); 3423 } 3424 3425 if (mem_cgroup_sk_enabled(sk)) { 3426 memcg_enabled = true; 3427 charged = mem_cgroup_sk_charge(sk, amt, gfp_memcg_charge()); 3428 if (!charged) 3429 goto suppress_allocation; 3430 } 3431 3432 if (!allocated) 3433 return 1; 3434 3435 /* Under limit. */ 3436 if (allocated <= sk_prot_mem_limits(sk, 0)) { 3437 sk_leave_memory_pressure(sk); 3438 return 1; 3439 } 3440 3441 /* Under pressure. */ 3442 if (allocated > sk_prot_mem_limits(sk, 1)) 3443 sk_enter_memory_pressure(sk); 3444 3445 /* Over hard limit. */ 3446 if (allocated > sk_prot_mem_limits(sk, 2)) 3447 goto suppress_allocation; 3448 3449 /* Guarantee minimum buffer size under pressure (either global 3450 * or memcg) to make sure features described in RFC 7323 (TCP 3451 * Extensions for High Performance) work properly. 3452 * 3453 * This rule does NOT stand when exceeds global or memcg's hard 3454 * limit, or else a DoS attack can be taken place by spawning 3455 * lots of sockets whose usage are under minimum buffer size. 3456 */ 3457 if (kind == SK_MEM_RECV) { 3458 if (atomic_read(&sk->sk_rmem_alloc) < sk_get_rmem0(sk, prot)) 3459 return 1; 3460 3461 } else { /* SK_MEM_SEND */ 3462 int wmem0 = sk_get_wmem0(sk, prot); 3463 3464 if (sk->sk_type == SOCK_STREAM) { 3465 if (sk->sk_wmem_queued < wmem0) 3466 return 1; 3467 } else if (refcount_read(&sk->sk_wmem_alloc) < wmem0) { 3468 return 1; 3469 } 3470 } 3471 3472 if (sk_has_memory_pressure(sk)) { 3473 u64 alloc; 3474 3475 /* The following 'average' heuristic is within the 3476 * scope of global accounting, so it only makes 3477 * sense for global memory pressure. 3478 */ 3479 if (!sk_under_global_memory_pressure(sk)) 3480 return 1; 3481 3482 /* Try to be fair among all the sockets under global 3483 * pressure by allowing the ones that below average 3484 * usage to raise. 3485 */ 3486 alloc = sk_sockets_allocated_read_positive(sk); 3487 if (sk_prot_mem_limits(sk, 2) > alloc * 3488 sk_mem_pages(sk->sk_wmem_queued + 3489 atomic_read(&sk->sk_rmem_alloc) + 3490 sk->sk_forward_alloc)) 3491 return 1; 3492 } 3493 3494 suppress_allocation: 3495 3496 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) { 3497 sk_stream_moderate_sndbuf(sk); 3498 3499 /* Fail only if socket is _under_ its sndbuf. 3500 * In this case we cannot block, so that we have to fail. 3501 */ 3502 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf) { 3503 /* Force charge with __GFP_NOFAIL */ 3504 if (memcg_enabled && !charged) 3505 mem_cgroup_sk_charge(sk, amt, 3506 gfp_memcg_charge() | __GFP_NOFAIL); 3507 return 1; 3508 } 3509 } 3510 3511 trace_sock_exceed_buf_limit(sk, prot, allocated, kind); 3512 3513 if (allocated) 3514 sk_memory_allocated_sub(sk, amt); 3515 3516 if (charged) 3517 mem_cgroup_sk_uncharge(sk, amt); 3518 3519 return 0; 3520 } 3521 3522 /** 3523 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated 3524 * @sk: socket 3525 * @size: memory size to allocate 3526 * @kind: allocation type 3527 * 3528 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means 3529 * rmem allocation. This function assumes that protocols which have 3530 * memory_pressure use sk_wmem_queued as write buffer accounting. 3531 */ 3532 int __sk_mem_schedule(struct sock *sk, int size, int kind) 3533 { 3534 int ret, amt = sk_mem_pages(size); 3535 3536 sk_forward_alloc_add(sk, amt << PAGE_SHIFT); 3537 ret = __sk_mem_raise_allocated(sk, size, amt, kind); 3538 if (!ret) 3539 sk_forward_alloc_add(sk, -(amt << PAGE_SHIFT)); 3540 return ret; 3541 } 3542 EXPORT_SYMBOL(__sk_mem_schedule); 3543 3544 /** 3545 * __sk_mem_reduce_allocated - reclaim memory_allocated 3546 * @sk: socket 3547 * @amount: number of quanta 3548 * 3549 * Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc 3550 */ 3551 void __sk_mem_reduce_allocated(struct sock *sk, int amount) 3552 { 3553 if (mem_cgroup_sk_enabled(sk)) 3554 mem_cgroup_sk_uncharge(sk, amount); 3555 3556 if (sk->sk_bypass_prot_mem) 3557 return; 3558 3559 sk_memory_allocated_sub(sk, amount); 3560 3561 if (sk_under_global_memory_pressure(sk) && 3562 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0))) 3563 sk_leave_memory_pressure(sk); 3564 } 3565 3566 /** 3567 * __sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated 3568 * @sk: socket 3569 * @amount: number of bytes (rounded down to a PAGE_SIZE multiple) 3570 */ 3571 void __sk_mem_reclaim(struct sock *sk, int amount) 3572 { 3573 amount >>= PAGE_SHIFT; 3574 sk_forward_alloc_add(sk, -(amount << PAGE_SHIFT)); 3575 __sk_mem_reduce_allocated(sk, amount); 3576 } 3577 EXPORT_SYMBOL(__sk_mem_reclaim); 3578 3579 void __sk_charge(struct sock *sk, gfp_t gfp) 3580 { 3581 int amt; 3582 3583 gfp |= __GFP_NOFAIL; 3584 if (mem_cgroup_from_sk(sk)) { 3585 /* The socket has not been accepted yet, no need 3586 * to look at newsk->sk_wmem_queued. 3587 */ 3588 amt = sk_mem_pages(sk->sk_forward_alloc + 3589 atomic_read(&sk->sk_rmem_alloc)); 3590 if (amt) 3591 mem_cgroup_sk_charge(sk, amt, gfp); 3592 } 3593 3594 kmem_cache_charge(sk, gfp); 3595 } 3596 3597 int sk_set_peek_off(struct sock *sk, int val) 3598 { 3599 WRITE_ONCE(sk->sk_peek_off, val); 3600 return 0; 3601 } 3602 EXPORT_SYMBOL_GPL(sk_set_peek_off); 3603 3604 /* 3605 * Set of default routines for initialising struct proto_ops when 3606 * the protocol does not support a particular function. In certain 3607 * cases where it makes no sense for a protocol to have a "do nothing" 3608 * function, some default processing is provided. 3609 */ 3610 3611 int sock_no_bind(struct socket *sock, struct sockaddr_unsized *saddr, int len) 3612 { 3613 return -EOPNOTSUPP; 3614 } 3615 EXPORT_SYMBOL(sock_no_bind); 3616 3617 int sock_no_connect(struct socket *sock, struct sockaddr_unsized *saddr, 3618 int len, int flags) 3619 { 3620 return -EOPNOTSUPP; 3621 } 3622 EXPORT_SYMBOL(sock_no_connect); 3623 3624 int sock_no_socketpair(struct socket *sock1, struct socket *sock2) 3625 { 3626 return -EOPNOTSUPP; 3627 } 3628 EXPORT_SYMBOL(sock_no_socketpair); 3629 3630 int sock_no_accept(struct socket *sock, struct socket *newsock, 3631 struct proto_accept_arg *arg) 3632 { 3633 return -EOPNOTSUPP; 3634 } 3635 EXPORT_SYMBOL(sock_no_accept); 3636 3637 int sock_no_getname(struct socket *sock, struct sockaddr *saddr, 3638 int peer) 3639 { 3640 return -EOPNOTSUPP; 3641 } 3642 EXPORT_SYMBOL(sock_no_getname); 3643 3644 int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) 3645 { 3646 return -EOPNOTSUPP; 3647 } 3648 EXPORT_SYMBOL(sock_no_ioctl); 3649 3650 int sock_no_listen(struct socket *sock, int backlog) 3651 { 3652 return -EOPNOTSUPP; 3653 } 3654 EXPORT_SYMBOL(sock_no_listen); 3655 3656 int sock_no_shutdown(struct socket *sock, int how) 3657 { 3658 return -EOPNOTSUPP; 3659 } 3660 EXPORT_SYMBOL(sock_no_shutdown); 3661 3662 int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len) 3663 { 3664 return -EOPNOTSUPP; 3665 } 3666 EXPORT_SYMBOL(sock_no_sendmsg); 3667 3668 int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len) 3669 { 3670 return -EOPNOTSUPP; 3671 } 3672 EXPORT_SYMBOL(sock_no_sendmsg_locked); 3673 3674 int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len, 3675 int flags) 3676 { 3677 return -EOPNOTSUPP; 3678 } 3679 EXPORT_SYMBOL(sock_no_recvmsg); 3680 3681 int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma) 3682 { 3683 /* Mirror missing mmap method error code */ 3684 return -ENODEV; 3685 } 3686 EXPORT_SYMBOL(sock_no_mmap); 3687 3688 /* 3689 * When a file is received (via SCM_RIGHTS, etc), we must bump the 3690 * various sock-based usage counts. 3691 */ 3692 void __receive_sock(struct file *file) 3693 { 3694 struct socket *sock; 3695 3696 sock = sock_from_file(file); 3697 if (sock) { 3698 sock_update_netprioidx(&sock->sk->sk_cgrp_data); 3699 sock_update_classid(&sock->sk->sk_cgrp_data); 3700 } 3701 } 3702 3703 /* 3704 * Default Socket Callbacks 3705 */ 3706 3707 static void sock_def_wakeup(struct sock *sk) 3708 { 3709 struct socket_wq *wq; 3710 3711 rcu_read_lock(); 3712 wq = rcu_dereference(sk->sk_wq); 3713 if (skwq_has_sleeper(wq)) 3714 wake_up_interruptible_all(&wq->wait); 3715 rcu_read_unlock(); 3716 } 3717 3718 static void sock_def_error_report(struct sock *sk) 3719 { 3720 struct socket_wq *wq; 3721 3722 rcu_read_lock(); 3723 wq = rcu_dereference(sk->sk_wq); 3724 if (skwq_has_sleeper(wq)) 3725 wake_up_interruptible_poll(&wq->wait, EPOLLERR); 3726 sk_wake_async_rcu(sk, SOCK_WAKE_IO, POLL_ERR); 3727 rcu_read_unlock(); 3728 } 3729 3730 void sock_def_readable(struct sock *sk) 3731 { 3732 struct socket_wq *wq; 3733 3734 trace_sk_data_ready(sk); 3735 3736 rcu_read_lock(); 3737 wq = rcu_dereference(sk->sk_wq); 3738 if (skwq_has_sleeper(wq)) 3739 wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI | 3740 EPOLLRDNORM | EPOLLRDBAND); 3741 sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_IN); 3742 rcu_read_unlock(); 3743 } 3744 3745 static void sock_def_write_space(struct sock *sk) 3746 { 3747 struct socket_wq *wq; 3748 3749 rcu_read_lock(); 3750 3751 /* Do not wake up a writer until he can make "significant" 3752 * progress. --DaveM 3753 */ 3754 if (sock_writeable(sk)) { 3755 wq = rcu_dereference(sk->sk_wq); 3756 if (skwq_has_sleeper(wq)) 3757 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 3758 EPOLLWRNORM | EPOLLWRBAND); 3759 3760 /* Should agree with poll, otherwise some programs break */ 3761 sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT); 3762 } 3763 3764 rcu_read_unlock(); 3765 } 3766 3767 /* An optimised version of sock_def_write_space(), should only be called 3768 * for SOCK_RCU_FREE sockets under RCU read section and after putting 3769 * ->sk_wmem_alloc. 3770 */ 3771 static void sock_def_write_space_wfree(struct sock *sk, int wmem_alloc) 3772 { 3773 /* Do not wake up a writer until he can make "significant" 3774 * progress. --DaveM 3775 */ 3776 if (__sock_writeable(sk, wmem_alloc)) { 3777 struct socket_wq *wq = rcu_dereference(sk->sk_wq); 3778 3779 /* rely on refcount_sub from sock_wfree() */ 3780 smp_mb__after_atomic(); 3781 if (wq && waitqueue_active(&wq->wait)) 3782 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 3783 EPOLLWRNORM | EPOLLWRBAND); 3784 3785 /* Should agree with poll, otherwise some programs break */ 3786 sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT); 3787 } 3788 } 3789 3790 static void sock_def_destruct(struct sock *sk) 3791 { 3792 } 3793 3794 void sk_send_sigurg(struct sock *sk) 3795 { 3796 if (sk->sk_socket && sk->sk_socket->file) 3797 if (send_sigurg(sk->sk_socket->file)) 3798 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI); 3799 } 3800 EXPORT_SYMBOL(sk_send_sigurg); 3801 3802 void sk_reset_timer(struct sock *sk, struct timer_list* timer, 3803 unsigned long expires) 3804 { 3805 if (!mod_timer(timer, expires)) 3806 sock_hold(sk); 3807 } 3808 EXPORT_SYMBOL(sk_reset_timer); 3809 3810 void sk_stop_timer(struct sock *sk, struct timer_list* timer) 3811 { 3812 if (timer_delete(timer)) 3813 __sock_put(sk); 3814 } 3815 EXPORT_SYMBOL(sk_stop_timer); 3816 3817 void sk_stop_timer_sync(struct sock *sk, struct timer_list *timer) 3818 { 3819 if (timer_delete_sync(timer)) 3820 __sock_put(sk); 3821 } 3822 EXPORT_SYMBOL(sk_stop_timer_sync); 3823 3824 void sock_init_data_uid(struct socket *sock, struct sock *sk, kuid_t uid) 3825 { 3826 sk_init_common(sk); 3827 sk->sk_send_head = NULL; 3828 3829 timer_setup(&sk->sk_timer, NULL, 0); 3830 3831 sk->sk_allocation = GFP_KERNEL; 3832 sk->sk_rcvbuf = READ_ONCE(sysctl_rmem_default); 3833 sk->sk_sndbuf = READ_ONCE(sysctl_wmem_default); 3834 sk->sk_state = TCP_CLOSE; 3835 sk->sk_use_task_frag = true; 3836 sk_set_socket(sk, sock); 3837 3838 sock_set_flag(sk, SOCK_ZAPPED); 3839 3840 if (sock) { 3841 sk->sk_type = sock->type; 3842 RCU_INIT_POINTER(sk->sk_wq, &sock->wq); 3843 sock->sk = sk; 3844 } else { 3845 RCU_INIT_POINTER(sk->sk_wq, NULL); 3846 } 3847 sk->sk_uid = uid; 3848 3849 sk->sk_state_change = sock_def_wakeup; 3850 sk->sk_data_ready = sock_def_readable; 3851 sk->sk_write_space = sock_def_write_space; 3852 sk->sk_error_report = sock_def_error_report; 3853 sk->sk_destruct = sock_def_destruct; 3854 3855 sk->sk_frag.page = NULL; 3856 sk->sk_frag.offset = 0; 3857 sk->sk_peek_off = -1; 3858 3859 memset(sk->sk_peer_pid, 0, sizeof(sk->sk_peer_pid)); 3860 sk->sk_peer_cred = NULL; 3861 spin_lock_init(&sk->sk_peer_lock); 3862 3863 sk->sk_write_pending = 0; 3864 sk->sk_rcvlowat = 1; 3865 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT; 3866 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT; 3867 3868 sk->sk_stamp = SK_DEFAULT_STAMP; 3869 #if BITS_PER_LONG==32 3870 seqlock_init(&sk->sk_stamp_seq); 3871 #endif 3872 atomic_set(&sk->sk_zckey, 0); 3873 3874 #ifdef CONFIG_NET_RX_BUSY_POLL 3875 sk->sk_napi_id = 0; 3876 sk->sk_ll_usec = READ_ONCE(sysctl_net_busy_read); 3877 #endif 3878 3879 sk->sk_max_pacing_rate = ~0UL; 3880 sk->sk_pacing_rate = ~0UL; 3881 WRITE_ONCE(sk->sk_pacing_shift, 10); 3882 sk->sk_incoming_cpu = -1; 3883 3884 sk_rx_queue_clear(sk); 3885 /* 3886 * Before updating sk_refcnt, we must commit prior changes to memory 3887 * (Documentation/RCU/rculist_nulls.rst for details) 3888 */ 3889 smp_wmb(); 3890 refcount_set(&sk->sk_refcnt, 1); 3891 sk_drops_reset(sk); 3892 } 3893 EXPORT_SYMBOL(sock_init_data_uid); 3894 3895 void sock_init_data(struct socket *sock, struct sock *sk) 3896 { 3897 kuid_t uid = sock ? 3898 SOCK_INODE(sock)->i_uid : 3899 make_kuid(sock_net(sk)->user_ns, 0); 3900 3901 sock_init_data_uid(sock, sk, uid); 3902 } 3903 EXPORT_SYMBOL(sock_init_data); 3904 3905 void noinline lock_sock_nested(struct sock *sk, int subclass) 3906 { 3907 /* The sk_lock has mutex_lock() semantics here. */ 3908 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_); 3909 3910 might_sleep(); 3911 #ifdef CONFIG_64BIT 3912 if (sizeof(struct slock_owned) == sizeof(long)) { 3913 socket_lock_t tmp = { 3914 .slock = __SPIN_LOCK_UNLOCKED(tmp.slock), 3915 .owned = 1, 3916 }; 3917 socket_lock_t old = { 3918 .slock = __SPIN_LOCK_UNLOCKED(old.slock), 3919 .owned = 0, 3920 }; 3921 3922 if (likely(try_cmpxchg(&sk->sk_lock.combined, 3923 &old.combined, tmp.combined))) 3924 return; 3925 } 3926 #endif 3927 spin_lock_bh(&sk->sk_lock.slock); 3928 if (unlikely(sock_owned_by_user_nocheck(sk))) 3929 __lock_sock(sk); 3930 sk->sk_lock.owned = 1; 3931 spin_unlock_bh(&sk->sk_lock.slock); 3932 } 3933 EXPORT_SYMBOL(lock_sock_nested); 3934 3935 void release_sock(struct sock *sk) 3936 { 3937 spin_lock_bh(&sk->sk_lock.slock); 3938 3939 if (unlikely(sk->sk_backlog.tail)) 3940 __release_sock(sk); 3941 3942 if (sk->sk_prot->release_cb) { 3943 if (!tcp_release_cb_cond(sk)) 3944 sk->sk_prot->release_cb(sk); 3945 } 3946 sock_release_ownership(sk); 3947 if (unlikely(waitqueue_active(&sk->sk_lock.wq))) 3948 wake_up(&sk->sk_lock.wq); 3949 3950 spin_unlock_bh(&sk->sk_lock.slock); 3951 } 3952 EXPORT_SYMBOL(release_sock); 3953 3954 bool __lock_sock_fast(struct sock *sk) __acquires(&sk->sk_lock.slock) 3955 { 3956 might_sleep(); 3957 spin_lock_bh(&sk->sk_lock.slock); 3958 3959 if (likely(!sock_owned_by_user_nocheck(sk))) { 3960 /* 3961 * Fast path return with bottom halves disabled and 3962 * sock::sk_lock.slock held. 3963 * 3964 * The 'mutex' is not contended and holding 3965 * sock::sk_lock.slock prevents all other lockers to 3966 * proceed so the corresponding unlock_sock_fast() can 3967 * avoid the slow path of release_sock() completely and 3968 * just release slock. 3969 * 3970 * From a semantical POV this is equivalent to 'acquiring' 3971 * the 'mutex', hence the corresponding lockdep 3972 * mutex_release() has to happen in the fast path of 3973 * unlock_sock_fast(). 3974 */ 3975 return false; 3976 } 3977 3978 __lock_sock(sk); 3979 sk->sk_lock.owned = 1; 3980 __acquire(&sk->sk_lock.slock); 3981 spin_unlock_bh(&sk->sk_lock.slock); 3982 return true; 3983 } 3984 EXPORT_SYMBOL(__lock_sock_fast); 3985 3986 int sock_gettstamp(struct socket *sock, void __user *userstamp, 3987 bool timeval, bool time32) 3988 { 3989 struct sock *sk = sock->sk; 3990 struct timespec64 ts; 3991 3992 /* sk->sk_flags must only be changed under the socket lock, 3993 * because sock_set_flag() uses non atomic operations. 3994 */ 3995 if (!sock_flag(sk, SOCK_TIMESTAMP)) { 3996 lock_sock(sk); 3997 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 3998 release_sock(sk); 3999 } 4000 ts = ktime_to_timespec64(sock_read_timestamp(sk)); 4001 if (ts.tv_sec == -1) 4002 return -ENOENT; 4003 if (ts.tv_sec == 0) { 4004 ktime_t kt = ktime_get_real(); 4005 sock_write_timestamp(sk, kt); 4006 ts = ktime_to_timespec64(kt); 4007 } 4008 4009 if (timeval) 4010 ts.tv_nsec /= 1000; 4011 4012 #ifdef CONFIG_COMPAT_32BIT_TIME 4013 if (time32) 4014 return put_old_timespec32(&ts, userstamp); 4015 #endif 4016 #ifdef CONFIG_SPARC64 4017 /* beware of padding in sparc64 timeval */ 4018 if (timeval && !in_compat_syscall()) { 4019 struct __kernel_old_timeval __user tv = { 4020 .tv_sec = ts.tv_sec, 4021 .tv_usec = ts.tv_nsec, 4022 }; 4023 if (copy_to_user(userstamp, &tv, sizeof(tv))) 4024 return -EFAULT; 4025 return 0; 4026 } 4027 #endif 4028 return put_timespec64(&ts, userstamp); 4029 } 4030 EXPORT_SYMBOL(sock_gettstamp); 4031 4032 void sock_enable_timestamp(struct sock *sk, enum sock_flags flag) 4033 { 4034 if (!sock_flag(sk, flag)) { 4035 unsigned long previous_flags = sk->sk_flags; 4036 4037 sock_set_flag(sk, flag); 4038 /* 4039 * we just set one of the two flags which require net 4040 * time stamping, but time stamping might have been on 4041 * already because of the other one 4042 */ 4043 if (sock_needs_netstamp(sk) && 4044 !(previous_flags & SK_FLAGS_TIMESTAMP)) 4045 net_enable_timestamp(); 4046 } 4047 } 4048 4049 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, 4050 int level, int type) 4051 { 4052 struct sock_extended_err ee; 4053 struct sk_buff *skb; 4054 int copied, err; 4055 4056 err = -EAGAIN; 4057 skb = sock_dequeue_err_skb(sk); 4058 if (skb == NULL) 4059 goto out; 4060 4061 copied = skb->len; 4062 if (copied > len) { 4063 msg->msg_flags |= MSG_TRUNC; 4064 copied = len; 4065 } 4066 err = skb_copy_datagram_msg(skb, 0, msg, copied); 4067 if (err) 4068 goto out_free_skb; 4069 4070 sock_recv_timestamp(msg, sk, skb); 4071 4072 /* We must use a bounce buffer for CONFIG_HARDENED_USERCOPY=y */ 4073 ee = SKB_EXT_ERR(skb)->ee; 4074 put_cmsg(msg, level, type, sizeof(ee), &ee); 4075 4076 msg->msg_flags |= MSG_ERRQUEUE; 4077 err = copied; 4078 4079 out_free_skb: 4080 kfree_skb(skb); 4081 out: 4082 return err; 4083 } 4084 EXPORT_SYMBOL(sock_recv_errqueue); 4085 4086 /* 4087 * Get a socket option on an socket. 4088 * 4089 * FIX: POSIX 1003.1g is very ambiguous here. It states that 4090 * asynchronous errors should be reported by getsockopt. We assume 4091 * this means if you specify SO_ERROR (otherwise what is the point of it). 4092 */ 4093 int sock_common_getsockopt(struct socket *sock, int level, int optname, 4094 char __user *optval, int __user *optlen) 4095 { 4096 struct sock *sk = sock->sk; 4097 4098 return READ_ONCE(sk->sk_prot)->getsockopt(sk, level, optname, optval, optlen); 4099 } 4100 EXPORT_SYMBOL(sock_common_getsockopt); 4101 4102 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, 4103 int flags) 4104 { 4105 struct sock *sk = sock->sk; 4106 4107 return sk->sk_prot->recvmsg(sk, msg, size, flags); 4108 } 4109 EXPORT_SYMBOL(sock_common_recvmsg); 4110 4111 /* 4112 * Set socket options on an inet socket. 4113 */ 4114 int sock_common_setsockopt(struct socket *sock, int level, int optname, 4115 sockptr_t optval, unsigned int optlen) 4116 { 4117 struct sock *sk = sock->sk; 4118 4119 return READ_ONCE(sk->sk_prot)->setsockopt(sk, level, optname, optval, optlen); 4120 } 4121 EXPORT_SYMBOL(sock_common_setsockopt); 4122 4123 void sk_common_release(struct sock *sk) 4124 { 4125 if (sk->sk_prot->destroy) 4126 sk->sk_prot->destroy(sk); 4127 4128 /* 4129 * Observation: when sk_common_release is called, processes have 4130 * no access to socket. But net still has. 4131 * Step one, detach it from networking: 4132 * 4133 * A. Remove from hash tables. 4134 */ 4135 4136 sk->sk_prot->unhash(sk); 4137 4138 /* 4139 * In this point socket cannot receive new packets, but it is possible 4140 * that some packets are in flight because some CPU runs receiver and 4141 * did hash table lookup before we unhashed socket. They will achieve 4142 * receive queue and will be purged by socket destructor. 4143 * 4144 * Also we still have packets pending on receive queue and probably, 4145 * our own packets waiting in device queues. sock_destroy will drain 4146 * receive queue, but transmitted packets will delay socket destruction 4147 * until the last reference will be released. 4148 */ 4149 4150 sock_orphan(sk); 4151 4152 xfrm_sk_free_policy(sk); 4153 4154 sock_put(sk); 4155 } 4156 EXPORT_SYMBOL(sk_common_release); 4157 4158 void sk_get_meminfo(const struct sock *sk, u32 *mem) 4159 { 4160 memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS); 4161 4162 mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk); 4163 mem[SK_MEMINFO_RCVBUF] = READ_ONCE(sk->sk_rcvbuf); 4164 mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk); 4165 mem[SK_MEMINFO_SNDBUF] = READ_ONCE(sk->sk_sndbuf); 4166 mem[SK_MEMINFO_FWD_ALLOC] = READ_ONCE(sk->sk_forward_alloc); 4167 mem[SK_MEMINFO_WMEM_QUEUED] = READ_ONCE(sk->sk_wmem_queued); 4168 mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc); 4169 mem[SK_MEMINFO_BACKLOG] = READ_ONCE(sk->sk_backlog.len); 4170 mem[SK_MEMINFO_DROPS] = sk_drops_read(sk); 4171 } 4172 4173 #ifdef CONFIG_PROC_FS 4174 static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR); 4175 4176 int sock_prot_inuse_get(struct net *net, struct proto *prot) 4177 { 4178 int cpu, idx = prot->inuse_idx; 4179 int res = 0; 4180 4181 for_each_possible_cpu(cpu) 4182 res += per_cpu_ptr(net->core.prot_inuse, cpu)->val[idx]; 4183 4184 return res >= 0 ? res : 0; 4185 } 4186 EXPORT_SYMBOL_GPL(sock_prot_inuse_get); 4187 4188 int sock_inuse_get(struct net *net) 4189 { 4190 int cpu, res = 0; 4191 4192 for_each_possible_cpu(cpu) 4193 res += per_cpu_ptr(net->core.prot_inuse, cpu)->all; 4194 4195 return res; 4196 } 4197 4198 EXPORT_SYMBOL_GPL(sock_inuse_get); 4199 4200 static int __net_init sock_inuse_init_net(struct net *net) 4201 { 4202 net->core.prot_inuse = alloc_percpu(struct prot_inuse); 4203 if (net->core.prot_inuse == NULL) 4204 return -ENOMEM; 4205 return 0; 4206 } 4207 4208 static void __net_exit sock_inuse_exit_net(struct net *net) 4209 { 4210 free_percpu(net->core.prot_inuse); 4211 } 4212 4213 static struct pernet_operations net_inuse_ops = { 4214 .init = sock_inuse_init_net, 4215 .exit = sock_inuse_exit_net, 4216 }; 4217 4218 static __init int net_inuse_init(void) 4219 { 4220 if (register_pernet_subsys(&net_inuse_ops)) 4221 panic("Cannot initialize net inuse counters"); 4222 4223 return 0; 4224 } 4225 4226 core_initcall(net_inuse_init); 4227 4228 static int assign_proto_idx(struct proto *prot) 4229 { 4230 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR); 4231 4232 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR)) { 4233 pr_err("PROTO_INUSE_NR exhausted\n"); 4234 return -ENOSPC; 4235 } 4236 4237 set_bit(prot->inuse_idx, proto_inuse_idx); 4238 return 0; 4239 } 4240 4241 static void release_proto_idx(struct proto *prot) 4242 { 4243 if (prot->inuse_idx != PROTO_INUSE_NR) 4244 clear_bit(prot->inuse_idx, proto_inuse_idx); 4245 } 4246 #else 4247 static inline int assign_proto_idx(struct proto *prot) 4248 { 4249 return 0; 4250 } 4251 4252 static inline void release_proto_idx(struct proto *prot) 4253 { 4254 } 4255 4256 #endif 4257 4258 static void tw_prot_cleanup(struct timewait_sock_ops *twsk_prot) 4259 { 4260 if (!twsk_prot) 4261 return; 4262 kfree(twsk_prot->twsk_slab_name); 4263 twsk_prot->twsk_slab_name = NULL; 4264 kmem_cache_destroy(twsk_prot->twsk_slab); 4265 twsk_prot->twsk_slab = NULL; 4266 } 4267 4268 static int tw_prot_init(const struct proto *prot) 4269 { 4270 struct timewait_sock_ops *twsk_prot = prot->twsk_prot; 4271 4272 if (!twsk_prot) 4273 return 0; 4274 4275 twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", 4276 prot->name); 4277 if (!twsk_prot->twsk_slab_name) 4278 return -ENOMEM; 4279 4280 twsk_prot->twsk_slab = 4281 kmem_cache_create(twsk_prot->twsk_slab_name, 4282 twsk_prot->twsk_obj_size, 0, 4283 SLAB_ACCOUNT | prot->slab_flags, 4284 NULL); 4285 if (!twsk_prot->twsk_slab) { 4286 pr_crit("%s: Can't create timewait sock SLAB cache!\n", 4287 prot->name); 4288 return -ENOMEM; 4289 } 4290 4291 return 0; 4292 } 4293 4294 static void req_prot_cleanup(struct request_sock_ops *rsk_prot) 4295 { 4296 if (!rsk_prot) 4297 return; 4298 kfree(rsk_prot->slab_name); 4299 rsk_prot->slab_name = NULL; 4300 kmem_cache_destroy(rsk_prot->slab); 4301 rsk_prot->slab = NULL; 4302 } 4303 4304 static int req_prot_init(const struct proto *prot) 4305 { 4306 struct request_sock_ops *rsk_prot = prot->rsk_prot; 4307 4308 if (!rsk_prot) 4309 return 0; 4310 4311 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s", 4312 prot->name); 4313 if (!rsk_prot->slab_name) 4314 return -ENOMEM; 4315 4316 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name, 4317 rsk_prot->obj_size, 0, 4318 SLAB_ACCOUNT | prot->slab_flags, 4319 NULL); 4320 4321 if (!rsk_prot->slab) { 4322 pr_crit("%s: Can't create request sock SLAB cache!\n", 4323 prot->name); 4324 return -ENOMEM; 4325 } 4326 return 0; 4327 } 4328 4329 int proto_register(struct proto *prot, int alloc_slab) 4330 { 4331 int ret = -ENOBUFS; 4332 4333 if (prot->memory_allocated && !prot->sysctl_mem) { 4334 pr_err("%s: missing sysctl_mem\n", prot->name); 4335 return -EINVAL; 4336 } 4337 if (prot->memory_allocated && !prot->per_cpu_fw_alloc) { 4338 pr_err("%s: missing per_cpu_fw_alloc\n", prot->name); 4339 return -EINVAL; 4340 } 4341 if (alloc_slab) { 4342 struct kmem_cache_args args = { 4343 .useroffset = prot->useroffset, 4344 .usersize = prot->usersize, 4345 .freeptr_offset = prot->freeptr_offset, 4346 .use_freeptr_offset = !!prot->freeptr_offset, 4347 }; 4348 4349 prot->slab = kmem_cache_create(prot->name, prot->obj_size, 4350 &args, 4351 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT | 4352 prot->slab_flags); 4353 if (prot->slab == NULL) { 4354 pr_crit("%s: Can't create sock SLAB cache!\n", 4355 prot->name); 4356 goto out; 4357 } 4358 4359 if (req_prot_init(prot)) 4360 goto out_free_request_sock_slab; 4361 4362 if (tw_prot_init(prot)) 4363 goto out_free_timewait_sock_slab; 4364 } 4365 4366 mutex_lock(&proto_list_mutex); 4367 ret = assign_proto_idx(prot); 4368 if (ret) { 4369 mutex_unlock(&proto_list_mutex); 4370 goto out_free_timewait_sock_slab; 4371 } 4372 list_add(&prot->node, &proto_list); 4373 mutex_unlock(&proto_list_mutex); 4374 return ret; 4375 4376 out_free_timewait_sock_slab: 4377 if (alloc_slab) 4378 tw_prot_cleanup(prot->twsk_prot); 4379 out_free_request_sock_slab: 4380 if (alloc_slab) { 4381 req_prot_cleanup(prot->rsk_prot); 4382 4383 kmem_cache_destroy(prot->slab); 4384 prot->slab = NULL; 4385 } 4386 out: 4387 return ret; 4388 } 4389 EXPORT_SYMBOL(proto_register); 4390 4391 void proto_unregister(struct proto *prot) 4392 { 4393 mutex_lock(&proto_list_mutex); 4394 release_proto_idx(prot); 4395 list_del(&prot->node); 4396 mutex_unlock(&proto_list_mutex); 4397 4398 kmem_cache_destroy(prot->slab); 4399 prot->slab = NULL; 4400 4401 req_prot_cleanup(prot->rsk_prot); 4402 tw_prot_cleanup(prot->twsk_prot); 4403 } 4404 EXPORT_SYMBOL(proto_unregister); 4405 4406 int sock_load_diag_module(int family, int protocol) 4407 { 4408 if (!protocol) { 4409 if (!sock_is_registered(family)) 4410 return -ENOENT; 4411 4412 return request_module("net-pf-%d-proto-%d-type-%d", PF_NETLINK, 4413 NETLINK_SOCK_DIAG, family); 4414 } 4415 4416 #ifdef CONFIG_INET 4417 if (family == AF_INET && 4418 protocol != IPPROTO_RAW && 4419 protocol < MAX_INET_PROTOS && 4420 !rcu_access_pointer(inet_protos[protocol])) 4421 return -ENOENT; 4422 #endif 4423 4424 return request_module("net-pf-%d-proto-%d-type-%d-%d", PF_NETLINK, 4425 NETLINK_SOCK_DIAG, family, protocol); 4426 } 4427 EXPORT_SYMBOL(sock_load_diag_module); 4428 4429 #ifdef CONFIG_PROC_FS 4430 static void *proto_seq_start(struct seq_file *seq, loff_t *pos) 4431 __acquires(proto_list_mutex) 4432 { 4433 mutex_lock(&proto_list_mutex); 4434 return seq_list_start_head(&proto_list, *pos); 4435 } 4436 4437 static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos) 4438 { 4439 return seq_list_next(v, &proto_list, pos); 4440 } 4441 4442 static void proto_seq_stop(struct seq_file *seq, void *v) 4443 __releases(proto_list_mutex) 4444 { 4445 mutex_unlock(&proto_list_mutex); 4446 } 4447 4448 static char proto_method_implemented(const void *method) 4449 { 4450 return method == NULL ? 'n' : 'y'; 4451 } 4452 static long sock_prot_memory_allocated(struct proto *proto) 4453 { 4454 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L; 4455 } 4456 4457 static const char *sock_prot_memory_pressure(struct proto *proto) 4458 { 4459 return proto->memory_pressure != NULL ? 4460 proto_memory_pressure(proto) ? "yes" : "no" : "NI"; 4461 } 4462 4463 static void proto_seq_printf(struct seq_file *seq, struct proto *proto) 4464 { 4465 4466 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s " 4467 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n", 4468 proto->name, 4469 proto->obj_size, 4470 sock_prot_inuse_get(seq_file_net(seq), proto), 4471 sock_prot_memory_allocated(proto), 4472 sock_prot_memory_pressure(proto), 4473 proto->max_header, 4474 proto->slab == NULL ? "no" : "yes", 4475 module_name(proto->owner), 4476 proto_method_implemented(proto->close), 4477 proto_method_implemented(proto->connect), 4478 proto_method_implemented(proto->disconnect), 4479 proto_method_implemented(proto->accept), 4480 proto_method_implemented(proto->ioctl), 4481 proto_method_implemented(proto->init), 4482 proto_method_implemented(proto->destroy), 4483 proto_method_implemented(proto->shutdown), 4484 proto_method_implemented(proto->setsockopt), 4485 proto_method_implemented(proto->getsockopt), 4486 proto_method_implemented(proto->sendmsg), 4487 proto_method_implemented(proto->recvmsg), 4488 proto_method_implemented(proto->bind), 4489 proto_method_implemented(proto->backlog_rcv), 4490 proto_method_implemented(proto->hash), 4491 proto_method_implemented(proto->unhash), 4492 proto_method_implemented(proto->get_port), 4493 proto_method_implemented(proto->enter_memory_pressure)); 4494 } 4495 4496 static int proto_seq_show(struct seq_file *seq, void *v) 4497 { 4498 if (v == &proto_list) 4499 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s", 4500 "protocol", 4501 "size", 4502 "sockets", 4503 "memory", 4504 "press", 4505 "maxhdr", 4506 "slab", 4507 "module", 4508 "cl co di ac io in de sh ss gs se re bi br ha uh gp em\n"); 4509 else 4510 proto_seq_printf(seq, list_entry(v, struct proto, node)); 4511 return 0; 4512 } 4513 4514 static const struct seq_operations proto_seq_ops = { 4515 .start = proto_seq_start, 4516 .next = proto_seq_next, 4517 .stop = proto_seq_stop, 4518 .show = proto_seq_show, 4519 }; 4520 4521 static __net_init int proto_init_net(struct net *net) 4522 { 4523 if (!proc_create_net("protocols", 0444, net->proc_net, &proto_seq_ops, 4524 sizeof(struct seq_net_private))) 4525 return -ENOMEM; 4526 4527 return 0; 4528 } 4529 4530 static __net_exit void proto_exit_net(struct net *net) 4531 { 4532 remove_proc_entry("protocols", net->proc_net); 4533 } 4534 4535 4536 static __net_initdata struct pernet_operations proto_net_ops = { 4537 .init = proto_init_net, 4538 .exit = proto_exit_net, 4539 }; 4540 4541 static int __init proto_init(void) 4542 { 4543 return register_pernet_subsys(&proto_net_ops); 4544 } 4545 4546 subsys_initcall(proto_init); 4547 4548 #endif /* PROC_FS */ 4549 4550 #ifdef CONFIG_NET_RX_BUSY_POLL 4551 bool sk_busy_loop_end(void *p, unsigned long start_time) 4552 { 4553 struct sock *sk = p; 4554 4555 if (!skb_queue_empty_lockless(&sk->sk_receive_queue)) 4556 return true; 4557 4558 if (sk_is_udp(sk) && 4559 !skb_queue_empty_lockless(&udp_sk(sk)->reader_queue)) 4560 return true; 4561 4562 return sk_busy_loop_timeout(sk, start_time); 4563 } 4564 EXPORT_SYMBOL(sk_busy_loop_end); 4565 #endif /* CONFIG_NET_RX_BUSY_POLL */ 4566 4567 int sock_bind_add(struct sock *sk, struct sockaddr_unsized *addr, int addr_len) 4568 { 4569 if (!sk->sk_prot->bind_add) 4570 return -EOPNOTSUPP; 4571 return sk->sk_prot->bind_add(sk, addr, addr_len); 4572 } 4573 EXPORT_SYMBOL(sock_bind_add); 4574 4575 /* Copy 'size' bytes from userspace and return `size` back to userspace */ 4576 int sock_ioctl_inout(struct sock *sk, unsigned int cmd, 4577 void __user *arg, void *karg, size_t size) 4578 { 4579 int ret; 4580 4581 if (copy_from_user(karg, arg, size)) 4582 return -EFAULT; 4583 4584 ret = READ_ONCE(sk->sk_prot)->ioctl(sk, cmd, karg); 4585 if (ret) 4586 return ret; 4587 4588 if (copy_to_user(arg, karg, size)) 4589 return -EFAULT; 4590 4591 return 0; 4592 } 4593 EXPORT_SYMBOL(sock_ioctl_inout); 4594 4595 /* This is the most common ioctl prep function, where the result (4 bytes) is 4596 * copied back to userspace if the ioctl() returns successfully. No input is 4597 * copied from userspace as input argument. 4598 */ 4599 static int sock_ioctl_out(struct sock *sk, unsigned int cmd, void __user *arg) 4600 { 4601 int ret, karg = 0; 4602 4603 ret = READ_ONCE(sk->sk_prot)->ioctl(sk, cmd, &karg); 4604 if (ret) 4605 return ret; 4606 4607 return put_user(karg, (int __user *)arg); 4608 } 4609 4610 /* A wrapper around sock ioctls, which copies the data from userspace 4611 * (depending on the protocol/ioctl), and copies back the result to userspace. 4612 * The main motivation for this function is to pass kernel memory to the 4613 * protocol ioctl callbacks, instead of userspace memory. 4614 */ 4615 int sk_ioctl(struct sock *sk, unsigned int cmd, void __user *arg) 4616 { 4617 int rc = 1; 4618 4619 if (sk->sk_type == SOCK_RAW && sk->sk_family == AF_INET) 4620 rc = ipmr_sk_ioctl(sk, cmd, arg); 4621 else if (sk->sk_type == SOCK_RAW && sk->sk_family == AF_INET6) 4622 rc = ip6mr_sk_ioctl(sk, cmd, arg); 4623 else if (sk_is_phonet(sk)) 4624 rc = phonet_sk_ioctl(sk, cmd, arg); 4625 4626 /* If ioctl was processed, returns its value */ 4627 if (rc <= 0) 4628 return rc; 4629 4630 /* Otherwise call the default handler */ 4631 return sock_ioctl_out(sk, cmd, arg); 4632 } 4633 EXPORT_SYMBOL(sk_ioctl); 4634 4635 static int __init sock_struct_check(void) 4636 { 4637 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_drops); 4638 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_peek_off); 4639 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_error_queue); 4640 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_receive_queue); 4641 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rx, sk_backlog); 4642 4643 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rx_dst); 4644 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rx_dst_ifindex); 4645 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rx_dst_cookie); 4646 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rcvbuf); 4647 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_filter); 4648 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_wq); 4649 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_data_ready); 4650 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rcvtimeo); 4651 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rx, sk_rcvlowat); 4652 4653 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rxtx, sk_err); 4654 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rxtx, sk_socket); 4655 #ifdef CONFIG_MEMCG 4656 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_rxtx, sk_memcg); 4657 #endif 4658 4659 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_lock); 4660 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_reserved_mem); 4661 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_forward_alloc); 4662 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_rxtx, sk_tsflags); 4663 4664 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_omem_alloc); 4665 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_omem_alloc); 4666 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_err_soft); 4667 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_wmem_queued); 4668 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_wmem_alloc); 4669 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_tsq_flags); 4670 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_send_head); 4671 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_write_queue); 4672 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_write_pending); 4673 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_frag); 4674 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_timer); 4675 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_pacing_rate); 4676 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_zckey); 4677 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_write_tx, sk_tskey); 4678 4679 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_dst_pending_confirm); 4680 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_pacing_status); 4681 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_max_pacing_rate); 4682 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_sndtimeo); 4683 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_priority); 4684 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_mark); 4685 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_uid); 4686 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_protocol); 4687 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_dst_cache); 4688 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_route_caps); 4689 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_gso_type); 4690 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_gso_max_size); 4691 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_allocation); 4692 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_txhash); 4693 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_sndbuf); 4694 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_gso_max_segs); 4695 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_pacing_shift); 4696 CACHELINE_ASSERT_GROUP_MEMBER(struct sock, sock_read_tx, sk_use_task_frag); 4697 return 0; 4698 } 4699 4700 core_initcall(sock_struct_check); 4701