1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * Generic socket support routines. Memory allocators, socket lock/release 8 * handler for protocols to use and generic option handler. 9 * 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Florian La Roche, <flla@stud.uni-sb.de> 13 * Alan Cox, <A.Cox@swansea.ac.uk> 14 * 15 * Fixes: 16 * Alan Cox : Numerous verify_area() problems 17 * Alan Cox : Connecting on a connecting socket 18 * now returns an error for tcp. 19 * Alan Cox : sock->protocol is set correctly. 20 * and is not sometimes left as 0. 21 * Alan Cox : connect handles icmp errors on a 22 * connect properly. Unfortunately there 23 * is a restart syscall nasty there. I 24 * can't match BSD without hacking the C 25 * library. Ideas urgently sought! 26 * Alan Cox : Disallow bind() to addresses that are 27 * not ours - especially broadcast ones!! 28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost) 29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets, 30 * instead they leave that for the DESTROY timer. 31 * Alan Cox : Clean up error flag in accept 32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer 33 * was buggy. Put a remove_sock() in the handler 34 * for memory when we hit 0. Also altered the timer 35 * code. The ACK stuff can wait and needs major 36 * TCP layer surgery. 37 * Alan Cox : Fixed TCP ack bug, removed remove sock 38 * and fixed timer/inet_bh race. 39 * Alan Cox : Added zapped flag for TCP 40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code 41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb 42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources 43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing. 44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so... 45 * Rick Sladkey : Relaxed UDP rules for matching packets. 46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support 47 * Pauline Middelink : identd support 48 * Alan Cox : Fixed connect() taking signals I think. 49 * Alan Cox : SO_LINGER supported 50 * Alan Cox : Error reporting fixes 51 * Anonymous : inet_create tidied up (sk->reuse setting) 52 * Alan Cox : inet sockets don't set sk->type! 53 * Alan Cox : Split socket option code 54 * Alan Cox : Callbacks 55 * Alan Cox : Nagle flag for Charles & Johannes stuff 56 * Alex : Removed restriction on inet fioctl 57 * Alan Cox : Splitting INET from NET core 58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt() 59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code 60 * Alan Cox : Split IP from generic code 61 * Alan Cox : New kfree_skbmem() 62 * Alan Cox : Make SO_DEBUG superuser only. 63 * Alan Cox : Allow anyone to clear SO_DEBUG 64 * (compatibility fix) 65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput. 66 * Alan Cox : Allocator for a socket is settable. 67 * Alan Cox : SO_ERROR includes soft errors. 68 * Alan Cox : Allow NULL arguments on some SO_ opts 69 * Alan Cox : Generic socket allocation to make hooks 70 * easier (suggested by Craig Metz). 71 * Michael Pall : SO_ERROR returns positive errno again 72 * Steve Whitehouse: Added default destructor to free 73 * protocol private data. 74 * Steve Whitehouse: Added various other default routines 75 * common to several socket families. 76 * Chris Evans : Call suser() check last on F_SETOWN 77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER. 78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s() 79 * Andi Kleen : Fix write_space callback 80 * Chris Evans : Security fixes - signedness again 81 * Arnaldo C. Melo : cleanups, use skb_queue_purge 82 * 83 * To Fix: 84 */ 85 86 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 87 88 #include <asm/unaligned.h> 89 #include <linux/capability.h> 90 #include <linux/errno.h> 91 #include <linux/errqueue.h> 92 #include <linux/types.h> 93 #include <linux/socket.h> 94 #include <linux/in.h> 95 #include <linux/kernel.h> 96 #include <linux/module.h> 97 #include <linux/proc_fs.h> 98 #include <linux/seq_file.h> 99 #include <linux/sched.h> 100 #include <linux/sched/mm.h> 101 #include <linux/timer.h> 102 #include <linux/string.h> 103 #include <linux/sockios.h> 104 #include <linux/net.h> 105 #include <linux/mm.h> 106 #include <linux/slab.h> 107 #include <linux/interrupt.h> 108 #include <linux/poll.h> 109 #include <linux/tcp.h> 110 #include <linux/init.h> 111 #include <linux/highmem.h> 112 #include <linux/user_namespace.h> 113 #include <linux/static_key.h> 114 #include <linux/memcontrol.h> 115 #include <linux/prefetch.h> 116 #include <linux/compat.h> 117 118 #include <linux/uaccess.h> 119 120 #include <linux/netdevice.h> 121 #include <net/protocol.h> 122 #include <linux/skbuff.h> 123 #include <net/net_namespace.h> 124 #include <net/request_sock.h> 125 #include <net/sock.h> 126 #include <linux/net_tstamp.h> 127 #include <net/xfrm.h> 128 #include <linux/ipsec.h> 129 #include <net/cls_cgroup.h> 130 #include <net/netprio_cgroup.h> 131 #include <linux/sock_diag.h> 132 133 #include <linux/filter.h> 134 #include <net/sock_reuseport.h> 135 #include <net/bpf_sk_storage.h> 136 137 #include <trace/events/sock.h> 138 139 #include <net/tcp.h> 140 #include <net/busy_poll.h> 141 142 static DEFINE_MUTEX(proto_list_mutex); 143 static LIST_HEAD(proto_list); 144 145 static void sock_inuse_add(struct net *net, int val); 146 147 /** 148 * sk_ns_capable - General socket capability test 149 * @sk: Socket to use a capability on or through 150 * @user_ns: The user namespace of the capability to use 151 * @cap: The capability to use 152 * 153 * Test to see if the opener of the socket had when the socket was 154 * created and the current process has the capability @cap in the user 155 * namespace @user_ns. 156 */ 157 bool sk_ns_capable(const struct sock *sk, 158 struct user_namespace *user_ns, int cap) 159 { 160 return file_ns_capable(sk->sk_socket->file, user_ns, cap) && 161 ns_capable(user_ns, cap); 162 } 163 EXPORT_SYMBOL(sk_ns_capable); 164 165 /** 166 * sk_capable - Socket global capability test 167 * @sk: Socket to use a capability on or through 168 * @cap: The global capability to use 169 * 170 * Test to see if the opener of the socket had when the socket was 171 * created and the current process has the capability @cap in all user 172 * namespaces. 173 */ 174 bool sk_capable(const struct sock *sk, int cap) 175 { 176 return sk_ns_capable(sk, &init_user_ns, cap); 177 } 178 EXPORT_SYMBOL(sk_capable); 179 180 /** 181 * sk_net_capable - Network namespace socket capability test 182 * @sk: Socket to use a capability on or through 183 * @cap: The capability to use 184 * 185 * Test to see if the opener of the socket had when the socket was created 186 * and the current process has the capability @cap over the network namespace 187 * the socket is a member of. 188 */ 189 bool sk_net_capable(const struct sock *sk, int cap) 190 { 191 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap); 192 } 193 EXPORT_SYMBOL(sk_net_capable); 194 195 /* 196 * Each address family might have different locking rules, so we have 197 * one slock key per address family and separate keys for internal and 198 * userspace sockets. 199 */ 200 static struct lock_class_key af_family_keys[AF_MAX]; 201 static struct lock_class_key af_family_kern_keys[AF_MAX]; 202 static struct lock_class_key af_family_slock_keys[AF_MAX]; 203 static struct lock_class_key af_family_kern_slock_keys[AF_MAX]; 204 205 /* 206 * Make lock validator output more readable. (we pre-construct these 207 * strings build-time, so that runtime initialization of socket 208 * locks is fast): 209 */ 210 211 #define _sock_locks(x) \ 212 x "AF_UNSPEC", x "AF_UNIX" , x "AF_INET" , \ 213 x "AF_AX25" , x "AF_IPX" , x "AF_APPLETALK", \ 214 x "AF_NETROM", x "AF_BRIDGE" , x "AF_ATMPVC" , \ 215 x "AF_X25" , x "AF_INET6" , x "AF_ROSE" , \ 216 x "AF_DECnet", x "AF_NETBEUI" , x "AF_SECURITY" , \ 217 x "AF_KEY" , x "AF_NETLINK" , x "AF_PACKET" , \ 218 x "AF_ASH" , x "AF_ECONET" , x "AF_ATMSVC" , \ 219 x "AF_RDS" , x "AF_SNA" , x "AF_IRDA" , \ 220 x "AF_PPPOX" , x "AF_WANPIPE" , x "AF_LLC" , \ 221 x "27" , x "28" , x "AF_CAN" , \ 222 x "AF_TIPC" , x "AF_BLUETOOTH", x "IUCV" , \ 223 x "AF_RXRPC" , x "AF_ISDN" , x "AF_PHONET" , \ 224 x "AF_IEEE802154", x "AF_CAIF" , x "AF_ALG" , \ 225 x "AF_NFC" , x "AF_VSOCK" , x "AF_KCM" , \ 226 x "AF_QIPCRTR", x "AF_SMC" , x "AF_XDP" , \ 227 x "AF_MAX" 228 229 static const char *const af_family_key_strings[AF_MAX+1] = { 230 _sock_locks("sk_lock-") 231 }; 232 static const char *const af_family_slock_key_strings[AF_MAX+1] = { 233 _sock_locks("slock-") 234 }; 235 static const char *const af_family_clock_key_strings[AF_MAX+1] = { 236 _sock_locks("clock-") 237 }; 238 239 static const char *const af_family_kern_key_strings[AF_MAX+1] = { 240 _sock_locks("k-sk_lock-") 241 }; 242 static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = { 243 _sock_locks("k-slock-") 244 }; 245 static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = { 246 _sock_locks("k-clock-") 247 }; 248 static const char *const af_family_rlock_key_strings[AF_MAX+1] = { 249 _sock_locks("rlock-") 250 }; 251 static const char *const af_family_wlock_key_strings[AF_MAX+1] = { 252 _sock_locks("wlock-") 253 }; 254 static const char *const af_family_elock_key_strings[AF_MAX+1] = { 255 _sock_locks("elock-") 256 }; 257 258 /* 259 * sk_callback_lock and sk queues locking rules are per-address-family, 260 * so split the lock classes by using a per-AF key: 261 */ 262 static struct lock_class_key af_callback_keys[AF_MAX]; 263 static struct lock_class_key af_rlock_keys[AF_MAX]; 264 static struct lock_class_key af_wlock_keys[AF_MAX]; 265 static struct lock_class_key af_elock_keys[AF_MAX]; 266 static struct lock_class_key af_kern_callback_keys[AF_MAX]; 267 268 /* Run time adjustable parameters. */ 269 __u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX; 270 EXPORT_SYMBOL(sysctl_wmem_max); 271 __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX; 272 EXPORT_SYMBOL(sysctl_rmem_max); 273 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX; 274 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX; 275 276 /* Maximal space eaten by iovec or ancillary data plus some space */ 277 int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512); 278 EXPORT_SYMBOL(sysctl_optmem_max); 279 280 int sysctl_tstamp_allow_data __read_mostly = 1; 281 282 DEFINE_STATIC_KEY_FALSE(memalloc_socks_key); 283 EXPORT_SYMBOL_GPL(memalloc_socks_key); 284 285 /** 286 * sk_set_memalloc - sets %SOCK_MEMALLOC 287 * @sk: socket to set it on 288 * 289 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves. 290 * It's the responsibility of the admin to adjust min_free_kbytes 291 * to meet the requirements 292 */ 293 void sk_set_memalloc(struct sock *sk) 294 { 295 sock_set_flag(sk, SOCK_MEMALLOC); 296 sk->sk_allocation |= __GFP_MEMALLOC; 297 static_branch_inc(&memalloc_socks_key); 298 } 299 EXPORT_SYMBOL_GPL(sk_set_memalloc); 300 301 void sk_clear_memalloc(struct sock *sk) 302 { 303 sock_reset_flag(sk, SOCK_MEMALLOC); 304 sk->sk_allocation &= ~__GFP_MEMALLOC; 305 static_branch_dec(&memalloc_socks_key); 306 307 /* 308 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward 309 * progress of swapping. SOCK_MEMALLOC may be cleared while 310 * it has rmem allocations due to the last swapfile being deactivated 311 * but there is a risk that the socket is unusable due to exceeding 312 * the rmem limits. Reclaim the reserves and obey rmem limits again. 313 */ 314 sk_mem_reclaim(sk); 315 } 316 EXPORT_SYMBOL_GPL(sk_clear_memalloc); 317 318 int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb) 319 { 320 int ret; 321 unsigned int noreclaim_flag; 322 323 /* these should have been dropped before queueing */ 324 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC)); 325 326 noreclaim_flag = memalloc_noreclaim_save(); 327 ret = sk->sk_backlog_rcv(sk, skb); 328 memalloc_noreclaim_restore(noreclaim_flag); 329 330 return ret; 331 } 332 EXPORT_SYMBOL(__sk_backlog_rcv); 333 334 static int sock_get_timeout(long timeo, void *optval, bool old_timeval) 335 { 336 struct __kernel_sock_timeval tv; 337 338 if (timeo == MAX_SCHEDULE_TIMEOUT) { 339 tv.tv_sec = 0; 340 tv.tv_usec = 0; 341 } else { 342 tv.tv_sec = timeo / HZ; 343 tv.tv_usec = ((timeo % HZ) * USEC_PER_SEC) / HZ; 344 } 345 346 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) { 347 struct old_timeval32 tv32 = { tv.tv_sec, tv.tv_usec }; 348 *(struct old_timeval32 *)optval = tv32; 349 return sizeof(tv32); 350 } 351 352 if (old_timeval) { 353 struct __kernel_old_timeval old_tv; 354 old_tv.tv_sec = tv.tv_sec; 355 old_tv.tv_usec = tv.tv_usec; 356 *(struct __kernel_old_timeval *)optval = old_tv; 357 return sizeof(old_tv); 358 } 359 360 *(struct __kernel_sock_timeval *)optval = tv; 361 return sizeof(tv); 362 } 363 364 static int sock_set_timeout(long *timeo_p, sockptr_t optval, int optlen, 365 bool old_timeval) 366 { 367 struct __kernel_sock_timeval tv; 368 369 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) { 370 struct old_timeval32 tv32; 371 372 if (optlen < sizeof(tv32)) 373 return -EINVAL; 374 375 if (copy_from_sockptr(&tv32, optval, sizeof(tv32))) 376 return -EFAULT; 377 tv.tv_sec = tv32.tv_sec; 378 tv.tv_usec = tv32.tv_usec; 379 } else if (old_timeval) { 380 struct __kernel_old_timeval old_tv; 381 382 if (optlen < sizeof(old_tv)) 383 return -EINVAL; 384 if (copy_from_sockptr(&old_tv, optval, sizeof(old_tv))) 385 return -EFAULT; 386 tv.tv_sec = old_tv.tv_sec; 387 tv.tv_usec = old_tv.tv_usec; 388 } else { 389 if (optlen < sizeof(tv)) 390 return -EINVAL; 391 if (copy_from_sockptr(&tv, optval, sizeof(tv))) 392 return -EFAULT; 393 } 394 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC) 395 return -EDOM; 396 397 if (tv.tv_sec < 0) { 398 static int warned __read_mostly; 399 400 *timeo_p = 0; 401 if (warned < 10 && net_ratelimit()) { 402 warned++; 403 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n", 404 __func__, current->comm, task_pid_nr(current)); 405 } 406 return 0; 407 } 408 *timeo_p = MAX_SCHEDULE_TIMEOUT; 409 if (tv.tv_sec == 0 && tv.tv_usec == 0) 410 return 0; 411 if (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT / HZ - 1)) 412 *timeo_p = tv.tv_sec * HZ + DIV_ROUND_UP((unsigned long)tv.tv_usec, USEC_PER_SEC / HZ); 413 return 0; 414 } 415 416 static void sock_warn_obsolete_bsdism(const char *name) 417 { 418 static int warned; 419 static char warncomm[TASK_COMM_LEN]; 420 if (strcmp(warncomm, current->comm) && warned < 5) { 421 strcpy(warncomm, current->comm); 422 pr_warn("process `%s' is using obsolete %s SO_BSDCOMPAT\n", 423 warncomm, name); 424 warned++; 425 } 426 } 427 428 static bool sock_needs_netstamp(const struct sock *sk) 429 { 430 switch (sk->sk_family) { 431 case AF_UNSPEC: 432 case AF_UNIX: 433 return false; 434 default: 435 return true; 436 } 437 } 438 439 static void sock_disable_timestamp(struct sock *sk, unsigned long flags) 440 { 441 if (sk->sk_flags & flags) { 442 sk->sk_flags &= ~flags; 443 if (sock_needs_netstamp(sk) && 444 !(sk->sk_flags & SK_FLAGS_TIMESTAMP)) 445 net_disable_timestamp(); 446 } 447 } 448 449 450 int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 451 { 452 unsigned long flags; 453 struct sk_buff_head *list = &sk->sk_receive_queue; 454 455 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) { 456 atomic_inc(&sk->sk_drops); 457 trace_sock_rcvqueue_full(sk, skb); 458 return -ENOMEM; 459 } 460 461 if (!sk_rmem_schedule(sk, skb, skb->truesize)) { 462 atomic_inc(&sk->sk_drops); 463 return -ENOBUFS; 464 } 465 466 skb->dev = NULL; 467 skb_set_owner_r(skb, sk); 468 469 /* we escape from rcu protected region, make sure we dont leak 470 * a norefcounted dst 471 */ 472 skb_dst_force(skb); 473 474 spin_lock_irqsave(&list->lock, flags); 475 sock_skb_set_dropcount(sk, skb); 476 __skb_queue_tail(list, skb); 477 spin_unlock_irqrestore(&list->lock, flags); 478 479 if (!sock_flag(sk, SOCK_DEAD)) 480 sk->sk_data_ready(sk); 481 return 0; 482 } 483 EXPORT_SYMBOL(__sock_queue_rcv_skb); 484 485 int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 486 { 487 int err; 488 489 err = sk_filter(sk, skb); 490 if (err) 491 return err; 492 493 return __sock_queue_rcv_skb(sk, skb); 494 } 495 EXPORT_SYMBOL(sock_queue_rcv_skb); 496 497 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, 498 const int nested, unsigned int trim_cap, bool refcounted) 499 { 500 int rc = NET_RX_SUCCESS; 501 502 if (sk_filter_trim_cap(sk, skb, trim_cap)) 503 goto discard_and_relse; 504 505 skb->dev = NULL; 506 507 if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) { 508 atomic_inc(&sk->sk_drops); 509 goto discard_and_relse; 510 } 511 if (nested) 512 bh_lock_sock_nested(sk); 513 else 514 bh_lock_sock(sk); 515 if (!sock_owned_by_user(sk)) { 516 /* 517 * trylock + unlock semantics: 518 */ 519 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_); 520 521 rc = sk_backlog_rcv(sk, skb); 522 523 mutex_release(&sk->sk_lock.dep_map, _RET_IP_); 524 } else if (sk_add_backlog(sk, skb, READ_ONCE(sk->sk_rcvbuf))) { 525 bh_unlock_sock(sk); 526 atomic_inc(&sk->sk_drops); 527 goto discard_and_relse; 528 } 529 530 bh_unlock_sock(sk); 531 out: 532 if (refcounted) 533 sock_put(sk); 534 return rc; 535 discard_and_relse: 536 kfree_skb(skb); 537 goto out; 538 } 539 EXPORT_SYMBOL(__sk_receive_skb); 540 541 struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie) 542 { 543 struct dst_entry *dst = __sk_dst_get(sk); 544 545 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) { 546 sk_tx_queue_clear(sk); 547 sk->sk_dst_pending_confirm = 0; 548 RCU_INIT_POINTER(sk->sk_dst_cache, NULL); 549 dst_release(dst); 550 return NULL; 551 } 552 553 return dst; 554 } 555 EXPORT_SYMBOL(__sk_dst_check); 556 557 struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie) 558 { 559 struct dst_entry *dst = sk_dst_get(sk); 560 561 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) { 562 sk_dst_reset(sk); 563 dst_release(dst); 564 return NULL; 565 } 566 567 return dst; 568 } 569 EXPORT_SYMBOL(sk_dst_check); 570 571 static int sock_bindtoindex_locked(struct sock *sk, int ifindex) 572 { 573 int ret = -ENOPROTOOPT; 574 #ifdef CONFIG_NETDEVICES 575 struct net *net = sock_net(sk); 576 577 /* Sorry... */ 578 ret = -EPERM; 579 if (sk->sk_bound_dev_if && !ns_capable(net->user_ns, CAP_NET_RAW)) 580 goto out; 581 582 ret = -EINVAL; 583 if (ifindex < 0) 584 goto out; 585 586 sk->sk_bound_dev_if = ifindex; 587 if (sk->sk_prot->rehash) 588 sk->sk_prot->rehash(sk); 589 sk_dst_reset(sk); 590 591 ret = 0; 592 593 out: 594 #endif 595 596 return ret; 597 } 598 599 int sock_bindtoindex(struct sock *sk, int ifindex, bool lock_sk) 600 { 601 int ret; 602 603 if (lock_sk) 604 lock_sock(sk); 605 ret = sock_bindtoindex_locked(sk, ifindex); 606 if (lock_sk) 607 release_sock(sk); 608 609 return ret; 610 } 611 EXPORT_SYMBOL(sock_bindtoindex); 612 613 static int sock_setbindtodevice(struct sock *sk, sockptr_t optval, int optlen) 614 { 615 int ret = -ENOPROTOOPT; 616 #ifdef CONFIG_NETDEVICES 617 struct net *net = sock_net(sk); 618 char devname[IFNAMSIZ]; 619 int index; 620 621 ret = -EINVAL; 622 if (optlen < 0) 623 goto out; 624 625 /* Bind this socket to a particular device like "eth0", 626 * as specified in the passed interface name. If the 627 * name is "" or the option length is zero the socket 628 * is not bound. 629 */ 630 if (optlen > IFNAMSIZ - 1) 631 optlen = IFNAMSIZ - 1; 632 memset(devname, 0, sizeof(devname)); 633 634 ret = -EFAULT; 635 if (copy_from_sockptr(devname, optval, optlen)) 636 goto out; 637 638 index = 0; 639 if (devname[0] != '\0') { 640 struct net_device *dev; 641 642 rcu_read_lock(); 643 dev = dev_get_by_name_rcu(net, devname); 644 if (dev) 645 index = dev->ifindex; 646 rcu_read_unlock(); 647 ret = -ENODEV; 648 if (!dev) 649 goto out; 650 } 651 652 return sock_bindtoindex(sk, index, true); 653 out: 654 #endif 655 656 return ret; 657 } 658 659 static int sock_getbindtodevice(struct sock *sk, char __user *optval, 660 int __user *optlen, int len) 661 { 662 int ret = -ENOPROTOOPT; 663 #ifdef CONFIG_NETDEVICES 664 struct net *net = sock_net(sk); 665 char devname[IFNAMSIZ]; 666 667 if (sk->sk_bound_dev_if == 0) { 668 len = 0; 669 goto zero; 670 } 671 672 ret = -EINVAL; 673 if (len < IFNAMSIZ) 674 goto out; 675 676 ret = netdev_get_name(net, devname, sk->sk_bound_dev_if); 677 if (ret) 678 goto out; 679 680 len = strlen(devname) + 1; 681 682 ret = -EFAULT; 683 if (copy_to_user(optval, devname, len)) 684 goto out; 685 686 zero: 687 ret = -EFAULT; 688 if (put_user(len, optlen)) 689 goto out; 690 691 ret = 0; 692 693 out: 694 #endif 695 696 return ret; 697 } 698 699 bool sk_mc_loop(struct sock *sk) 700 { 701 if (dev_recursion_level()) 702 return false; 703 if (!sk) 704 return true; 705 switch (sk->sk_family) { 706 case AF_INET: 707 return inet_sk(sk)->mc_loop; 708 #if IS_ENABLED(CONFIG_IPV6) 709 case AF_INET6: 710 return inet6_sk(sk)->mc_loop; 711 #endif 712 } 713 WARN_ON_ONCE(1); 714 return true; 715 } 716 EXPORT_SYMBOL(sk_mc_loop); 717 718 void sock_set_reuseaddr(struct sock *sk) 719 { 720 lock_sock(sk); 721 sk->sk_reuse = SK_CAN_REUSE; 722 release_sock(sk); 723 } 724 EXPORT_SYMBOL(sock_set_reuseaddr); 725 726 void sock_set_reuseport(struct sock *sk) 727 { 728 lock_sock(sk); 729 sk->sk_reuseport = true; 730 release_sock(sk); 731 } 732 EXPORT_SYMBOL(sock_set_reuseport); 733 734 void sock_no_linger(struct sock *sk) 735 { 736 lock_sock(sk); 737 sk->sk_lingertime = 0; 738 sock_set_flag(sk, SOCK_LINGER); 739 release_sock(sk); 740 } 741 EXPORT_SYMBOL(sock_no_linger); 742 743 void sock_set_priority(struct sock *sk, u32 priority) 744 { 745 lock_sock(sk); 746 sk->sk_priority = priority; 747 release_sock(sk); 748 } 749 EXPORT_SYMBOL(sock_set_priority); 750 751 void sock_set_sndtimeo(struct sock *sk, s64 secs) 752 { 753 lock_sock(sk); 754 if (secs && secs < MAX_SCHEDULE_TIMEOUT / HZ - 1) 755 sk->sk_sndtimeo = secs * HZ; 756 else 757 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT; 758 release_sock(sk); 759 } 760 EXPORT_SYMBOL(sock_set_sndtimeo); 761 762 static void __sock_set_timestamps(struct sock *sk, bool val, bool new, bool ns) 763 { 764 if (val) { 765 sock_valbool_flag(sk, SOCK_TSTAMP_NEW, new); 766 sock_valbool_flag(sk, SOCK_RCVTSTAMPNS, ns); 767 sock_set_flag(sk, SOCK_RCVTSTAMP); 768 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 769 } else { 770 sock_reset_flag(sk, SOCK_RCVTSTAMP); 771 sock_reset_flag(sk, SOCK_RCVTSTAMPNS); 772 sock_reset_flag(sk, SOCK_TSTAMP_NEW); 773 } 774 } 775 776 void sock_enable_timestamps(struct sock *sk) 777 { 778 lock_sock(sk); 779 __sock_set_timestamps(sk, true, false, true); 780 release_sock(sk); 781 } 782 EXPORT_SYMBOL(sock_enable_timestamps); 783 784 void sock_set_keepalive(struct sock *sk) 785 { 786 lock_sock(sk); 787 if (sk->sk_prot->keepalive) 788 sk->sk_prot->keepalive(sk, true); 789 sock_valbool_flag(sk, SOCK_KEEPOPEN, true); 790 release_sock(sk); 791 } 792 EXPORT_SYMBOL(sock_set_keepalive); 793 794 static void __sock_set_rcvbuf(struct sock *sk, int val) 795 { 796 /* Ensure val * 2 fits into an int, to prevent max_t() from treating it 797 * as a negative value. 798 */ 799 val = min_t(int, val, INT_MAX / 2); 800 sk->sk_userlocks |= SOCK_RCVBUF_LOCK; 801 802 /* We double it on the way in to account for "struct sk_buff" etc. 803 * overhead. Applications assume that the SO_RCVBUF setting they make 804 * will allow that much actual data to be received on that socket. 805 * 806 * Applications are unaware that "struct sk_buff" and other overheads 807 * allocate from the receive buffer during socket buffer allocation. 808 * 809 * And after considering the possible alternatives, returning the value 810 * we actually used in getsockopt is the most desirable behavior. 811 */ 812 WRITE_ONCE(sk->sk_rcvbuf, max_t(int, val * 2, SOCK_MIN_RCVBUF)); 813 } 814 815 void sock_set_rcvbuf(struct sock *sk, int val) 816 { 817 lock_sock(sk); 818 __sock_set_rcvbuf(sk, val); 819 release_sock(sk); 820 } 821 EXPORT_SYMBOL(sock_set_rcvbuf); 822 823 /* 824 * This is meant for all protocols to use and covers goings on 825 * at the socket level. Everything here is generic. 826 */ 827 828 int sock_setsockopt(struct socket *sock, int level, int optname, 829 sockptr_t optval, unsigned int optlen) 830 { 831 struct sock_txtime sk_txtime; 832 struct sock *sk = sock->sk; 833 int val; 834 int valbool; 835 struct linger ling; 836 int ret = 0; 837 838 /* 839 * Options without arguments 840 */ 841 842 if (optname == SO_BINDTODEVICE) 843 return sock_setbindtodevice(sk, optval, optlen); 844 845 if (optlen < sizeof(int)) 846 return -EINVAL; 847 848 if (copy_from_sockptr(&val, optval, sizeof(val))) 849 return -EFAULT; 850 851 valbool = val ? 1 : 0; 852 853 lock_sock(sk); 854 855 switch (optname) { 856 case SO_DEBUG: 857 if (val && !capable(CAP_NET_ADMIN)) 858 ret = -EACCES; 859 else 860 sock_valbool_flag(sk, SOCK_DBG, valbool); 861 break; 862 case SO_REUSEADDR: 863 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE); 864 break; 865 case SO_REUSEPORT: 866 sk->sk_reuseport = valbool; 867 break; 868 case SO_TYPE: 869 case SO_PROTOCOL: 870 case SO_DOMAIN: 871 case SO_ERROR: 872 ret = -ENOPROTOOPT; 873 break; 874 case SO_DONTROUTE: 875 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool); 876 sk_dst_reset(sk); 877 break; 878 case SO_BROADCAST: 879 sock_valbool_flag(sk, SOCK_BROADCAST, valbool); 880 break; 881 case SO_SNDBUF: 882 /* Don't error on this BSD doesn't and if you think 883 * about it this is right. Otherwise apps have to 884 * play 'guess the biggest size' games. RCVBUF/SNDBUF 885 * are treated in BSD as hints 886 */ 887 val = min_t(u32, val, sysctl_wmem_max); 888 set_sndbuf: 889 /* Ensure val * 2 fits into an int, to prevent max_t() 890 * from treating it as a negative value. 891 */ 892 val = min_t(int, val, INT_MAX / 2); 893 sk->sk_userlocks |= SOCK_SNDBUF_LOCK; 894 WRITE_ONCE(sk->sk_sndbuf, 895 max_t(int, val * 2, SOCK_MIN_SNDBUF)); 896 /* Wake up sending tasks if we upped the value. */ 897 sk->sk_write_space(sk); 898 break; 899 900 case SO_SNDBUFFORCE: 901 if (!capable(CAP_NET_ADMIN)) { 902 ret = -EPERM; 903 break; 904 } 905 906 /* No negative values (to prevent underflow, as val will be 907 * multiplied by 2). 908 */ 909 if (val < 0) 910 val = 0; 911 goto set_sndbuf; 912 913 case SO_RCVBUF: 914 /* Don't error on this BSD doesn't and if you think 915 * about it this is right. Otherwise apps have to 916 * play 'guess the biggest size' games. RCVBUF/SNDBUF 917 * are treated in BSD as hints 918 */ 919 __sock_set_rcvbuf(sk, min_t(u32, val, sysctl_rmem_max)); 920 break; 921 922 case SO_RCVBUFFORCE: 923 if (!capable(CAP_NET_ADMIN)) { 924 ret = -EPERM; 925 break; 926 } 927 928 /* No negative values (to prevent underflow, as val will be 929 * multiplied by 2). 930 */ 931 __sock_set_rcvbuf(sk, max(val, 0)); 932 break; 933 934 case SO_KEEPALIVE: 935 if (sk->sk_prot->keepalive) 936 sk->sk_prot->keepalive(sk, valbool); 937 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool); 938 break; 939 940 case SO_OOBINLINE: 941 sock_valbool_flag(sk, SOCK_URGINLINE, valbool); 942 break; 943 944 case SO_NO_CHECK: 945 sk->sk_no_check_tx = valbool; 946 break; 947 948 case SO_PRIORITY: 949 if ((val >= 0 && val <= 6) || 950 ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 951 sk->sk_priority = val; 952 else 953 ret = -EPERM; 954 break; 955 956 case SO_LINGER: 957 if (optlen < sizeof(ling)) { 958 ret = -EINVAL; /* 1003.1g */ 959 break; 960 } 961 if (copy_from_sockptr(&ling, optval, sizeof(ling))) { 962 ret = -EFAULT; 963 break; 964 } 965 if (!ling.l_onoff) 966 sock_reset_flag(sk, SOCK_LINGER); 967 else { 968 #if (BITS_PER_LONG == 32) 969 if ((unsigned int)ling.l_linger >= MAX_SCHEDULE_TIMEOUT/HZ) 970 sk->sk_lingertime = MAX_SCHEDULE_TIMEOUT; 971 else 972 #endif 973 sk->sk_lingertime = (unsigned int)ling.l_linger * HZ; 974 sock_set_flag(sk, SOCK_LINGER); 975 } 976 break; 977 978 case SO_BSDCOMPAT: 979 sock_warn_obsolete_bsdism("setsockopt"); 980 break; 981 982 case SO_PASSCRED: 983 if (valbool) 984 set_bit(SOCK_PASSCRED, &sock->flags); 985 else 986 clear_bit(SOCK_PASSCRED, &sock->flags); 987 break; 988 989 case SO_TIMESTAMP_OLD: 990 __sock_set_timestamps(sk, valbool, false, false); 991 break; 992 case SO_TIMESTAMP_NEW: 993 __sock_set_timestamps(sk, valbool, true, false); 994 break; 995 case SO_TIMESTAMPNS_OLD: 996 __sock_set_timestamps(sk, valbool, false, true); 997 break; 998 case SO_TIMESTAMPNS_NEW: 999 __sock_set_timestamps(sk, valbool, true, true); 1000 break; 1001 case SO_TIMESTAMPING_NEW: 1002 sock_set_flag(sk, SOCK_TSTAMP_NEW); 1003 /* fall through */ 1004 case SO_TIMESTAMPING_OLD: 1005 if (val & ~SOF_TIMESTAMPING_MASK) { 1006 ret = -EINVAL; 1007 break; 1008 } 1009 1010 if (val & SOF_TIMESTAMPING_OPT_ID && 1011 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) { 1012 if (sk->sk_protocol == IPPROTO_TCP && 1013 sk->sk_type == SOCK_STREAM) { 1014 if ((1 << sk->sk_state) & 1015 (TCPF_CLOSE | TCPF_LISTEN)) { 1016 ret = -EINVAL; 1017 break; 1018 } 1019 sk->sk_tskey = tcp_sk(sk)->snd_una; 1020 } else { 1021 sk->sk_tskey = 0; 1022 } 1023 } 1024 1025 if (val & SOF_TIMESTAMPING_OPT_STATS && 1026 !(val & SOF_TIMESTAMPING_OPT_TSONLY)) { 1027 ret = -EINVAL; 1028 break; 1029 } 1030 1031 sk->sk_tsflags = val; 1032 if (val & SOF_TIMESTAMPING_RX_SOFTWARE) 1033 sock_enable_timestamp(sk, 1034 SOCK_TIMESTAMPING_RX_SOFTWARE); 1035 else { 1036 if (optname == SO_TIMESTAMPING_NEW) 1037 sock_reset_flag(sk, SOCK_TSTAMP_NEW); 1038 1039 sock_disable_timestamp(sk, 1040 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE)); 1041 } 1042 break; 1043 1044 case SO_RCVLOWAT: 1045 if (val < 0) 1046 val = INT_MAX; 1047 if (sock->ops->set_rcvlowat) 1048 ret = sock->ops->set_rcvlowat(sk, val); 1049 else 1050 WRITE_ONCE(sk->sk_rcvlowat, val ? : 1); 1051 break; 1052 1053 case SO_RCVTIMEO_OLD: 1054 case SO_RCVTIMEO_NEW: 1055 ret = sock_set_timeout(&sk->sk_rcvtimeo, optval, 1056 optlen, optname == SO_RCVTIMEO_OLD); 1057 break; 1058 1059 case SO_SNDTIMEO_OLD: 1060 case SO_SNDTIMEO_NEW: 1061 ret = sock_set_timeout(&sk->sk_sndtimeo, optval, 1062 optlen, optname == SO_SNDTIMEO_OLD); 1063 break; 1064 1065 case SO_ATTACH_FILTER: { 1066 struct sock_fprog fprog; 1067 1068 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen); 1069 if (!ret) 1070 ret = sk_attach_filter(&fprog, sk); 1071 break; 1072 } 1073 case SO_ATTACH_BPF: 1074 ret = -EINVAL; 1075 if (optlen == sizeof(u32)) { 1076 u32 ufd; 1077 1078 ret = -EFAULT; 1079 if (copy_from_sockptr(&ufd, optval, sizeof(ufd))) 1080 break; 1081 1082 ret = sk_attach_bpf(ufd, sk); 1083 } 1084 break; 1085 1086 case SO_ATTACH_REUSEPORT_CBPF: { 1087 struct sock_fprog fprog; 1088 1089 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen); 1090 if (!ret) 1091 ret = sk_reuseport_attach_filter(&fprog, sk); 1092 break; 1093 } 1094 case SO_ATTACH_REUSEPORT_EBPF: 1095 ret = -EINVAL; 1096 if (optlen == sizeof(u32)) { 1097 u32 ufd; 1098 1099 ret = -EFAULT; 1100 if (copy_from_sockptr(&ufd, optval, sizeof(ufd))) 1101 break; 1102 1103 ret = sk_reuseport_attach_bpf(ufd, sk); 1104 } 1105 break; 1106 1107 case SO_DETACH_REUSEPORT_BPF: 1108 ret = reuseport_detach_prog(sk); 1109 break; 1110 1111 case SO_DETACH_FILTER: 1112 ret = sk_detach_filter(sk); 1113 break; 1114 1115 case SO_LOCK_FILTER: 1116 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool) 1117 ret = -EPERM; 1118 else 1119 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool); 1120 break; 1121 1122 case SO_PASSSEC: 1123 if (valbool) 1124 set_bit(SOCK_PASSSEC, &sock->flags); 1125 else 1126 clear_bit(SOCK_PASSSEC, &sock->flags); 1127 break; 1128 case SO_MARK: 1129 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1130 ret = -EPERM; 1131 } else if (val != sk->sk_mark) { 1132 sk->sk_mark = val; 1133 sk_dst_reset(sk); 1134 } 1135 break; 1136 1137 case SO_RXQ_OVFL: 1138 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool); 1139 break; 1140 1141 case SO_WIFI_STATUS: 1142 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool); 1143 break; 1144 1145 case SO_PEEK_OFF: 1146 if (sock->ops->set_peek_off) 1147 ret = sock->ops->set_peek_off(sk, val); 1148 else 1149 ret = -EOPNOTSUPP; 1150 break; 1151 1152 case SO_NOFCS: 1153 sock_valbool_flag(sk, SOCK_NOFCS, valbool); 1154 break; 1155 1156 case SO_SELECT_ERR_QUEUE: 1157 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool); 1158 break; 1159 1160 #ifdef CONFIG_NET_RX_BUSY_POLL 1161 case SO_BUSY_POLL: 1162 /* allow unprivileged users to decrease the value */ 1163 if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN)) 1164 ret = -EPERM; 1165 else { 1166 if (val < 0) 1167 ret = -EINVAL; 1168 else 1169 sk->sk_ll_usec = val; 1170 } 1171 break; 1172 #endif 1173 1174 case SO_MAX_PACING_RATE: 1175 { 1176 unsigned long ulval = (val == ~0U) ? ~0UL : val; 1177 1178 if (sizeof(ulval) != sizeof(val) && 1179 optlen >= sizeof(ulval) && 1180 copy_from_sockptr(&ulval, optval, sizeof(ulval))) { 1181 ret = -EFAULT; 1182 break; 1183 } 1184 if (ulval != ~0UL) 1185 cmpxchg(&sk->sk_pacing_status, 1186 SK_PACING_NONE, 1187 SK_PACING_NEEDED); 1188 sk->sk_max_pacing_rate = ulval; 1189 sk->sk_pacing_rate = min(sk->sk_pacing_rate, ulval); 1190 break; 1191 } 1192 case SO_INCOMING_CPU: 1193 WRITE_ONCE(sk->sk_incoming_cpu, val); 1194 break; 1195 1196 case SO_CNX_ADVICE: 1197 if (val == 1) 1198 dst_negative_advice(sk); 1199 break; 1200 1201 case SO_ZEROCOPY: 1202 if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) { 1203 if (!((sk->sk_type == SOCK_STREAM && 1204 sk->sk_protocol == IPPROTO_TCP) || 1205 (sk->sk_type == SOCK_DGRAM && 1206 sk->sk_protocol == IPPROTO_UDP))) 1207 ret = -ENOTSUPP; 1208 } else if (sk->sk_family != PF_RDS) { 1209 ret = -ENOTSUPP; 1210 } 1211 if (!ret) { 1212 if (val < 0 || val > 1) 1213 ret = -EINVAL; 1214 else 1215 sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool); 1216 } 1217 break; 1218 1219 case SO_TXTIME: 1220 if (optlen != sizeof(struct sock_txtime)) { 1221 ret = -EINVAL; 1222 break; 1223 } else if (copy_from_sockptr(&sk_txtime, optval, 1224 sizeof(struct sock_txtime))) { 1225 ret = -EFAULT; 1226 break; 1227 } else if (sk_txtime.flags & ~SOF_TXTIME_FLAGS_MASK) { 1228 ret = -EINVAL; 1229 break; 1230 } 1231 /* CLOCK_MONOTONIC is only used by sch_fq, and this packet 1232 * scheduler has enough safe guards. 1233 */ 1234 if (sk_txtime.clockid != CLOCK_MONOTONIC && 1235 !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1236 ret = -EPERM; 1237 break; 1238 } 1239 sock_valbool_flag(sk, SOCK_TXTIME, true); 1240 sk->sk_clockid = sk_txtime.clockid; 1241 sk->sk_txtime_deadline_mode = 1242 !!(sk_txtime.flags & SOF_TXTIME_DEADLINE_MODE); 1243 sk->sk_txtime_report_errors = 1244 !!(sk_txtime.flags & SOF_TXTIME_REPORT_ERRORS); 1245 break; 1246 1247 case SO_BINDTOIFINDEX: 1248 ret = sock_bindtoindex_locked(sk, val); 1249 break; 1250 1251 default: 1252 ret = -ENOPROTOOPT; 1253 break; 1254 } 1255 release_sock(sk); 1256 return ret; 1257 } 1258 EXPORT_SYMBOL(sock_setsockopt); 1259 1260 1261 static void cred_to_ucred(struct pid *pid, const struct cred *cred, 1262 struct ucred *ucred) 1263 { 1264 ucred->pid = pid_vnr(pid); 1265 ucred->uid = ucred->gid = -1; 1266 if (cred) { 1267 struct user_namespace *current_ns = current_user_ns(); 1268 1269 ucred->uid = from_kuid_munged(current_ns, cred->euid); 1270 ucred->gid = from_kgid_munged(current_ns, cred->egid); 1271 } 1272 } 1273 1274 static int groups_to_user(gid_t __user *dst, const struct group_info *src) 1275 { 1276 struct user_namespace *user_ns = current_user_ns(); 1277 int i; 1278 1279 for (i = 0; i < src->ngroups; i++) 1280 if (put_user(from_kgid_munged(user_ns, src->gid[i]), dst + i)) 1281 return -EFAULT; 1282 1283 return 0; 1284 } 1285 1286 int sock_getsockopt(struct socket *sock, int level, int optname, 1287 char __user *optval, int __user *optlen) 1288 { 1289 struct sock *sk = sock->sk; 1290 1291 union { 1292 int val; 1293 u64 val64; 1294 unsigned long ulval; 1295 struct linger ling; 1296 struct old_timeval32 tm32; 1297 struct __kernel_old_timeval tm; 1298 struct __kernel_sock_timeval stm; 1299 struct sock_txtime txtime; 1300 } v; 1301 1302 int lv = sizeof(int); 1303 int len; 1304 1305 if (get_user(len, optlen)) 1306 return -EFAULT; 1307 if (len < 0) 1308 return -EINVAL; 1309 1310 memset(&v, 0, sizeof(v)); 1311 1312 switch (optname) { 1313 case SO_DEBUG: 1314 v.val = sock_flag(sk, SOCK_DBG); 1315 break; 1316 1317 case SO_DONTROUTE: 1318 v.val = sock_flag(sk, SOCK_LOCALROUTE); 1319 break; 1320 1321 case SO_BROADCAST: 1322 v.val = sock_flag(sk, SOCK_BROADCAST); 1323 break; 1324 1325 case SO_SNDBUF: 1326 v.val = sk->sk_sndbuf; 1327 break; 1328 1329 case SO_RCVBUF: 1330 v.val = sk->sk_rcvbuf; 1331 break; 1332 1333 case SO_REUSEADDR: 1334 v.val = sk->sk_reuse; 1335 break; 1336 1337 case SO_REUSEPORT: 1338 v.val = sk->sk_reuseport; 1339 break; 1340 1341 case SO_KEEPALIVE: 1342 v.val = sock_flag(sk, SOCK_KEEPOPEN); 1343 break; 1344 1345 case SO_TYPE: 1346 v.val = sk->sk_type; 1347 break; 1348 1349 case SO_PROTOCOL: 1350 v.val = sk->sk_protocol; 1351 break; 1352 1353 case SO_DOMAIN: 1354 v.val = sk->sk_family; 1355 break; 1356 1357 case SO_ERROR: 1358 v.val = -sock_error(sk); 1359 if (v.val == 0) 1360 v.val = xchg(&sk->sk_err_soft, 0); 1361 break; 1362 1363 case SO_OOBINLINE: 1364 v.val = sock_flag(sk, SOCK_URGINLINE); 1365 break; 1366 1367 case SO_NO_CHECK: 1368 v.val = sk->sk_no_check_tx; 1369 break; 1370 1371 case SO_PRIORITY: 1372 v.val = sk->sk_priority; 1373 break; 1374 1375 case SO_LINGER: 1376 lv = sizeof(v.ling); 1377 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER); 1378 v.ling.l_linger = sk->sk_lingertime / HZ; 1379 break; 1380 1381 case SO_BSDCOMPAT: 1382 sock_warn_obsolete_bsdism("getsockopt"); 1383 break; 1384 1385 case SO_TIMESTAMP_OLD: 1386 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && 1387 !sock_flag(sk, SOCK_TSTAMP_NEW) && 1388 !sock_flag(sk, SOCK_RCVTSTAMPNS); 1389 break; 1390 1391 case SO_TIMESTAMPNS_OLD: 1392 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && !sock_flag(sk, SOCK_TSTAMP_NEW); 1393 break; 1394 1395 case SO_TIMESTAMP_NEW: 1396 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && sock_flag(sk, SOCK_TSTAMP_NEW); 1397 break; 1398 1399 case SO_TIMESTAMPNS_NEW: 1400 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && sock_flag(sk, SOCK_TSTAMP_NEW); 1401 break; 1402 1403 case SO_TIMESTAMPING_OLD: 1404 v.val = sk->sk_tsflags; 1405 break; 1406 1407 case SO_RCVTIMEO_OLD: 1408 case SO_RCVTIMEO_NEW: 1409 lv = sock_get_timeout(sk->sk_rcvtimeo, &v, SO_RCVTIMEO_OLD == optname); 1410 break; 1411 1412 case SO_SNDTIMEO_OLD: 1413 case SO_SNDTIMEO_NEW: 1414 lv = sock_get_timeout(sk->sk_sndtimeo, &v, SO_SNDTIMEO_OLD == optname); 1415 break; 1416 1417 case SO_RCVLOWAT: 1418 v.val = sk->sk_rcvlowat; 1419 break; 1420 1421 case SO_SNDLOWAT: 1422 v.val = 1; 1423 break; 1424 1425 case SO_PASSCRED: 1426 v.val = !!test_bit(SOCK_PASSCRED, &sock->flags); 1427 break; 1428 1429 case SO_PEERCRED: 1430 { 1431 struct ucred peercred; 1432 if (len > sizeof(peercred)) 1433 len = sizeof(peercred); 1434 cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred); 1435 if (copy_to_user(optval, &peercred, len)) 1436 return -EFAULT; 1437 goto lenout; 1438 } 1439 1440 case SO_PEERGROUPS: 1441 { 1442 int ret, n; 1443 1444 if (!sk->sk_peer_cred) 1445 return -ENODATA; 1446 1447 n = sk->sk_peer_cred->group_info->ngroups; 1448 if (len < n * sizeof(gid_t)) { 1449 len = n * sizeof(gid_t); 1450 return put_user(len, optlen) ? -EFAULT : -ERANGE; 1451 } 1452 len = n * sizeof(gid_t); 1453 1454 ret = groups_to_user((gid_t __user *)optval, 1455 sk->sk_peer_cred->group_info); 1456 if (ret) 1457 return ret; 1458 goto lenout; 1459 } 1460 1461 case SO_PEERNAME: 1462 { 1463 char address[128]; 1464 1465 lv = sock->ops->getname(sock, (struct sockaddr *)address, 2); 1466 if (lv < 0) 1467 return -ENOTCONN; 1468 if (lv < len) 1469 return -EINVAL; 1470 if (copy_to_user(optval, address, len)) 1471 return -EFAULT; 1472 goto lenout; 1473 } 1474 1475 /* Dubious BSD thing... Probably nobody even uses it, but 1476 * the UNIX standard wants it for whatever reason... -DaveM 1477 */ 1478 case SO_ACCEPTCONN: 1479 v.val = sk->sk_state == TCP_LISTEN; 1480 break; 1481 1482 case SO_PASSSEC: 1483 v.val = !!test_bit(SOCK_PASSSEC, &sock->flags); 1484 break; 1485 1486 case SO_PEERSEC: 1487 return security_socket_getpeersec_stream(sock, optval, optlen, len); 1488 1489 case SO_MARK: 1490 v.val = sk->sk_mark; 1491 break; 1492 1493 case SO_RXQ_OVFL: 1494 v.val = sock_flag(sk, SOCK_RXQ_OVFL); 1495 break; 1496 1497 case SO_WIFI_STATUS: 1498 v.val = sock_flag(sk, SOCK_WIFI_STATUS); 1499 break; 1500 1501 case SO_PEEK_OFF: 1502 if (!sock->ops->set_peek_off) 1503 return -EOPNOTSUPP; 1504 1505 v.val = sk->sk_peek_off; 1506 break; 1507 case SO_NOFCS: 1508 v.val = sock_flag(sk, SOCK_NOFCS); 1509 break; 1510 1511 case SO_BINDTODEVICE: 1512 return sock_getbindtodevice(sk, optval, optlen, len); 1513 1514 case SO_GET_FILTER: 1515 len = sk_get_filter(sk, (struct sock_filter __user *)optval, len); 1516 if (len < 0) 1517 return len; 1518 1519 goto lenout; 1520 1521 case SO_LOCK_FILTER: 1522 v.val = sock_flag(sk, SOCK_FILTER_LOCKED); 1523 break; 1524 1525 case SO_BPF_EXTENSIONS: 1526 v.val = bpf_tell_extensions(); 1527 break; 1528 1529 case SO_SELECT_ERR_QUEUE: 1530 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE); 1531 break; 1532 1533 #ifdef CONFIG_NET_RX_BUSY_POLL 1534 case SO_BUSY_POLL: 1535 v.val = sk->sk_ll_usec; 1536 break; 1537 #endif 1538 1539 case SO_MAX_PACING_RATE: 1540 if (sizeof(v.ulval) != sizeof(v.val) && len >= sizeof(v.ulval)) { 1541 lv = sizeof(v.ulval); 1542 v.ulval = sk->sk_max_pacing_rate; 1543 } else { 1544 /* 32bit version */ 1545 v.val = min_t(unsigned long, sk->sk_max_pacing_rate, ~0U); 1546 } 1547 break; 1548 1549 case SO_INCOMING_CPU: 1550 v.val = READ_ONCE(sk->sk_incoming_cpu); 1551 break; 1552 1553 case SO_MEMINFO: 1554 { 1555 u32 meminfo[SK_MEMINFO_VARS]; 1556 1557 sk_get_meminfo(sk, meminfo); 1558 1559 len = min_t(unsigned int, len, sizeof(meminfo)); 1560 if (copy_to_user(optval, &meminfo, len)) 1561 return -EFAULT; 1562 1563 goto lenout; 1564 } 1565 1566 #ifdef CONFIG_NET_RX_BUSY_POLL 1567 case SO_INCOMING_NAPI_ID: 1568 v.val = READ_ONCE(sk->sk_napi_id); 1569 1570 /* aggregate non-NAPI IDs down to 0 */ 1571 if (v.val < MIN_NAPI_ID) 1572 v.val = 0; 1573 1574 break; 1575 #endif 1576 1577 case SO_COOKIE: 1578 lv = sizeof(u64); 1579 if (len < lv) 1580 return -EINVAL; 1581 v.val64 = sock_gen_cookie(sk); 1582 break; 1583 1584 case SO_ZEROCOPY: 1585 v.val = sock_flag(sk, SOCK_ZEROCOPY); 1586 break; 1587 1588 case SO_TXTIME: 1589 lv = sizeof(v.txtime); 1590 v.txtime.clockid = sk->sk_clockid; 1591 v.txtime.flags |= sk->sk_txtime_deadline_mode ? 1592 SOF_TXTIME_DEADLINE_MODE : 0; 1593 v.txtime.flags |= sk->sk_txtime_report_errors ? 1594 SOF_TXTIME_REPORT_ERRORS : 0; 1595 break; 1596 1597 case SO_BINDTOIFINDEX: 1598 v.val = sk->sk_bound_dev_if; 1599 break; 1600 1601 default: 1602 /* We implement the SO_SNDLOWAT etc to not be settable 1603 * (1003.1g 7). 1604 */ 1605 return -ENOPROTOOPT; 1606 } 1607 1608 if (len > lv) 1609 len = lv; 1610 if (copy_to_user(optval, &v, len)) 1611 return -EFAULT; 1612 lenout: 1613 if (put_user(len, optlen)) 1614 return -EFAULT; 1615 return 0; 1616 } 1617 1618 /* 1619 * Initialize an sk_lock. 1620 * 1621 * (We also register the sk_lock with the lock validator.) 1622 */ 1623 static inline void sock_lock_init(struct sock *sk) 1624 { 1625 if (sk->sk_kern_sock) 1626 sock_lock_init_class_and_name( 1627 sk, 1628 af_family_kern_slock_key_strings[sk->sk_family], 1629 af_family_kern_slock_keys + sk->sk_family, 1630 af_family_kern_key_strings[sk->sk_family], 1631 af_family_kern_keys + sk->sk_family); 1632 else 1633 sock_lock_init_class_and_name( 1634 sk, 1635 af_family_slock_key_strings[sk->sk_family], 1636 af_family_slock_keys + sk->sk_family, 1637 af_family_key_strings[sk->sk_family], 1638 af_family_keys + sk->sk_family); 1639 } 1640 1641 /* 1642 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet, 1643 * even temporarly, because of RCU lookups. sk_node should also be left as is. 1644 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end 1645 */ 1646 static void sock_copy(struct sock *nsk, const struct sock *osk) 1647 { 1648 const struct proto *prot = READ_ONCE(osk->sk_prot); 1649 #ifdef CONFIG_SECURITY_NETWORK 1650 void *sptr = nsk->sk_security; 1651 #endif 1652 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin)); 1653 1654 memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end, 1655 prot->obj_size - offsetof(struct sock, sk_dontcopy_end)); 1656 1657 #ifdef CONFIG_SECURITY_NETWORK 1658 nsk->sk_security = sptr; 1659 security_sk_clone(osk, nsk); 1660 #endif 1661 } 1662 1663 static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority, 1664 int family) 1665 { 1666 struct sock *sk; 1667 struct kmem_cache *slab; 1668 1669 slab = prot->slab; 1670 if (slab != NULL) { 1671 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO); 1672 if (!sk) 1673 return sk; 1674 if (want_init_on_alloc(priority)) 1675 sk_prot_clear_nulls(sk, prot->obj_size); 1676 } else 1677 sk = kmalloc(prot->obj_size, priority); 1678 1679 if (sk != NULL) { 1680 if (security_sk_alloc(sk, family, priority)) 1681 goto out_free; 1682 1683 if (!try_module_get(prot->owner)) 1684 goto out_free_sec; 1685 sk_tx_queue_clear(sk); 1686 } 1687 1688 return sk; 1689 1690 out_free_sec: 1691 security_sk_free(sk); 1692 out_free: 1693 if (slab != NULL) 1694 kmem_cache_free(slab, sk); 1695 else 1696 kfree(sk); 1697 return NULL; 1698 } 1699 1700 static void sk_prot_free(struct proto *prot, struct sock *sk) 1701 { 1702 struct kmem_cache *slab; 1703 struct module *owner; 1704 1705 owner = prot->owner; 1706 slab = prot->slab; 1707 1708 cgroup_sk_free(&sk->sk_cgrp_data); 1709 mem_cgroup_sk_free(sk); 1710 security_sk_free(sk); 1711 if (slab != NULL) 1712 kmem_cache_free(slab, sk); 1713 else 1714 kfree(sk); 1715 module_put(owner); 1716 } 1717 1718 /** 1719 * sk_alloc - All socket objects are allocated here 1720 * @net: the applicable net namespace 1721 * @family: protocol family 1722 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 1723 * @prot: struct proto associated with this new sock instance 1724 * @kern: is this to be a kernel socket? 1725 */ 1726 struct sock *sk_alloc(struct net *net, int family, gfp_t priority, 1727 struct proto *prot, int kern) 1728 { 1729 struct sock *sk; 1730 1731 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family); 1732 if (sk) { 1733 sk->sk_family = family; 1734 /* 1735 * See comment in struct sock definition to understand 1736 * why we need sk_prot_creator -acme 1737 */ 1738 sk->sk_prot = sk->sk_prot_creator = prot; 1739 sk->sk_kern_sock = kern; 1740 sock_lock_init(sk); 1741 sk->sk_net_refcnt = kern ? 0 : 1; 1742 if (likely(sk->sk_net_refcnt)) { 1743 get_net(net); 1744 sock_inuse_add(net, 1); 1745 } 1746 1747 sock_net_set(sk, net); 1748 refcount_set(&sk->sk_wmem_alloc, 1); 1749 1750 mem_cgroup_sk_alloc(sk); 1751 cgroup_sk_alloc(&sk->sk_cgrp_data); 1752 sock_update_classid(&sk->sk_cgrp_data); 1753 sock_update_netprioidx(&sk->sk_cgrp_data); 1754 sk_tx_queue_clear(sk); 1755 } 1756 1757 return sk; 1758 } 1759 EXPORT_SYMBOL(sk_alloc); 1760 1761 /* Sockets having SOCK_RCU_FREE will call this function after one RCU 1762 * grace period. This is the case for UDP sockets and TCP listeners. 1763 */ 1764 static void __sk_destruct(struct rcu_head *head) 1765 { 1766 struct sock *sk = container_of(head, struct sock, sk_rcu); 1767 struct sk_filter *filter; 1768 1769 if (sk->sk_destruct) 1770 sk->sk_destruct(sk); 1771 1772 filter = rcu_dereference_check(sk->sk_filter, 1773 refcount_read(&sk->sk_wmem_alloc) == 0); 1774 if (filter) { 1775 sk_filter_uncharge(sk, filter); 1776 RCU_INIT_POINTER(sk->sk_filter, NULL); 1777 } 1778 1779 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP); 1780 1781 #ifdef CONFIG_BPF_SYSCALL 1782 bpf_sk_storage_free(sk); 1783 #endif 1784 1785 if (atomic_read(&sk->sk_omem_alloc)) 1786 pr_debug("%s: optmem leakage (%d bytes) detected\n", 1787 __func__, atomic_read(&sk->sk_omem_alloc)); 1788 1789 if (sk->sk_frag.page) { 1790 put_page(sk->sk_frag.page); 1791 sk->sk_frag.page = NULL; 1792 } 1793 1794 if (sk->sk_peer_cred) 1795 put_cred(sk->sk_peer_cred); 1796 put_pid(sk->sk_peer_pid); 1797 if (likely(sk->sk_net_refcnt)) 1798 put_net(sock_net(sk)); 1799 sk_prot_free(sk->sk_prot_creator, sk); 1800 } 1801 1802 void sk_destruct(struct sock *sk) 1803 { 1804 bool use_call_rcu = sock_flag(sk, SOCK_RCU_FREE); 1805 1806 if (rcu_access_pointer(sk->sk_reuseport_cb)) { 1807 reuseport_detach_sock(sk); 1808 use_call_rcu = true; 1809 } 1810 1811 if (use_call_rcu) 1812 call_rcu(&sk->sk_rcu, __sk_destruct); 1813 else 1814 __sk_destruct(&sk->sk_rcu); 1815 } 1816 1817 static void __sk_free(struct sock *sk) 1818 { 1819 if (likely(sk->sk_net_refcnt)) 1820 sock_inuse_add(sock_net(sk), -1); 1821 1822 if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk))) 1823 sock_diag_broadcast_destroy(sk); 1824 else 1825 sk_destruct(sk); 1826 } 1827 1828 void sk_free(struct sock *sk) 1829 { 1830 /* 1831 * We subtract one from sk_wmem_alloc and can know if 1832 * some packets are still in some tx queue. 1833 * If not null, sock_wfree() will call __sk_free(sk) later 1834 */ 1835 if (refcount_dec_and_test(&sk->sk_wmem_alloc)) 1836 __sk_free(sk); 1837 } 1838 EXPORT_SYMBOL(sk_free); 1839 1840 static void sk_init_common(struct sock *sk) 1841 { 1842 skb_queue_head_init(&sk->sk_receive_queue); 1843 skb_queue_head_init(&sk->sk_write_queue); 1844 skb_queue_head_init(&sk->sk_error_queue); 1845 1846 rwlock_init(&sk->sk_callback_lock); 1847 lockdep_set_class_and_name(&sk->sk_receive_queue.lock, 1848 af_rlock_keys + sk->sk_family, 1849 af_family_rlock_key_strings[sk->sk_family]); 1850 lockdep_set_class_and_name(&sk->sk_write_queue.lock, 1851 af_wlock_keys + sk->sk_family, 1852 af_family_wlock_key_strings[sk->sk_family]); 1853 lockdep_set_class_and_name(&sk->sk_error_queue.lock, 1854 af_elock_keys + sk->sk_family, 1855 af_family_elock_key_strings[sk->sk_family]); 1856 lockdep_set_class_and_name(&sk->sk_callback_lock, 1857 af_callback_keys + sk->sk_family, 1858 af_family_clock_key_strings[sk->sk_family]); 1859 } 1860 1861 /** 1862 * sk_clone_lock - clone a socket, and lock its clone 1863 * @sk: the socket to clone 1864 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 1865 * 1866 * Caller must unlock socket even in error path (bh_unlock_sock(newsk)) 1867 */ 1868 struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority) 1869 { 1870 struct proto *prot = READ_ONCE(sk->sk_prot); 1871 struct sock *newsk; 1872 bool is_charged = true; 1873 1874 newsk = sk_prot_alloc(prot, priority, sk->sk_family); 1875 if (newsk != NULL) { 1876 struct sk_filter *filter; 1877 1878 sock_copy(newsk, sk); 1879 1880 newsk->sk_prot_creator = prot; 1881 1882 /* SANITY */ 1883 if (likely(newsk->sk_net_refcnt)) 1884 get_net(sock_net(newsk)); 1885 sk_node_init(&newsk->sk_node); 1886 sock_lock_init(newsk); 1887 bh_lock_sock(newsk); 1888 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL; 1889 newsk->sk_backlog.len = 0; 1890 1891 atomic_set(&newsk->sk_rmem_alloc, 0); 1892 /* 1893 * sk_wmem_alloc set to one (see sk_free() and sock_wfree()) 1894 */ 1895 refcount_set(&newsk->sk_wmem_alloc, 1); 1896 atomic_set(&newsk->sk_omem_alloc, 0); 1897 sk_init_common(newsk); 1898 1899 newsk->sk_dst_cache = NULL; 1900 newsk->sk_dst_pending_confirm = 0; 1901 newsk->sk_wmem_queued = 0; 1902 newsk->sk_forward_alloc = 0; 1903 atomic_set(&newsk->sk_drops, 0); 1904 newsk->sk_send_head = NULL; 1905 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK; 1906 atomic_set(&newsk->sk_zckey, 0); 1907 1908 sock_reset_flag(newsk, SOCK_DONE); 1909 1910 /* sk->sk_memcg will be populated at accept() time */ 1911 newsk->sk_memcg = NULL; 1912 1913 cgroup_sk_clone(&newsk->sk_cgrp_data); 1914 1915 rcu_read_lock(); 1916 filter = rcu_dereference(sk->sk_filter); 1917 if (filter != NULL) 1918 /* though it's an empty new sock, the charging may fail 1919 * if sysctl_optmem_max was changed between creation of 1920 * original socket and cloning 1921 */ 1922 is_charged = sk_filter_charge(newsk, filter); 1923 RCU_INIT_POINTER(newsk->sk_filter, filter); 1924 rcu_read_unlock(); 1925 1926 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) { 1927 /* We need to make sure that we don't uncharge the new 1928 * socket if we couldn't charge it in the first place 1929 * as otherwise we uncharge the parent's filter. 1930 */ 1931 if (!is_charged) 1932 RCU_INIT_POINTER(newsk->sk_filter, NULL); 1933 sk_free_unlock_clone(newsk); 1934 newsk = NULL; 1935 goto out; 1936 } 1937 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL); 1938 1939 if (bpf_sk_storage_clone(sk, newsk)) { 1940 sk_free_unlock_clone(newsk); 1941 newsk = NULL; 1942 goto out; 1943 } 1944 1945 /* Clear sk_user_data if parent had the pointer tagged 1946 * as not suitable for copying when cloning. 1947 */ 1948 if (sk_user_data_is_nocopy(newsk)) 1949 newsk->sk_user_data = NULL; 1950 1951 newsk->sk_err = 0; 1952 newsk->sk_err_soft = 0; 1953 newsk->sk_priority = 0; 1954 newsk->sk_incoming_cpu = raw_smp_processor_id(); 1955 if (likely(newsk->sk_net_refcnt)) 1956 sock_inuse_add(sock_net(newsk), 1); 1957 1958 /* 1959 * Before updating sk_refcnt, we must commit prior changes to memory 1960 * (Documentation/RCU/rculist_nulls.rst for details) 1961 */ 1962 smp_wmb(); 1963 refcount_set(&newsk->sk_refcnt, 2); 1964 1965 /* 1966 * Increment the counter in the same struct proto as the master 1967 * sock (sk_refcnt_debug_inc uses newsk->sk_prot->socks, that 1968 * is the same as sk->sk_prot->socks, as this field was copied 1969 * with memcpy). 1970 * 1971 * This _changes_ the previous behaviour, where 1972 * tcp_create_openreq_child always was incrementing the 1973 * equivalent to tcp_prot->socks (inet_sock_nr), so this have 1974 * to be taken into account in all callers. -acme 1975 */ 1976 sk_refcnt_debug_inc(newsk); 1977 sk_set_socket(newsk, NULL); 1978 sk_tx_queue_clear(newsk); 1979 RCU_INIT_POINTER(newsk->sk_wq, NULL); 1980 1981 if (newsk->sk_prot->sockets_allocated) 1982 sk_sockets_allocated_inc(newsk); 1983 1984 if (sock_needs_netstamp(sk) && 1985 newsk->sk_flags & SK_FLAGS_TIMESTAMP) 1986 net_enable_timestamp(); 1987 } 1988 out: 1989 return newsk; 1990 } 1991 EXPORT_SYMBOL_GPL(sk_clone_lock); 1992 1993 void sk_free_unlock_clone(struct sock *sk) 1994 { 1995 /* It is still raw copy of parent, so invalidate 1996 * destructor and make plain sk_free() */ 1997 sk->sk_destruct = NULL; 1998 bh_unlock_sock(sk); 1999 sk_free(sk); 2000 } 2001 EXPORT_SYMBOL_GPL(sk_free_unlock_clone); 2002 2003 void sk_setup_caps(struct sock *sk, struct dst_entry *dst) 2004 { 2005 u32 max_segs = 1; 2006 2007 sk_dst_set(sk, dst); 2008 sk->sk_route_caps = dst->dev->features | sk->sk_route_forced_caps; 2009 if (sk->sk_route_caps & NETIF_F_GSO) 2010 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE; 2011 sk->sk_route_caps &= ~sk->sk_route_nocaps; 2012 if (sk_can_gso(sk)) { 2013 if (dst->header_len && !xfrm_dst_offload_ok(dst)) { 2014 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 2015 } else { 2016 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM; 2017 sk->sk_gso_max_size = dst->dev->gso_max_size; 2018 max_segs = max_t(u32, dst->dev->gso_max_segs, 1); 2019 } 2020 } 2021 sk->sk_gso_max_segs = max_segs; 2022 } 2023 EXPORT_SYMBOL_GPL(sk_setup_caps); 2024 2025 /* 2026 * Simple resource managers for sockets. 2027 */ 2028 2029 2030 /* 2031 * Write buffer destructor automatically called from kfree_skb. 2032 */ 2033 void sock_wfree(struct sk_buff *skb) 2034 { 2035 struct sock *sk = skb->sk; 2036 unsigned int len = skb->truesize; 2037 2038 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) { 2039 /* 2040 * Keep a reference on sk_wmem_alloc, this will be released 2041 * after sk_write_space() call 2042 */ 2043 WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc)); 2044 sk->sk_write_space(sk); 2045 len = 1; 2046 } 2047 /* 2048 * if sk_wmem_alloc reaches 0, we must finish what sk_free() 2049 * could not do because of in-flight packets 2050 */ 2051 if (refcount_sub_and_test(len, &sk->sk_wmem_alloc)) 2052 __sk_free(sk); 2053 } 2054 EXPORT_SYMBOL(sock_wfree); 2055 2056 /* This variant of sock_wfree() is used by TCP, 2057 * since it sets SOCK_USE_WRITE_QUEUE. 2058 */ 2059 void __sock_wfree(struct sk_buff *skb) 2060 { 2061 struct sock *sk = skb->sk; 2062 2063 if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)) 2064 __sk_free(sk); 2065 } 2066 2067 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk) 2068 { 2069 skb_orphan(skb); 2070 skb->sk = sk; 2071 #ifdef CONFIG_INET 2072 if (unlikely(!sk_fullsock(sk))) { 2073 skb->destructor = sock_edemux; 2074 sock_hold(sk); 2075 return; 2076 } 2077 #endif 2078 skb->destructor = sock_wfree; 2079 skb_set_hash_from_sk(skb, sk); 2080 /* 2081 * We used to take a refcount on sk, but following operation 2082 * is enough to guarantee sk_free() wont free this sock until 2083 * all in-flight packets are completed 2084 */ 2085 refcount_add(skb->truesize, &sk->sk_wmem_alloc); 2086 } 2087 EXPORT_SYMBOL(skb_set_owner_w); 2088 2089 static bool can_skb_orphan_partial(const struct sk_buff *skb) 2090 { 2091 #ifdef CONFIG_TLS_DEVICE 2092 /* Drivers depend on in-order delivery for crypto offload, 2093 * partial orphan breaks out-of-order-OK logic. 2094 */ 2095 if (skb->decrypted) 2096 return false; 2097 #endif 2098 return (skb->destructor == sock_wfree || 2099 (IS_ENABLED(CONFIG_INET) && skb->destructor == tcp_wfree)); 2100 } 2101 2102 /* This helper is used by netem, as it can hold packets in its 2103 * delay queue. We want to allow the owner socket to send more 2104 * packets, as if they were already TX completed by a typical driver. 2105 * But we also want to keep skb->sk set because some packet schedulers 2106 * rely on it (sch_fq for example). 2107 */ 2108 void skb_orphan_partial(struct sk_buff *skb) 2109 { 2110 if (skb_is_tcp_pure_ack(skb)) 2111 return; 2112 2113 if (can_skb_orphan_partial(skb)) { 2114 struct sock *sk = skb->sk; 2115 2116 if (refcount_inc_not_zero(&sk->sk_refcnt)) { 2117 WARN_ON(refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)); 2118 skb->destructor = sock_efree; 2119 } 2120 } else { 2121 skb_orphan(skb); 2122 } 2123 } 2124 EXPORT_SYMBOL(skb_orphan_partial); 2125 2126 /* 2127 * Read buffer destructor automatically called from kfree_skb. 2128 */ 2129 void sock_rfree(struct sk_buff *skb) 2130 { 2131 struct sock *sk = skb->sk; 2132 unsigned int len = skb->truesize; 2133 2134 atomic_sub(len, &sk->sk_rmem_alloc); 2135 sk_mem_uncharge(sk, len); 2136 } 2137 EXPORT_SYMBOL(sock_rfree); 2138 2139 /* 2140 * Buffer destructor for skbs that are not used directly in read or write 2141 * path, e.g. for error handler skbs. Automatically called from kfree_skb. 2142 */ 2143 void sock_efree(struct sk_buff *skb) 2144 { 2145 sock_put(skb->sk); 2146 } 2147 EXPORT_SYMBOL(sock_efree); 2148 2149 /* Buffer destructor for prefetch/receive path where reference count may 2150 * not be held, e.g. for listen sockets. 2151 */ 2152 #ifdef CONFIG_INET 2153 void sock_pfree(struct sk_buff *skb) 2154 { 2155 if (sk_is_refcounted(skb->sk)) 2156 sock_gen_put(skb->sk); 2157 } 2158 EXPORT_SYMBOL(sock_pfree); 2159 #endif /* CONFIG_INET */ 2160 2161 kuid_t sock_i_uid(struct sock *sk) 2162 { 2163 kuid_t uid; 2164 2165 read_lock_bh(&sk->sk_callback_lock); 2166 uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID; 2167 read_unlock_bh(&sk->sk_callback_lock); 2168 return uid; 2169 } 2170 EXPORT_SYMBOL(sock_i_uid); 2171 2172 unsigned long sock_i_ino(struct sock *sk) 2173 { 2174 unsigned long ino; 2175 2176 read_lock_bh(&sk->sk_callback_lock); 2177 ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0; 2178 read_unlock_bh(&sk->sk_callback_lock); 2179 return ino; 2180 } 2181 EXPORT_SYMBOL(sock_i_ino); 2182 2183 /* 2184 * Allocate a skb from the socket's send buffer. 2185 */ 2186 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force, 2187 gfp_t priority) 2188 { 2189 if (force || 2190 refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) { 2191 struct sk_buff *skb = alloc_skb(size, priority); 2192 2193 if (skb) { 2194 skb_set_owner_w(skb, sk); 2195 return skb; 2196 } 2197 } 2198 return NULL; 2199 } 2200 EXPORT_SYMBOL(sock_wmalloc); 2201 2202 static void sock_ofree(struct sk_buff *skb) 2203 { 2204 struct sock *sk = skb->sk; 2205 2206 atomic_sub(skb->truesize, &sk->sk_omem_alloc); 2207 } 2208 2209 struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size, 2210 gfp_t priority) 2211 { 2212 struct sk_buff *skb; 2213 2214 /* small safe race: SKB_TRUESIZE may differ from final skb->truesize */ 2215 if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) > 2216 sysctl_optmem_max) 2217 return NULL; 2218 2219 skb = alloc_skb(size, priority); 2220 if (!skb) 2221 return NULL; 2222 2223 atomic_add(skb->truesize, &sk->sk_omem_alloc); 2224 skb->sk = sk; 2225 skb->destructor = sock_ofree; 2226 return skb; 2227 } 2228 2229 /* 2230 * Allocate a memory block from the socket's option memory buffer. 2231 */ 2232 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority) 2233 { 2234 if ((unsigned int)size <= sysctl_optmem_max && 2235 atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) { 2236 void *mem; 2237 /* First do the add, to avoid the race if kmalloc 2238 * might sleep. 2239 */ 2240 atomic_add(size, &sk->sk_omem_alloc); 2241 mem = kmalloc(size, priority); 2242 if (mem) 2243 return mem; 2244 atomic_sub(size, &sk->sk_omem_alloc); 2245 } 2246 return NULL; 2247 } 2248 EXPORT_SYMBOL(sock_kmalloc); 2249 2250 /* Free an option memory block. Note, we actually want the inline 2251 * here as this allows gcc to detect the nullify and fold away the 2252 * condition entirely. 2253 */ 2254 static inline void __sock_kfree_s(struct sock *sk, void *mem, int size, 2255 const bool nullify) 2256 { 2257 if (WARN_ON_ONCE(!mem)) 2258 return; 2259 if (nullify) 2260 kfree_sensitive(mem); 2261 else 2262 kfree(mem); 2263 atomic_sub(size, &sk->sk_omem_alloc); 2264 } 2265 2266 void sock_kfree_s(struct sock *sk, void *mem, int size) 2267 { 2268 __sock_kfree_s(sk, mem, size, false); 2269 } 2270 EXPORT_SYMBOL(sock_kfree_s); 2271 2272 void sock_kzfree_s(struct sock *sk, void *mem, int size) 2273 { 2274 __sock_kfree_s(sk, mem, size, true); 2275 } 2276 EXPORT_SYMBOL(sock_kzfree_s); 2277 2278 /* It is almost wait_for_tcp_memory minus release_sock/lock_sock. 2279 I think, these locks should be removed for datagram sockets. 2280 */ 2281 static long sock_wait_for_wmem(struct sock *sk, long timeo) 2282 { 2283 DEFINE_WAIT(wait); 2284 2285 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 2286 for (;;) { 2287 if (!timeo) 2288 break; 2289 if (signal_pending(current)) 2290 break; 2291 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2292 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); 2293 if (refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) 2294 break; 2295 if (sk->sk_shutdown & SEND_SHUTDOWN) 2296 break; 2297 if (sk->sk_err) 2298 break; 2299 timeo = schedule_timeout(timeo); 2300 } 2301 finish_wait(sk_sleep(sk), &wait); 2302 return timeo; 2303 } 2304 2305 2306 /* 2307 * Generic send/receive buffer handlers 2308 */ 2309 2310 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len, 2311 unsigned long data_len, int noblock, 2312 int *errcode, int max_page_order) 2313 { 2314 struct sk_buff *skb; 2315 long timeo; 2316 int err; 2317 2318 timeo = sock_sndtimeo(sk, noblock); 2319 for (;;) { 2320 err = sock_error(sk); 2321 if (err != 0) 2322 goto failure; 2323 2324 err = -EPIPE; 2325 if (sk->sk_shutdown & SEND_SHUTDOWN) 2326 goto failure; 2327 2328 if (sk_wmem_alloc_get(sk) < READ_ONCE(sk->sk_sndbuf)) 2329 break; 2330 2331 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 2332 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2333 err = -EAGAIN; 2334 if (!timeo) 2335 goto failure; 2336 if (signal_pending(current)) 2337 goto interrupted; 2338 timeo = sock_wait_for_wmem(sk, timeo); 2339 } 2340 skb = alloc_skb_with_frags(header_len, data_len, max_page_order, 2341 errcode, sk->sk_allocation); 2342 if (skb) 2343 skb_set_owner_w(skb, sk); 2344 return skb; 2345 2346 interrupted: 2347 err = sock_intr_errno(timeo); 2348 failure: 2349 *errcode = err; 2350 return NULL; 2351 } 2352 EXPORT_SYMBOL(sock_alloc_send_pskb); 2353 2354 struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size, 2355 int noblock, int *errcode) 2356 { 2357 return sock_alloc_send_pskb(sk, size, 0, noblock, errcode, 0); 2358 } 2359 EXPORT_SYMBOL(sock_alloc_send_skb); 2360 2361 int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg, 2362 struct sockcm_cookie *sockc) 2363 { 2364 u32 tsflags; 2365 2366 switch (cmsg->cmsg_type) { 2367 case SO_MARK: 2368 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 2369 return -EPERM; 2370 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 2371 return -EINVAL; 2372 sockc->mark = *(u32 *)CMSG_DATA(cmsg); 2373 break; 2374 case SO_TIMESTAMPING_OLD: 2375 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 2376 return -EINVAL; 2377 2378 tsflags = *(u32 *)CMSG_DATA(cmsg); 2379 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK) 2380 return -EINVAL; 2381 2382 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK; 2383 sockc->tsflags |= tsflags; 2384 break; 2385 case SCM_TXTIME: 2386 if (!sock_flag(sk, SOCK_TXTIME)) 2387 return -EINVAL; 2388 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64))) 2389 return -EINVAL; 2390 sockc->transmit_time = get_unaligned((u64 *)CMSG_DATA(cmsg)); 2391 break; 2392 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */ 2393 case SCM_RIGHTS: 2394 case SCM_CREDENTIALS: 2395 break; 2396 default: 2397 return -EINVAL; 2398 } 2399 return 0; 2400 } 2401 EXPORT_SYMBOL(__sock_cmsg_send); 2402 2403 int sock_cmsg_send(struct sock *sk, struct msghdr *msg, 2404 struct sockcm_cookie *sockc) 2405 { 2406 struct cmsghdr *cmsg; 2407 int ret; 2408 2409 for_each_cmsghdr(cmsg, msg) { 2410 if (!CMSG_OK(msg, cmsg)) 2411 return -EINVAL; 2412 if (cmsg->cmsg_level != SOL_SOCKET) 2413 continue; 2414 ret = __sock_cmsg_send(sk, msg, cmsg, sockc); 2415 if (ret) 2416 return ret; 2417 } 2418 return 0; 2419 } 2420 EXPORT_SYMBOL(sock_cmsg_send); 2421 2422 static void sk_enter_memory_pressure(struct sock *sk) 2423 { 2424 if (!sk->sk_prot->enter_memory_pressure) 2425 return; 2426 2427 sk->sk_prot->enter_memory_pressure(sk); 2428 } 2429 2430 static void sk_leave_memory_pressure(struct sock *sk) 2431 { 2432 if (sk->sk_prot->leave_memory_pressure) { 2433 sk->sk_prot->leave_memory_pressure(sk); 2434 } else { 2435 unsigned long *memory_pressure = sk->sk_prot->memory_pressure; 2436 2437 if (memory_pressure && READ_ONCE(*memory_pressure)) 2438 WRITE_ONCE(*memory_pressure, 0); 2439 } 2440 } 2441 2442 #define SKB_FRAG_PAGE_ORDER get_order(32768) 2443 DEFINE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key); 2444 2445 /** 2446 * skb_page_frag_refill - check that a page_frag contains enough room 2447 * @sz: minimum size of the fragment we want to get 2448 * @pfrag: pointer to page_frag 2449 * @gfp: priority for memory allocation 2450 * 2451 * Note: While this allocator tries to use high order pages, there is 2452 * no guarantee that allocations succeed. Therefore, @sz MUST be 2453 * less or equal than PAGE_SIZE. 2454 */ 2455 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp) 2456 { 2457 if (pfrag->page) { 2458 if (page_ref_count(pfrag->page) == 1) { 2459 pfrag->offset = 0; 2460 return true; 2461 } 2462 if (pfrag->offset + sz <= pfrag->size) 2463 return true; 2464 put_page(pfrag->page); 2465 } 2466 2467 pfrag->offset = 0; 2468 if (SKB_FRAG_PAGE_ORDER && 2469 !static_branch_unlikely(&net_high_order_alloc_disable_key)) { 2470 /* Avoid direct reclaim but allow kswapd to wake */ 2471 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) | 2472 __GFP_COMP | __GFP_NOWARN | 2473 __GFP_NORETRY, 2474 SKB_FRAG_PAGE_ORDER); 2475 if (likely(pfrag->page)) { 2476 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER; 2477 return true; 2478 } 2479 } 2480 pfrag->page = alloc_page(gfp); 2481 if (likely(pfrag->page)) { 2482 pfrag->size = PAGE_SIZE; 2483 return true; 2484 } 2485 return false; 2486 } 2487 EXPORT_SYMBOL(skb_page_frag_refill); 2488 2489 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 2490 { 2491 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation))) 2492 return true; 2493 2494 sk_enter_memory_pressure(sk); 2495 sk_stream_moderate_sndbuf(sk); 2496 return false; 2497 } 2498 EXPORT_SYMBOL(sk_page_frag_refill); 2499 2500 static void __lock_sock(struct sock *sk) 2501 __releases(&sk->sk_lock.slock) 2502 __acquires(&sk->sk_lock.slock) 2503 { 2504 DEFINE_WAIT(wait); 2505 2506 for (;;) { 2507 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait, 2508 TASK_UNINTERRUPTIBLE); 2509 spin_unlock_bh(&sk->sk_lock.slock); 2510 schedule(); 2511 spin_lock_bh(&sk->sk_lock.slock); 2512 if (!sock_owned_by_user(sk)) 2513 break; 2514 } 2515 finish_wait(&sk->sk_lock.wq, &wait); 2516 } 2517 2518 void __release_sock(struct sock *sk) 2519 __releases(&sk->sk_lock.slock) 2520 __acquires(&sk->sk_lock.slock) 2521 { 2522 struct sk_buff *skb, *next; 2523 2524 while ((skb = sk->sk_backlog.head) != NULL) { 2525 sk->sk_backlog.head = sk->sk_backlog.tail = NULL; 2526 2527 spin_unlock_bh(&sk->sk_lock.slock); 2528 2529 do { 2530 next = skb->next; 2531 prefetch(next); 2532 WARN_ON_ONCE(skb_dst_is_noref(skb)); 2533 skb_mark_not_on_list(skb); 2534 sk_backlog_rcv(sk, skb); 2535 2536 cond_resched(); 2537 2538 skb = next; 2539 } while (skb != NULL); 2540 2541 spin_lock_bh(&sk->sk_lock.slock); 2542 } 2543 2544 /* 2545 * Doing the zeroing here guarantee we can not loop forever 2546 * while a wild producer attempts to flood us. 2547 */ 2548 sk->sk_backlog.len = 0; 2549 } 2550 2551 void __sk_flush_backlog(struct sock *sk) 2552 { 2553 spin_lock_bh(&sk->sk_lock.slock); 2554 __release_sock(sk); 2555 spin_unlock_bh(&sk->sk_lock.slock); 2556 } 2557 2558 /** 2559 * sk_wait_data - wait for data to arrive at sk_receive_queue 2560 * @sk: sock to wait on 2561 * @timeo: for how long 2562 * @skb: last skb seen on sk_receive_queue 2563 * 2564 * Now socket state including sk->sk_err is changed only under lock, 2565 * hence we may omit checks after joining wait queue. 2566 * We check receive queue before schedule() only as optimization; 2567 * it is very likely that release_sock() added new data. 2568 */ 2569 int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb) 2570 { 2571 DEFINE_WAIT_FUNC(wait, woken_wake_function); 2572 int rc; 2573 2574 add_wait_queue(sk_sleep(sk), &wait); 2575 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 2576 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait); 2577 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 2578 remove_wait_queue(sk_sleep(sk), &wait); 2579 return rc; 2580 } 2581 EXPORT_SYMBOL(sk_wait_data); 2582 2583 /** 2584 * __sk_mem_raise_allocated - increase memory_allocated 2585 * @sk: socket 2586 * @size: memory size to allocate 2587 * @amt: pages to allocate 2588 * @kind: allocation type 2589 * 2590 * Similar to __sk_mem_schedule(), but does not update sk_forward_alloc 2591 */ 2592 int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind) 2593 { 2594 struct proto *prot = sk->sk_prot; 2595 long allocated = sk_memory_allocated_add(sk, amt); 2596 bool charged = true; 2597 2598 if (mem_cgroup_sockets_enabled && sk->sk_memcg && 2599 !(charged = mem_cgroup_charge_skmem(sk->sk_memcg, amt))) 2600 goto suppress_allocation; 2601 2602 /* Under limit. */ 2603 if (allocated <= sk_prot_mem_limits(sk, 0)) { 2604 sk_leave_memory_pressure(sk); 2605 return 1; 2606 } 2607 2608 /* Under pressure. */ 2609 if (allocated > sk_prot_mem_limits(sk, 1)) 2610 sk_enter_memory_pressure(sk); 2611 2612 /* Over hard limit. */ 2613 if (allocated > sk_prot_mem_limits(sk, 2)) 2614 goto suppress_allocation; 2615 2616 /* guarantee minimum buffer size under pressure */ 2617 if (kind == SK_MEM_RECV) { 2618 if (atomic_read(&sk->sk_rmem_alloc) < sk_get_rmem0(sk, prot)) 2619 return 1; 2620 2621 } else { /* SK_MEM_SEND */ 2622 int wmem0 = sk_get_wmem0(sk, prot); 2623 2624 if (sk->sk_type == SOCK_STREAM) { 2625 if (sk->sk_wmem_queued < wmem0) 2626 return 1; 2627 } else if (refcount_read(&sk->sk_wmem_alloc) < wmem0) { 2628 return 1; 2629 } 2630 } 2631 2632 if (sk_has_memory_pressure(sk)) { 2633 u64 alloc; 2634 2635 if (!sk_under_memory_pressure(sk)) 2636 return 1; 2637 alloc = sk_sockets_allocated_read_positive(sk); 2638 if (sk_prot_mem_limits(sk, 2) > alloc * 2639 sk_mem_pages(sk->sk_wmem_queued + 2640 atomic_read(&sk->sk_rmem_alloc) + 2641 sk->sk_forward_alloc)) 2642 return 1; 2643 } 2644 2645 suppress_allocation: 2646 2647 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) { 2648 sk_stream_moderate_sndbuf(sk); 2649 2650 /* Fail only if socket is _under_ its sndbuf. 2651 * In this case we cannot block, so that we have to fail. 2652 */ 2653 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf) 2654 return 1; 2655 } 2656 2657 if (kind == SK_MEM_SEND || (kind == SK_MEM_RECV && charged)) 2658 trace_sock_exceed_buf_limit(sk, prot, allocated, kind); 2659 2660 sk_memory_allocated_sub(sk, amt); 2661 2662 if (mem_cgroup_sockets_enabled && sk->sk_memcg) 2663 mem_cgroup_uncharge_skmem(sk->sk_memcg, amt); 2664 2665 return 0; 2666 } 2667 EXPORT_SYMBOL(__sk_mem_raise_allocated); 2668 2669 /** 2670 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated 2671 * @sk: socket 2672 * @size: memory size to allocate 2673 * @kind: allocation type 2674 * 2675 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means 2676 * rmem allocation. This function assumes that protocols which have 2677 * memory_pressure use sk_wmem_queued as write buffer accounting. 2678 */ 2679 int __sk_mem_schedule(struct sock *sk, int size, int kind) 2680 { 2681 int ret, amt = sk_mem_pages(size); 2682 2683 sk->sk_forward_alloc += amt << SK_MEM_QUANTUM_SHIFT; 2684 ret = __sk_mem_raise_allocated(sk, size, amt, kind); 2685 if (!ret) 2686 sk->sk_forward_alloc -= amt << SK_MEM_QUANTUM_SHIFT; 2687 return ret; 2688 } 2689 EXPORT_SYMBOL(__sk_mem_schedule); 2690 2691 /** 2692 * __sk_mem_reduce_allocated - reclaim memory_allocated 2693 * @sk: socket 2694 * @amount: number of quanta 2695 * 2696 * Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc 2697 */ 2698 void __sk_mem_reduce_allocated(struct sock *sk, int amount) 2699 { 2700 sk_memory_allocated_sub(sk, amount); 2701 2702 if (mem_cgroup_sockets_enabled && sk->sk_memcg) 2703 mem_cgroup_uncharge_skmem(sk->sk_memcg, amount); 2704 2705 if (sk_under_memory_pressure(sk) && 2706 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0))) 2707 sk_leave_memory_pressure(sk); 2708 } 2709 EXPORT_SYMBOL(__sk_mem_reduce_allocated); 2710 2711 /** 2712 * __sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated 2713 * @sk: socket 2714 * @amount: number of bytes (rounded down to a SK_MEM_QUANTUM multiple) 2715 */ 2716 void __sk_mem_reclaim(struct sock *sk, int amount) 2717 { 2718 amount >>= SK_MEM_QUANTUM_SHIFT; 2719 sk->sk_forward_alloc -= amount << SK_MEM_QUANTUM_SHIFT; 2720 __sk_mem_reduce_allocated(sk, amount); 2721 } 2722 EXPORT_SYMBOL(__sk_mem_reclaim); 2723 2724 int sk_set_peek_off(struct sock *sk, int val) 2725 { 2726 sk->sk_peek_off = val; 2727 return 0; 2728 } 2729 EXPORT_SYMBOL_GPL(sk_set_peek_off); 2730 2731 /* 2732 * Set of default routines for initialising struct proto_ops when 2733 * the protocol does not support a particular function. In certain 2734 * cases where it makes no sense for a protocol to have a "do nothing" 2735 * function, some default processing is provided. 2736 */ 2737 2738 int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len) 2739 { 2740 return -EOPNOTSUPP; 2741 } 2742 EXPORT_SYMBOL(sock_no_bind); 2743 2744 int sock_no_connect(struct socket *sock, struct sockaddr *saddr, 2745 int len, int flags) 2746 { 2747 return -EOPNOTSUPP; 2748 } 2749 EXPORT_SYMBOL(sock_no_connect); 2750 2751 int sock_no_socketpair(struct socket *sock1, struct socket *sock2) 2752 { 2753 return -EOPNOTSUPP; 2754 } 2755 EXPORT_SYMBOL(sock_no_socketpair); 2756 2757 int sock_no_accept(struct socket *sock, struct socket *newsock, int flags, 2758 bool kern) 2759 { 2760 return -EOPNOTSUPP; 2761 } 2762 EXPORT_SYMBOL(sock_no_accept); 2763 2764 int sock_no_getname(struct socket *sock, struct sockaddr *saddr, 2765 int peer) 2766 { 2767 return -EOPNOTSUPP; 2768 } 2769 EXPORT_SYMBOL(sock_no_getname); 2770 2771 int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) 2772 { 2773 return -EOPNOTSUPP; 2774 } 2775 EXPORT_SYMBOL(sock_no_ioctl); 2776 2777 int sock_no_listen(struct socket *sock, int backlog) 2778 { 2779 return -EOPNOTSUPP; 2780 } 2781 EXPORT_SYMBOL(sock_no_listen); 2782 2783 int sock_no_shutdown(struct socket *sock, int how) 2784 { 2785 return -EOPNOTSUPP; 2786 } 2787 EXPORT_SYMBOL(sock_no_shutdown); 2788 2789 int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len) 2790 { 2791 return -EOPNOTSUPP; 2792 } 2793 EXPORT_SYMBOL(sock_no_sendmsg); 2794 2795 int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len) 2796 { 2797 return -EOPNOTSUPP; 2798 } 2799 EXPORT_SYMBOL(sock_no_sendmsg_locked); 2800 2801 int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len, 2802 int flags) 2803 { 2804 return -EOPNOTSUPP; 2805 } 2806 EXPORT_SYMBOL(sock_no_recvmsg); 2807 2808 int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma) 2809 { 2810 /* Mirror missing mmap method error code */ 2811 return -ENODEV; 2812 } 2813 EXPORT_SYMBOL(sock_no_mmap); 2814 2815 /* 2816 * When a file is received (via SCM_RIGHTS, etc), we must bump the 2817 * various sock-based usage counts. 2818 */ 2819 void __receive_sock(struct file *file) 2820 { 2821 struct socket *sock; 2822 int error; 2823 2824 /* 2825 * The resulting value of "error" is ignored here since we only 2826 * need to take action when the file is a socket and testing 2827 * "sock" for NULL is sufficient. 2828 */ 2829 sock = sock_from_file(file, &error); 2830 if (sock) { 2831 sock_update_netprioidx(&sock->sk->sk_cgrp_data); 2832 sock_update_classid(&sock->sk->sk_cgrp_data); 2833 } 2834 } 2835 2836 ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags) 2837 { 2838 ssize_t res; 2839 struct msghdr msg = {.msg_flags = flags}; 2840 struct kvec iov; 2841 char *kaddr = kmap(page); 2842 iov.iov_base = kaddr + offset; 2843 iov.iov_len = size; 2844 res = kernel_sendmsg(sock, &msg, &iov, 1, size); 2845 kunmap(page); 2846 return res; 2847 } 2848 EXPORT_SYMBOL(sock_no_sendpage); 2849 2850 ssize_t sock_no_sendpage_locked(struct sock *sk, struct page *page, 2851 int offset, size_t size, int flags) 2852 { 2853 ssize_t res; 2854 struct msghdr msg = {.msg_flags = flags}; 2855 struct kvec iov; 2856 char *kaddr = kmap(page); 2857 2858 iov.iov_base = kaddr + offset; 2859 iov.iov_len = size; 2860 res = kernel_sendmsg_locked(sk, &msg, &iov, 1, size); 2861 kunmap(page); 2862 return res; 2863 } 2864 EXPORT_SYMBOL(sock_no_sendpage_locked); 2865 2866 /* 2867 * Default Socket Callbacks 2868 */ 2869 2870 static void sock_def_wakeup(struct sock *sk) 2871 { 2872 struct socket_wq *wq; 2873 2874 rcu_read_lock(); 2875 wq = rcu_dereference(sk->sk_wq); 2876 if (skwq_has_sleeper(wq)) 2877 wake_up_interruptible_all(&wq->wait); 2878 rcu_read_unlock(); 2879 } 2880 2881 static void sock_def_error_report(struct sock *sk) 2882 { 2883 struct socket_wq *wq; 2884 2885 rcu_read_lock(); 2886 wq = rcu_dereference(sk->sk_wq); 2887 if (skwq_has_sleeper(wq)) 2888 wake_up_interruptible_poll(&wq->wait, EPOLLERR); 2889 sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR); 2890 rcu_read_unlock(); 2891 } 2892 2893 void sock_def_readable(struct sock *sk) 2894 { 2895 struct socket_wq *wq; 2896 2897 rcu_read_lock(); 2898 wq = rcu_dereference(sk->sk_wq); 2899 if (skwq_has_sleeper(wq)) 2900 wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI | 2901 EPOLLRDNORM | EPOLLRDBAND); 2902 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 2903 rcu_read_unlock(); 2904 } 2905 2906 static void sock_def_write_space(struct sock *sk) 2907 { 2908 struct socket_wq *wq; 2909 2910 rcu_read_lock(); 2911 2912 /* Do not wake up a writer until he can make "significant" 2913 * progress. --DaveM 2914 */ 2915 if ((refcount_read(&sk->sk_wmem_alloc) << 1) <= READ_ONCE(sk->sk_sndbuf)) { 2916 wq = rcu_dereference(sk->sk_wq); 2917 if (skwq_has_sleeper(wq)) 2918 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 2919 EPOLLWRNORM | EPOLLWRBAND); 2920 2921 /* Should agree with poll, otherwise some programs break */ 2922 if (sock_writeable(sk)) 2923 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); 2924 } 2925 2926 rcu_read_unlock(); 2927 } 2928 2929 static void sock_def_destruct(struct sock *sk) 2930 { 2931 } 2932 2933 void sk_send_sigurg(struct sock *sk) 2934 { 2935 if (sk->sk_socket && sk->sk_socket->file) 2936 if (send_sigurg(&sk->sk_socket->file->f_owner)) 2937 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI); 2938 } 2939 EXPORT_SYMBOL(sk_send_sigurg); 2940 2941 void sk_reset_timer(struct sock *sk, struct timer_list* timer, 2942 unsigned long expires) 2943 { 2944 if (!mod_timer(timer, expires)) 2945 sock_hold(sk); 2946 } 2947 EXPORT_SYMBOL(sk_reset_timer); 2948 2949 void sk_stop_timer(struct sock *sk, struct timer_list* timer) 2950 { 2951 if (del_timer(timer)) 2952 __sock_put(sk); 2953 } 2954 EXPORT_SYMBOL(sk_stop_timer); 2955 2956 void sock_init_data(struct socket *sock, struct sock *sk) 2957 { 2958 sk_init_common(sk); 2959 sk->sk_send_head = NULL; 2960 2961 timer_setup(&sk->sk_timer, NULL, 0); 2962 2963 sk->sk_allocation = GFP_KERNEL; 2964 sk->sk_rcvbuf = sysctl_rmem_default; 2965 sk->sk_sndbuf = sysctl_wmem_default; 2966 sk->sk_state = TCP_CLOSE; 2967 sk_set_socket(sk, sock); 2968 2969 sock_set_flag(sk, SOCK_ZAPPED); 2970 2971 if (sock) { 2972 sk->sk_type = sock->type; 2973 RCU_INIT_POINTER(sk->sk_wq, &sock->wq); 2974 sock->sk = sk; 2975 sk->sk_uid = SOCK_INODE(sock)->i_uid; 2976 } else { 2977 RCU_INIT_POINTER(sk->sk_wq, NULL); 2978 sk->sk_uid = make_kuid(sock_net(sk)->user_ns, 0); 2979 } 2980 2981 rwlock_init(&sk->sk_callback_lock); 2982 if (sk->sk_kern_sock) 2983 lockdep_set_class_and_name( 2984 &sk->sk_callback_lock, 2985 af_kern_callback_keys + sk->sk_family, 2986 af_family_kern_clock_key_strings[sk->sk_family]); 2987 else 2988 lockdep_set_class_and_name( 2989 &sk->sk_callback_lock, 2990 af_callback_keys + sk->sk_family, 2991 af_family_clock_key_strings[sk->sk_family]); 2992 2993 sk->sk_state_change = sock_def_wakeup; 2994 sk->sk_data_ready = sock_def_readable; 2995 sk->sk_write_space = sock_def_write_space; 2996 sk->sk_error_report = sock_def_error_report; 2997 sk->sk_destruct = sock_def_destruct; 2998 2999 sk->sk_frag.page = NULL; 3000 sk->sk_frag.offset = 0; 3001 sk->sk_peek_off = -1; 3002 3003 sk->sk_peer_pid = NULL; 3004 sk->sk_peer_cred = NULL; 3005 sk->sk_write_pending = 0; 3006 sk->sk_rcvlowat = 1; 3007 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT; 3008 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT; 3009 3010 sk->sk_stamp = SK_DEFAULT_STAMP; 3011 #if BITS_PER_LONG==32 3012 seqlock_init(&sk->sk_stamp_seq); 3013 #endif 3014 atomic_set(&sk->sk_zckey, 0); 3015 3016 #ifdef CONFIG_NET_RX_BUSY_POLL 3017 sk->sk_napi_id = 0; 3018 sk->sk_ll_usec = sysctl_net_busy_read; 3019 #endif 3020 3021 sk->sk_max_pacing_rate = ~0UL; 3022 sk->sk_pacing_rate = ~0UL; 3023 WRITE_ONCE(sk->sk_pacing_shift, 10); 3024 sk->sk_incoming_cpu = -1; 3025 3026 sk_rx_queue_clear(sk); 3027 /* 3028 * Before updating sk_refcnt, we must commit prior changes to memory 3029 * (Documentation/RCU/rculist_nulls.rst for details) 3030 */ 3031 smp_wmb(); 3032 refcount_set(&sk->sk_refcnt, 1); 3033 atomic_set(&sk->sk_drops, 0); 3034 } 3035 EXPORT_SYMBOL(sock_init_data); 3036 3037 void lock_sock_nested(struct sock *sk, int subclass) 3038 { 3039 might_sleep(); 3040 spin_lock_bh(&sk->sk_lock.slock); 3041 if (sk->sk_lock.owned) 3042 __lock_sock(sk); 3043 sk->sk_lock.owned = 1; 3044 spin_unlock(&sk->sk_lock.slock); 3045 /* 3046 * The sk_lock has mutex_lock() semantics here: 3047 */ 3048 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_); 3049 local_bh_enable(); 3050 } 3051 EXPORT_SYMBOL(lock_sock_nested); 3052 3053 void release_sock(struct sock *sk) 3054 { 3055 spin_lock_bh(&sk->sk_lock.slock); 3056 if (sk->sk_backlog.tail) 3057 __release_sock(sk); 3058 3059 /* Warning : release_cb() might need to release sk ownership, 3060 * ie call sock_release_ownership(sk) before us. 3061 */ 3062 if (sk->sk_prot->release_cb) 3063 sk->sk_prot->release_cb(sk); 3064 3065 sock_release_ownership(sk); 3066 if (waitqueue_active(&sk->sk_lock.wq)) 3067 wake_up(&sk->sk_lock.wq); 3068 spin_unlock_bh(&sk->sk_lock.slock); 3069 } 3070 EXPORT_SYMBOL(release_sock); 3071 3072 /** 3073 * lock_sock_fast - fast version of lock_sock 3074 * @sk: socket 3075 * 3076 * This version should be used for very small section, where process wont block 3077 * return false if fast path is taken: 3078 * 3079 * sk_lock.slock locked, owned = 0, BH disabled 3080 * 3081 * return true if slow path is taken: 3082 * 3083 * sk_lock.slock unlocked, owned = 1, BH enabled 3084 */ 3085 bool lock_sock_fast(struct sock *sk) 3086 { 3087 might_sleep(); 3088 spin_lock_bh(&sk->sk_lock.slock); 3089 3090 if (!sk->sk_lock.owned) 3091 /* 3092 * Note : We must disable BH 3093 */ 3094 return false; 3095 3096 __lock_sock(sk); 3097 sk->sk_lock.owned = 1; 3098 spin_unlock(&sk->sk_lock.slock); 3099 /* 3100 * The sk_lock has mutex_lock() semantics here: 3101 */ 3102 mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_); 3103 local_bh_enable(); 3104 return true; 3105 } 3106 EXPORT_SYMBOL(lock_sock_fast); 3107 3108 int sock_gettstamp(struct socket *sock, void __user *userstamp, 3109 bool timeval, bool time32) 3110 { 3111 struct sock *sk = sock->sk; 3112 struct timespec64 ts; 3113 3114 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 3115 ts = ktime_to_timespec64(sock_read_timestamp(sk)); 3116 if (ts.tv_sec == -1) 3117 return -ENOENT; 3118 if (ts.tv_sec == 0) { 3119 ktime_t kt = ktime_get_real(); 3120 sock_write_timestamp(sk, kt); 3121 ts = ktime_to_timespec64(kt); 3122 } 3123 3124 if (timeval) 3125 ts.tv_nsec /= 1000; 3126 3127 #ifdef CONFIG_COMPAT_32BIT_TIME 3128 if (time32) 3129 return put_old_timespec32(&ts, userstamp); 3130 #endif 3131 #ifdef CONFIG_SPARC64 3132 /* beware of padding in sparc64 timeval */ 3133 if (timeval && !in_compat_syscall()) { 3134 struct __kernel_old_timeval __user tv = { 3135 .tv_sec = ts.tv_sec, 3136 .tv_usec = ts.tv_nsec, 3137 }; 3138 if (copy_to_user(userstamp, &tv, sizeof(tv))) 3139 return -EFAULT; 3140 return 0; 3141 } 3142 #endif 3143 return put_timespec64(&ts, userstamp); 3144 } 3145 EXPORT_SYMBOL(sock_gettstamp); 3146 3147 void sock_enable_timestamp(struct sock *sk, enum sock_flags flag) 3148 { 3149 if (!sock_flag(sk, flag)) { 3150 unsigned long previous_flags = sk->sk_flags; 3151 3152 sock_set_flag(sk, flag); 3153 /* 3154 * we just set one of the two flags which require net 3155 * time stamping, but time stamping might have been on 3156 * already because of the other one 3157 */ 3158 if (sock_needs_netstamp(sk) && 3159 !(previous_flags & SK_FLAGS_TIMESTAMP)) 3160 net_enable_timestamp(); 3161 } 3162 } 3163 3164 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, 3165 int level, int type) 3166 { 3167 struct sock_exterr_skb *serr; 3168 struct sk_buff *skb; 3169 int copied, err; 3170 3171 err = -EAGAIN; 3172 skb = sock_dequeue_err_skb(sk); 3173 if (skb == NULL) 3174 goto out; 3175 3176 copied = skb->len; 3177 if (copied > len) { 3178 msg->msg_flags |= MSG_TRUNC; 3179 copied = len; 3180 } 3181 err = skb_copy_datagram_msg(skb, 0, msg, copied); 3182 if (err) 3183 goto out_free_skb; 3184 3185 sock_recv_timestamp(msg, sk, skb); 3186 3187 serr = SKB_EXT_ERR(skb); 3188 put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee); 3189 3190 msg->msg_flags |= MSG_ERRQUEUE; 3191 err = copied; 3192 3193 out_free_skb: 3194 kfree_skb(skb); 3195 out: 3196 return err; 3197 } 3198 EXPORT_SYMBOL(sock_recv_errqueue); 3199 3200 /* 3201 * Get a socket option on an socket. 3202 * 3203 * FIX: POSIX 1003.1g is very ambiguous here. It states that 3204 * asynchronous errors should be reported by getsockopt. We assume 3205 * this means if you specify SO_ERROR (otherwise whats the point of it). 3206 */ 3207 int sock_common_getsockopt(struct socket *sock, int level, int optname, 3208 char __user *optval, int __user *optlen) 3209 { 3210 struct sock *sk = sock->sk; 3211 3212 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen); 3213 } 3214 EXPORT_SYMBOL(sock_common_getsockopt); 3215 3216 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, 3217 int flags) 3218 { 3219 struct sock *sk = sock->sk; 3220 int addr_len = 0; 3221 int err; 3222 3223 err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT, 3224 flags & ~MSG_DONTWAIT, &addr_len); 3225 if (err >= 0) 3226 msg->msg_namelen = addr_len; 3227 return err; 3228 } 3229 EXPORT_SYMBOL(sock_common_recvmsg); 3230 3231 /* 3232 * Set socket options on an inet socket. 3233 */ 3234 int sock_common_setsockopt(struct socket *sock, int level, int optname, 3235 sockptr_t optval, unsigned int optlen) 3236 { 3237 struct sock *sk = sock->sk; 3238 3239 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen); 3240 } 3241 EXPORT_SYMBOL(sock_common_setsockopt); 3242 3243 void sk_common_release(struct sock *sk) 3244 { 3245 if (sk->sk_prot->destroy) 3246 sk->sk_prot->destroy(sk); 3247 3248 /* 3249 * Observation: when sock_common_release is called, processes have 3250 * no access to socket. But net still has. 3251 * Step one, detach it from networking: 3252 * 3253 * A. Remove from hash tables. 3254 */ 3255 3256 sk->sk_prot->unhash(sk); 3257 3258 /* 3259 * In this point socket cannot receive new packets, but it is possible 3260 * that some packets are in flight because some CPU runs receiver and 3261 * did hash table lookup before we unhashed socket. They will achieve 3262 * receive queue and will be purged by socket destructor. 3263 * 3264 * Also we still have packets pending on receive queue and probably, 3265 * our own packets waiting in device queues. sock_destroy will drain 3266 * receive queue, but transmitted packets will delay socket destruction 3267 * until the last reference will be released. 3268 */ 3269 3270 sock_orphan(sk); 3271 3272 xfrm_sk_free_policy(sk); 3273 3274 sk_refcnt_debug_release(sk); 3275 3276 sock_put(sk); 3277 } 3278 EXPORT_SYMBOL(sk_common_release); 3279 3280 void sk_get_meminfo(const struct sock *sk, u32 *mem) 3281 { 3282 memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS); 3283 3284 mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk); 3285 mem[SK_MEMINFO_RCVBUF] = READ_ONCE(sk->sk_rcvbuf); 3286 mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk); 3287 mem[SK_MEMINFO_SNDBUF] = READ_ONCE(sk->sk_sndbuf); 3288 mem[SK_MEMINFO_FWD_ALLOC] = sk->sk_forward_alloc; 3289 mem[SK_MEMINFO_WMEM_QUEUED] = READ_ONCE(sk->sk_wmem_queued); 3290 mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc); 3291 mem[SK_MEMINFO_BACKLOG] = READ_ONCE(sk->sk_backlog.len); 3292 mem[SK_MEMINFO_DROPS] = atomic_read(&sk->sk_drops); 3293 } 3294 3295 #ifdef CONFIG_PROC_FS 3296 #define PROTO_INUSE_NR 64 /* should be enough for the first time */ 3297 struct prot_inuse { 3298 int val[PROTO_INUSE_NR]; 3299 }; 3300 3301 static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR); 3302 3303 void sock_prot_inuse_add(struct net *net, struct proto *prot, int val) 3304 { 3305 __this_cpu_add(net->core.prot_inuse->val[prot->inuse_idx], val); 3306 } 3307 EXPORT_SYMBOL_GPL(sock_prot_inuse_add); 3308 3309 int sock_prot_inuse_get(struct net *net, struct proto *prot) 3310 { 3311 int cpu, idx = prot->inuse_idx; 3312 int res = 0; 3313 3314 for_each_possible_cpu(cpu) 3315 res += per_cpu_ptr(net->core.prot_inuse, cpu)->val[idx]; 3316 3317 return res >= 0 ? res : 0; 3318 } 3319 EXPORT_SYMBOL_GPL(sock_prot_inuse_get); 3320 3321 static void sock_inuse_add(struct net *net, int val) 3322 { 3323 this_cpu_add(*net->core.sock_inuse, val); 3324 } 3325 3326 int sock_inuse_get(struct net *net) 3327 { 3328 int cpu, res = 0; 3329 3330 for_each_possible_cpu(cpu) 3331 res += *per_cpu_ptr(net->core.sock_inuse, cpu); 3332 3333 return res; 3334 } 3335 3336 EXPORT_SYMBOL_GPL(sock_inuse_get); 3337 3338 static int __net_init sock_inuse_init_net(struct net *net) 3339 { 3340 net->core.prot_inuse = alloc_percpu(struct prot_inuse); 3341 if (net->core.prot_inuse == NULL) 3342 return -ENOMEM; 3343 3344 net->core.sock_inuse = alloc_percpu(int); 3345 if (net->core.sock_inuse == NULL) 3346 goto out; 3347 3348 return 0; 3349 3350 out: 3351 free_percpu(net->core.prot_inuse); 3352 return -ENOMEM; 3353 } 3354 3355 static void __net_exit sock_inuse_exit_net(struct net *net) 3356 { 3357 free_percpu(net->core.prot_inuse); 3358 free_percpu(net->core.sock_inuse); 3359 } 3360 3361 static struct pernet_operations net_inuse_ops = { 3362 .init = sock_inuse_init_net, 3363 .exit = sock_inuse_exit_net, 3364 }; 3365 3366 static __init int net_inuse_init(void) 3367 { 3368 if (register_pernet_subsys(&net_inuse_ops)) 3369 panic("Cannot initialize net inuse counters"); 3370 3371 return 0; 3372 } 3373 3374 core_initcall(net_inuse_init); 3375 3376 static int assign_proto_idx(struct proto *prot) 3377 { 3378 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR); 3379 3380 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) { 3381 pr_err("PROTO_INUSE_NR exhausted\n"); 3382 return -ENOSPC; 3383 } 3384 3385 set_bit(prot->inuse_idx, proto_inuse_idx); 3386 return 0; 3387 } 3388 3389 static void release_proto_idx(struct proto *prot) 3390 { 3391 if (prot->inuse_idx != PROTO_INUSE_NR - 1) 3392 clear_bit(prot->inuse_idx, proto_inuse_idx); 3393 } 3394 #else 3395 static inline int assign_proto_idx(struct proto *prot) 3396 { 3397 return 0; 3398 } 3399 3400 static inline void release_proto_idx(struct proto *prot) 3401 { 3402 } 3403 3404 static void sock_inuse_add(struct net *net, int val) 3405 { 3406 } 3407 #endif 3408 3409 static void req_prot_cleanup(struct request_sock_ops *rsk_prot) 3410 { 3411 if (!rsk_prot) 3412 return; 3413 kfree(rsk_prot->slab_name); 3414 rsk_prot->slab_name = NULL; 3415 kmem_cache_destroy(rsk_prot->slab); 3416 rsk_prot->slab = NULL; 3417 } 3418 3419 static int req_prot_init(const struct proto *prot) 3420 { 3421 struct request_sock_ops *rsk_prot = prot->rsk_prot; 3422 3423 if (!rsk_prot) 3424 return 0; 3425 3426 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s", 3427 prot->name); 3428 if (!rsk_prot->slab_name) 3429 return -ENOMEM; 3430 3431 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name, 3432 rsk_prot->obj_size, 0, 3433 SLAB_ACCOUNT | prot->slab_flags, 3434 NULL); 3435 3436 if (!rsk_prot->slab) { 3437 pr_crit("%s: Can't create request sock SLAB cache!\n", 3438 prot->name); 3439 return -ENOMEM; 3440 } 3441 return 0; 3442 } 3443 3444 int proto_register(struct proto *prot, int alloc_slab) 3445 { 3446 int ret = -ENOBUFS; 3447 3448 if (alloc_slab) { 3449 prot->slab = kmem_cache_create_usercopy(prot->name, 3450 prot->obj_size, 0, 3451 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT | 3452 prot->slab_flags, 3453 prot->useroffset, prot->usersize, 3454 NULL); 3455 3456 if (prot->slab == NULL) { 3457 pr_crit("%s: Can't create sock SLAB cache!\n", 3458 prot->name); 3459 goto out; 3460 } 3461 3462 if (req_prot_init(prot)) 3463 goto out_free_request_sock_slab; 3464 3465 if (prot->twsk_prot != NULL) { 3466 prot->twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", prot->name); 3467 3468 if (prot->twsk_prot->twsk_slab_name == NULL) 3469 goto out_free_request_sock_slab; 3470 3471 prot->twsk_prot->twsk_slab = 3472 kmem_cache_create(prot->twsk_prot->twsk_slab_name, 3473 prot->twsk_prot->twsk_obj_size, 3474 0, 3475 SLAB_ACCOUNT | 3476 prot->slab_flags, 3477 NULL); 3478 if (prot->twsk_prot->twsk_slab == NULL) 3479 goto out_free_timewait_sock_slab_name; 3480 } 3481 } 3482 3483 mutex_lock(&proto_list_mutex); 3484 ret = assign_proto_idx(prot); 3485 if (ret) { 3486 mutex_unlock(&proto_list_mutex); 3487 goto out_free_timewait_sock_slab_name; 3488 } 3489 list_add(&prot->node, &proto_list); 3490 mutex_unlock(&proto_list_mutex); 3491 return ret; 3492 3493 out_free_timewait_sock_slab_name: 3494 if (alloc_slab && prot->twsk_prot) 3495 kfree(prot->twsk_prot->twsk_slab_name); 3496 out_free_request_sock_slab: 3497 if (alloc_slab) { 3498 req_prot_cleanup(prot->rsk_prot); 3499 3500 kmem_cache_destroy(prot->slab); 3501 prot->slab = NULL; 3502 } 3503 out: 3504 return ret; 3505 } 3506 EXPORT_SYMBOL(proto_register); 3507 3508 void proto_unregister(struct proto *prot) 3509 { 3510 mutex_lock(&proto_list_mutex); 3511 release_proto_idx(prot); 3512 list_del(&prot->node); 3513 mutex_unlock(&proto_list_mutex); 3514 3515 kmem_cache_destroy(prot->slab); 3516 prot->slab = NULL; 3517 3518 req_prot_cleanup(prot->rsk_prot); 3519 3520 if (prot->twsk_prot != NULL && prot->twsk_prot->twsk_slab != NULL) { 3521 kmem_cache_destroy(prot->twsk_prot->twsk_slab); 3522 kfree(prot->twsk_prot->twsk_slab_name); 3523 prot->twsk_prot->twsk_slab = NULL; 3524 } 3525 } 3526 EXPORT_SYMBOL(proto_unregister); 3527 3528 int sock_load_diag_module(int family, int protocol) 3529 { 3530 if (!protocol) { 3531 if (!sock_is_registered(family)) 3532 return -ENOENT; 3533 3534 return request_module("net-pf-%d-proto-%d-type-%d", PF_NETLINK, 3535 NETLINK_SOCK_DIAG, family); 3536 } 3537 3538 #ifdef CONFIG_INET 3539 if (family == AF_INET && 3540 protocol != IPPROTO_RAW && 3541 protocol < MAX_INET_PROTOS && 3542 !rcu_access_pointer(inet_protos[protocol])) 3543 return -ENOENT; 3544 #endif 3545 3546 return request_module("net-pf-%d-proto-%d-type-%d-%d", PF_NETLINK, 3547 NETLINK_SOCK_DIAG, family, protocol); 3548 } 3549 EXPORT_SYMBOL(sock_load_diag_module); 3550 3551 #ifdef CONFIG_PROC_FS 3552 static void *proto_seq_start(struct seq_file *seq, loff_t *pos) 3553 __acquires(proto_list_mutex) 3554 { 3555 mutex_lock(&proto_list_mutex); 3556 return seq_list_start_head(&proto_list, *pos); 3557 } 3558 3559 static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos) 3560 { 3561 return seq_list_next(v, &proto_list, pos); 3562 } 3563 3564 static void proto_seq_stop(struct seq_file *seq, void *v) 3565 __releases(proto_list_mutex) 3566 { 3567 mutex_unlock(&proto_list_mutex); 3568 } 3569 3570 static char proto_method_implemented(const void *method) 3571 { 3572 return method == NULL ? 'n' : 'y'; 3573 } 3574 static long sock_prot_memory_allocated(struct proto *proto) 3575 { 3576 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L; 3577 } 3578 3579 static const char *sock_prot_memory_pressure(struct proto *proto) 3580 { 3581 return proto->memory_pressure != NULL ? 3582 proto_memory_pressure(proto) ? "yes" : "no" : "NI"; 3583 } 3584 3585 static void proto_seq_printf(struct seq_file *seq, struct proto *proto) 3586 { 3587 3588 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s " 3589 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n", 3590 proto->name, 3591 proto->obj_size, 3592 sock_prot_inuse_get(seq_file_net(seq), proto), 3593 sock_prot_memory_allocated(proto), 3594 sock_prot_memory_pressure(proto), 3595 proto->max_header, 3596 proto->slab == NULL ? "no" : "yes", 3597 module_name(proto->owner), 3598 proto_method_implemented(proto->close), 3599 proto_method_implemented(proto->connect), 3600 proto_method_implemented(proto->disconnect), 3601 proto_method_implemented(proto->accept), 3602 proto_method_implemented(proto->ioctl), 3603 proto_method_implemented(proto->init), 3604 proto_method_implemented(proto->destroy), 3605 proto_method_implemented(proto->shutdown), 3606 proto_method_implemented(proto->setsockopt), 3607 proto_method_implemented(proto->getsockopt), 3608 proto_method_implemented(proto->sendmsg), 3609 proto_method_implemented(proto->recvmsg), 3610 proto_method_implemented(proto->sendpage), 3611 proto_method_implemented(proto->bind), 3612 proto_method_implemented(proto->backlog_rcv), 3613 proto_method_implemented(proto->hash), 3614 proto_method_implemented(proto->unhash), 3615 proto_method_implemented(proto->get_port), 3616 proto_method_implemented(proto->enter_memory_pressure)); 3617 } 3618 3619 static int proto_seq_show(struct seq_file *seq, void *v) 3620 { 3621 if (v == &proto_list) 3622 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s", 3623 "protocol", 3624 "size", 3625 "sockets", 3626 "memory", 3627 "press", 3628 "maxhdr", 3629 "slab", 3630 "module", 3631 "cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n"); 3632 else 3633 proto_seq_printf(seq, list_entry(v, struct proto, node)); 3634 return 0; 3635 } 3636 3637 static const struct seq_operations proto_seq_ops = { 3638 .start = proto_seq_start, 3639 .next = proto_seq_next, 3640 .stop = proto_seq_stop, 3641 .show = proto_seq_show, 3642 }; 3643 3644 static __net_init int proto_init_net(struct net *net) 3645 { 3646 if (!proc_create_net("protocols", 0444, net->proc_net, &proto_seq_ops, 3647 sizeof(struct seq_net_private))) 3648 return -ENOMEM; 3649 3650 return 0; 3651 } 3652 3653 static __net_exit void proto_exit_net(struct net *net) 3654 { 3655 remove_proc_entry("protocols", net->proc_net); 3656 } 3657 3658 3659 static __net_initdata struct pernet_operations proto_net_ops = { 3660 .init = proto_init_net, 3661 .exit = proto_exit_net, 3662 }; 3663 3664 static int __init proto_init(void) 3665 { 3666 return register_pernet_subsys(&proto_net_ops); 3667 } 3668 3669 subsys_initcall(proto_init); 3670 3671 #endif /* PROC_FS */ 3672 3673 #ifdef CONFIG_NET_RX_BUSY_POLL 3674 bool sk_busy_loop_end(void *p, unsigned long start_time) 3675 { 3676 struct sock *sk = p; 3677 3678 return !skb_queue_empty_lockless(&sk->sk_receive_queue) || 3679 sk_busy_loop_timeout(sk, start_time); 3680 } 3681 EXPORT_SYMBOL(sk_busy_loop_end); 3682 #endif /* CONFIG_NET_RX_BUSY_POLL */ 3683 3684 int sock_bind_add(struct sock *sk, struct sockaddr *addr, int addr_len) 3685 { 3686 if (!sk->sk_prot->bind_add) 3687 return -EOPNOTSUPP; 3688 return sk->sk_prot->bind_add(sk, addr, addr_len); 3689 } 3690 EXPORT_SYMBOL(sock_bind_add); 3691