1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * Generic socket support routines. Memory allocators, socket lock/release 8 * handler for protocols to use and generic option handler. 9 * 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Florian La Roche, <flla@stud.uni-sb.de> 13 * Alan Cox, <A.Cox@swansea.ac.uk> 14 * 15 * Fixes: 16 * Alan Cox : Numerous verify_area() problems 17 * Alan Cox : Connecting on a connecting socket 18 * now returns an error for tcp. 19 * Alan Cox : sock->protocol is set correctly. 20 * and is not sometimes left as 0. 21 * Alan Cox : connect handles icmp errors on a 22 * connect properly. Unfortunately there 23 * is a restart syscall nasty there. I 24 * can't match BSD without hacking the C 25 * library. Ideas urgently sought! 26 * Alan Cox : Disallow bind() to addresses that are 27 * not ours - especially broadcast ones!! 28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost) 29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets, 30 * instead they leave that for the DESTROY timer. 31 * Alan Cox : Clean up error flag in accept 32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer 33 * was buggy. Put a remove_sock() in the handler 34 * for memory when we hit 0. Also altered the timer 35 * code. The ACK stuff can wait and needs major 36 * TCP layer surgery. 37 * Alan Cox : Fixed TCP ack bug, removed remove sock 38 * and fixed timer/inet_bh race. 39 * Alan Cox : Added zapped flag for TCP 40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code 41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb 42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources 43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing. 44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so... 45 * Rick Sladkey : Relaxed UDP rules for matching packets. 46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support 47 * Pauline Middelink : identd support 48 * Alan Cox : Fixed connect() taking signals I think. 49 * Alan Cox : SO_LINGER supported 50 * Alan Cox : Error reporting fixes 51 * Anonymous : inet_create tidied up (sk->reuse setting) 52 * Alan Cox : inet sockets don't set sk->type! 53 * Alan Cox : Split socket option code 54 * Alan Cox : Callbacks 55 * Alan Cox : Nagle flag for Charles & Johannes stuff 56 * Alex : Removed restriction on inet fioctl 57 * Alan Cox : Splitting INET from NET core 58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt() 59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code 60 * Alan Cox : Split IP from generic code 61 * Alan Cox : New kfree_skbmem() 62 * Alan Cox : Make SO_DEBUG superuser only. 63 * Alan Cox : Allow anyone to clear SO_DEBUG 64 * (compatibility fix) 65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput. 66 * Alan Cox : Allocator for a socket is settable. 67 * Alan Cox : SO_ERROR includes soft errors. 68 * Alan Cox : Allow NULL arguments on some SO_ opts 69 * Alan Cox : Generic socket allocation to make hooks 70 * easier (suggested by Craig Metz). 71 * Michael Pall : SO_ERROR returns positive errno again 72 * Steve Whitehouse: Added default destructor to free 73 * protocol private data. 74 * Steve Whitehouse: Added various other default routines 75 * common to several socket families. 76 * Chris Evans : Call suser() check last on F_SETOWN 77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER. 78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s() 79 * Andi Kleen : Fix write_space callback 80 * Chris Evans : Security fixes - signedness again 81 * Arnaldo C. Melo : cleanups, use skb_queue_purge 82 * 83 * To Fix: 84 */ 85 86 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 87 88 #include <asm/unaligned.h> 89 #include <linux/capability.h> 90 #include <linux/errno.h> 91 #include <linux/errqueue.h> 92 #include <linux/types.h> 93 #include <linux/socket.h> 94 #include <linux/in.h> 95 #include <linux/kernel.h> 96 #include <linux/module.h> 97 #include <linux/proc_fs.h> 98 #include <linux/seq_file.h> 99 #include <linux/sched.h> 100 #include <linux/sched/mm.h> 101 #include <linux/timer.h> 102 #include <linux/string.h> 103 #include <linux/sockios.h> 104 #include <linux/net.h> 105 #include <linux/mm.h> 106 #include <linux/slab.h> 107 #include <linux/interrupt.h> 108 #include <linux/poll.h> 109 #include <linux/tcp.h> 110 #include <linux/init.h> 111 #include <linux/highmem.h> 112 #include <linux/user_namespace.h> 113 #include <linux/static_key.h> 114 #include <linux/memcontrol.h> 115 #include <linux/prefetch.h> 116 #include <linux/compat.h> 117 #include <linux/mroute.h> 118 #include <linux/mroute6.h> 119 #include <linux/icmpv6.h> 120 121 #include <linux/uaccess.h> 122 123 #include <linux/netdevice.h> 124 #include <net/protocol.h> 125 #include <linux/skbuff.h> 126 #include <net/net_namespace.h> 127 #include <net/request_sock.h> 128 #include <net/sock.h> 129 #include <linux/net_tstamp.h> 130 #include <net/xfrm.h> 131 #include <linux/ipsec.h> 132 #include <net/cls_cgroup.h> 133 #include <net/netprio_cgroup.h> 134 #include <linux/sock_diag.h> 135 136 #include <linux/filter.h> 137 #include <net/sock_reuseport.h> 138 #include <net/bpf_sk_storage.h> 139 140 #include <trace/events/sock.h> 141 142 #include <net/tcp.h> 143 #include <net/busy_poll.h> 144 #include <net/phonet/phonet.h> 145 146 #include <linux/ethtool.h> 147 148 #include "dev.h" 149 150 static DEFINE_MUTEX(proto_list_mutex); 151 static LIST_HEAD(proto_list); 152 153 static void sock_def_write_space_wfree(struct sock *sk); 154 static void sock_def_write_space(struct sock *sk); 155 156 /** 157 * sk_ns_capable - General socket capability test 158 * @sk: Socket to use a capability on or through 159 * @user_ns: The user namespace of the capability to use 160 * @cap: The capability to use 161 * 162 * Test to see if the opener of the socket had when the socket was 163 * created and the current process has the capability @cap in the user 164 * namespace @user_ns. 165 */ 166 bool sk_ns_capable(const struct sock *sk, 167 struct user_namespace *user_ns, int cap) 168 { 169 return file_ns_capable(sk->sk_socket->file, user_ns, cap) && 170 ns_capable(user_ns, cap); 171 } 172 EXPORT_SYMBOL(sk_ns_capable); 173 174 /** 175 * sk_capable - Socket global capability test 176 * @sk: Socket to use a capability on or through 177 * @cap: The global capability to use 178 * 179 * Test to see if the opener of the socket had when the socket was 180 * created and the current process has the capability @cap in all user 181 * namespaces. 182 */ 183 bool sk_capable(const struct sock *sk, int cap) 184 { 185 return sk_ns_capable(sk, &init_user_ns, cap); 186 } 187 EXPORT_SYMBOL(sk_capable); 188 189 /** 190 * sk_net_capable - Network namespace socket capability test 191 * @sk: Socket to use a capability on or through 192 * @cap: The capability to use 193 * 194 * Test to see if the opener of the socket had when the socket was created 195 * and the current process has the capability @cap over the network namespace 196 * the socket is a member of. 197 */ 198 bool sk_net_capable(const struct sock *sk, int cap) 199 { 200 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap); 201 } 202 EXPORT_SYMBOL(sk_net_capable); 203 204 /* 205 * Each address family might have different locking rules, so we have 206 * one slock key per address family and separate keys for internal and 207 * userspace sockets. 208 */ 209 static struct lock_class_key af_family_keys[AF_MAX]; 210 static struct lock_class_key af_family_kern_keys[AF_MAX]; 211 static struct lock_class_key af_family_slock_keys[AF_MAX]; 212 static struct lock_class_key af_family_kern_slock_keys[AF_MAX]; 213 214 /* 215 * Make lock validator output more readable. (we pre-construct these 216 * strings build-time, so that runtime initialization of socket 217 * locks is fast): 218 */ 219 220 #define _sock_locks(x) \ 221 x "AF_UNSPEC", x "AF_UNIX" , x "AF_INET" , \ 222 x "AF_AX25" , x "AF_IPX" , x "AF_APPLETALK", \ 223 x "AF_NETROM", x "AF_BRIDGE" , x "AF_ATMPVC" , \ 224 x "AF_X25" , x "AF_INET6" , x "AF_ROSE" , \ 225 x "AF_DECnet", x "AF_NETBEUI" , x "AF_SECURITY" , \ 226 x "AF_KEY" , x "AF_NETLINK" , x "AF_PACKET" , \ 227 x "AF_ASH" , x "AF_ECONET" , x "AF_ATMSVC" , \ 228 x "AF_RDS" , x "AF_SNA" , x "AF_IRDA" , \ 229 x "AF_PPPOX" , x "AF_WANPIPE" , x "AF_LLC" , \ 230 x "27" , x "28" , x "AF_CAN" , \ 231 x "AF_TIPC" , x "AF_BLUETOOTH", x "IUCV" , \ 232 x "AF_RXRPC" , x "AF_ISDN" , x "AF_PHONET" , \ 233 x "AF_IEEE802154", x "AF_CAIF" , x "AF_ALG" , \ 234 x "AF_NFC" , x "AF_VSOCK" , x "AF_KCM" , \ 235 x "AF_QIPCRTR", x "AF_SMC" , x "AF_XDP" , \ 236 x "AF_MCTP" , \ 237 x "AF_MAX" 238 239 static const char *const af_family_key_strings[AF_MAX+1] = { 240 _sock_locks("sk_lock-") 241 }; 242 static const char *const af_family_slock_key_strings[AF_MAX+1] = { 243 _sock_locks("slock-") 244 }; 245 static const char *const af_family_clock_key_strings[AF_MAX+1] = { 246 _sock_locks("clock-") 247 }; 248 249 static const char *const af_family_kern_key_strings[AF_MAX+1] = { 250 _sock_locks("k-sk_lock-") 251 }; 252 static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = { 253 _sock_locks("k-slock-") 254 }; 255 static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = { 256 _sock_locks("k-clock-") 257 }; 258 static const char *const af_family_rlock_key_strings[AF_MAX+1] = { 259 _sock_locks("rlock-") 260 }; 261 static const char *const af_family_wlock_key_strings[AF_MAX+1] = { 262 _sock_locks("wlock-") 263 }; 264 static const char *const af_family_elock_key_strings[AF_MAX+1] = { 265 _sock_locks("elock-") 266 }; 267 268 /* 269 * sk_callback_lock and sk queues locking rules are per-address-family, 270 * so split the lock classes by using a per-AF key: 271 */ 272 static struct lock_class_key af_callback_keys[AF_MAX]; 273 static struct lock_class_key af_rlock_keys[AF_MAX]; 274 static struct lock_class_key af_wlock_keys[AF_MAX]; 275 static struct lock_class_key af_elock_keys[AF_MAX]; 276 static struct lock_class_key af_kern_callback_keys[AF_MAX]; 277 278 /* Run time adjustable parameters. */ 279 __u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX; 280 EXPORT_SYMBOL(sysctl_wmem_max); 281 __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX; 282 EXPORT_SYMBOL(sysctl_rmem_max); 283 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX; 284 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX; 285 286 /* Maximal space eaten by iovec or ancillary data plus some space */ 287 int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512); 288 EXPORT_SYMBOL(sysctl_optmem_max); 289 290 int sysctl_tstamp_allow_data __read_mostly = 1; 291 292 DEFINE_STATIC_KEY_FALSE(memalloc_socks_key); 293 EXPORT_SYMBOL_GPL(memalloc_socks_key); 294 295 /** 296 * sk_set_memalloc - sets %SOCK_MEMALLOC 297 * @sk: socket to set it on 298 * 299 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves. 300 * It's the responsibility of the admin to adjust min_free_kbytes 301 * to meet the requirements 302 */ 303 void sk_set_memalloc(struct sock *sk) 304 { 305 sock_set_flag(sk, SOCK_MEMALLOC); 306 sk->sk_allocation |= __GFP_MEMALLOC; 307 static_branch_inc(&memalloc_socks_key); 308 } 309 EXPORT_SYMBOL_GPL(sk_set_memalloc); 310 311 void sk_clear_memalloc(struct sock *sk) 312 { 313 sock_reset_flag(sk, SOCK_MEMALLOC); 314 sk->sk_allocation &= ~__GFP_MEMALLOC; 315 static_branch_dec(&memalloc_socks_key); 316 317 /* 318 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward 319 * progress of swapping. SOCK_MEMALLOC may be cleared while 320 * it has rmem allocations due to the last swapfile being deactivated 321 * but there is a risk that the socket is unusable due to exceeding 322 * the rmem limits. Reclaim the reserves and obey rmem limits again. 323 */ 324 sk_mem_reclaim(sk); 325 } 326 EXPORT_SYMBOL_GPL(sk_clear_memalloc); 327 328 int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb) 329 { 330 int ret; 331 unsigned int noreclaim_flag; 332 333 /* these should have been dropped before queueing */ 334 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC)); 335 336 noreclaim_flag = memalloc_noreclaim_save(); 337 ret = INDIRECT_CALL_INET(sk->sk_backlog_rcv, 338 tcp_v6_do_rcv, 339 tcp_v4_do_rcv, 340 sk, skb); 341 memalloc_noreclaim_restore(noreclaim_flag); 342 343 return ret; 344 } 345 EXPORT_SYMBOL(__sk_backlog_rcv); 346 347 void sk_error_report(struct sock *sk) 348 { 349 sk->sk_error_report(sk); 350 351 switch (sk->sk_family) { 352 case AF_INET: 353 fallthrough; 354 case AF_INET6: 355 trace_inet_sk_error_report(sk); 356 break; 357 default: 358 break; 359 } 360 } 361 EXPORT_SYMBOL(sk_error_report); 362 363 int sock_get_timeout(long timeo, void *optval, bool old_timeval) 364 { 365 struct __kernel_sock_timeval tv; 366 367 if (timeo == MAX_SCHEDULE_TIMEOUT) { 368 tv.tv_sec = 0; 369 tv.tv_usec = 0; 370 } else { 371 tv.tv_sec = timeo / HZ; 372 tv.tv_usec = ((timeo % HZ) * USEC_PER_SEC) / HZ; 373 } 374 375 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) { 376 struct old_timeval32 tv32 = { tv.tv_sec, tv.tv_usec }; 377 *(struct old_timeval32 *)optval = tv32; 378 return sizeof(tv32); 379 } 380 381 if (old_timeval) { 382 struct __kernel_old_timeval old_tv; 383 old_tv.tv_sec = tv.tv_sec; 384 old_tv.tv_usec = tv.tv_usec; 385 *(struct __kernel_old_timeval *)optval = old_tv; 386 return sizeof(old_tv); 387 } 388 389 *(struct __kernel_sock_timeval *)optval = tv; 390 return sizeof(tv); 391 } 392 EXPORT_SYMBOL(sock_get_timeout); 393 394 int sock_copy_user_timeval(struct __kernel_sock_timeval *tv, 395 sockptr_t optval, int optlen, bool old_timeval) 396 { 397 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) { 398 struct old_timeval32 tv32; 399 400 if (optlen < sizeof(tv32)) 401 return -EINVAL; 402 403 if (copy_from_sockptr(&tv32, optval, sizeof(tv32))) 404 return -EFAULT; 405 tv->tv_sec = tv32.tv_sec; 406 tv->tv_usec = tv32.tv_usec; 407 } else if (old_timeval) { 408 struct __kernel_old_timeval old_tv; 409 410 if (optlen < sizeof(old_tv)) 411 return -EINVAL; 412 if (copy_from_sockptr(&old_tv, optval, sizeof(old_tv))) 413 return -EFAULT; 414 tv->tv_sec = old_tv.tv_sec; 415 tv->tv_usec = old_tv.tv_usec; 416 } else { 417 if (optlen < sizeof(*tv)) 418 return -EINVAL; 419 if (copy_from_sockptr(tv, optval, sizeof(*tv))) 420 return -EFAULT; 421 } 422 423 return 0; 424 } 425 EXPORT_SYMBOL(sock_copy_user_timeval); 426 427 static int sock_set_timeout(long *timeo_p, sockptr_t optval, int optlen, 428 bool old_timeval) 429 { 430 struct __kernel_sock_timeval tv; 431 int err = sock_copy_user_timeval(&tv, optval, optlen, old_timeval); 432 long val; 433 434 if (err) 435 return err; 436 437 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC) 438 return -EDOM; 439 440 if (tv.tv_sec < 0) { 441 static int warned __read_mostly; 442 443 WRITE_ONCE(*timeo_p, 0); 444 if (warned < 10 && net_ratelimit()) { 445 warned++; 446 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n", 447 __func__, current->comm, task_pid_nr(current)); 448 } 449 return 0; 450 } 451 val = MAX_SCHEDULE_TIMEOUT; 452 if ((tv.tv_sec || tv.tv_usec) && 453 (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT / HZ - 1))) 454 val = tv.tv_sec * HZ + DIV_ROUND_UP((unsigned long)tv.tv_usec, 455 USEC_PER_SEC / HZ); 456 WRITE_ONCE(*timeo_p, val); 457 return 0; 458 } 459 460 static bool sock_needs_netstamp(const struct sock *sk) 461 { 462 switch (sk->sk_family) { 463 case AF_UNSPEC: 464 case AF_UNIX: 465 return false; 466 default: 467 return true; 468 } 469 } 470 471 static void sock_disable_timestamp(struct sock *sk, unsigned long flags) 472 { 473 if (sk->sk_flags & flags) { 474 sk->sk_flags &= ~flags; 475 if (sock_needs_netstamp(sk) && 476 !(sk->sk_flags & SK_FLAGS_TIMESTAMP)) 477 net_disable_timestamp(); 478 } 479 } 480 481 482 int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 483 { 484 unsigned long flags; 485 struct sk_buff_head *list = &sk->sk_receive_queue; 486 487 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) { 488 atomic_inc(&sk->sk_drops); 489 trace_sock_rcvqueue_full(sk, skb); 490 return -ENOMEM; 491 } 492 493 if (!sk_rmem_schedule(sk, skb, skb->truesize)) { 494 atomic_inc(&sk->sk_drops); 495 return -ENOBUFS; 496 } 497 498 skb->dev = NULL; 499 skb_set_owner_r(skb, sk); 500 501 /* we escape from rcu protected region, make sure we dont leak 502 * a norefcounted dst 503 */ 504 skb_dst_force(skb); 505 506 spin_lock_irqsave(&list->lock, flags); 507 sock_skb_set_dropcount(sk, skb); 508 __skb_queue_tail(list, skb); 509 spin_unlock_irqrestore(&list->lock, flags); 510 511 if (!sock_flag(sk, SOCK_DEAD)) 512 sk->sk_data_ready(sk); 513 return 0; 514 } 515 EXPORT_SYMBOL(__sock_queue_rcv_skb); 516 517 int sock_queue_rcv_skb_reason(struct sock *sk, struct sk_buff *skb, 518 enum skb_drop_reason *reason) 519 { 520 enum skb_drop_reason drop_reason; 521 int err; 522 523 err = sk_filter(sk, skb); 524 if (err) { 525 drop_reason = SKB_DROP_REASON_SOCKET_FILTER; 526 goto out; 527 } 528 err = __sock_queue_rcv_skb(sk, skb); 529 switch (err) { 530 case -ENOMEM: 531 drop_reason = SKB_DROP_REASON_SOCKET_RCVBUFF; 532 break; 533 case -ENOBUFS: 534 drop_reason = SKB_DROP_REASON_PROTO_MEM; 535 break; 536 default: 537 drop_reason = SKB_NOT_DROPPED_YET; 538 break; 539 } 540 out: 541 if (reason) 542 *reason = drop_reason; 543 return err; 544 } 545 EXPORT_SYMBOL(sock_queue_rcv_skb_reason); 546 547 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, 548 const int nested, unsigned int trim_cap, bool refcounted) 549 { 550 int rc = NET_RX_SUCCESS; 551 552 if (sk_filter_trim_cap(sk, skb, trim_cap)) 553 goto discard_and_relse; 554 555 skb->dev = NULL; 556 557 if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) { 558 atomic_inc(&sk->sk_drops); 559 goto discard_and_relse; 560 } 561 if (nested) 562 bh_lock_sock_nested(sk); 563 else 564 bh_lock_sock(sk); 565 if (!sock_owned_by_user(sk)) { 566 /* 567 * trylock + unlock semantics: 568 */ 569 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_); 570 571 rc = sk_backlog_rcv(sk, skb); 572 573 mutex_release(&sk->sk_lock.dep_map, _RET_IP_); 574 } else if (sk_add_backlog(sk, skb, READ_ONCE(sk->sk_rcvbuf))) { 575 bh_unlock_sock(sk); 576 atomic_inc(&sk->sk_drops); 577 goto discard_and_relse; 578 } 579 580 bh_unlock_sock(sk); 581 out: 582 if (refcounted) 583 sock_put(sk); 584 return rc; 585 discard_and_relse: 586 kfree_skb(skb); 587 goto out; 588 } 589 EXPORT_SYMBOL(__sk_receive_skb); 590 591 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ip6_dst_check(struct dst_entry *, 592 u32)); 593 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *, 594 u32)); 595 struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie) 596 { 597 struct dst_entry *dst = __sk_dst_get(sk); 598 599 if (dst && dst->obsolete && 600 INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check, 601 dst, cookie) == NULL) { 602 sk_tx_queue_clear(sk); 603 WRITE_ONCE(sk->sk_dst_pending_confirm, 0); 604 RCU_INIT_POINTER(sk->sk_dst_cache, NULL); 605 dst_release(dst); 606 return NULL; 607 } 608 609 return dst; 610 } 611 EXPORT_SYMBOL(__sk_dst_check); 612 613 struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie) 614 { 615 struct dst_entry *dst = sk_dst_get(sk); 616 617 if (dst && dst->obsolete && 618 INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check, 619 dst, cookie) == NULL) { 620 sk_dst_reset(sk); 621 dst_release(dst); 622 return NULL; 623 } 624 625 return dst; 626 } 627 EXPORT_SYMBOL(sk_dst_check); 628 629 static int sock_bindtoindex_locked(struct sock *sk, int ifindex) 630 { 631 int ret = -ENOPROTOOPT; 632 #ifdef CONFIG_NETDEVICES 633 struct net *net = sock_net(sk); 634 635 /* Sorry... */ 636 ret = -EPERM; 637 if (sk->sk_bound_dev_if && !ns_capable(net->user_ns, CAP_NET_RAW)) 638 goto out; 639 640 ret = -EINVAL; 641 if (ifindex < 0) 642 goto out; 643 644 /* Paired with all READ_ONCE() done locklessly. */ 645 WRITE_ONCE(sk->sk_bound_dev_if, ifindex); 646 647 if (sk->sk_prot->rehash) 648 sk->sk_prot->rehash(sk); 649 sk_dst_reset(sk); 650 651 ret = 0; 652 653 out: 654 #endif 655 656 return ret; 657 } 658 659 int sock_bindtoindex(struct sock *sk, int ifindex, bool lock_sk) 660 { 661 int ret; 662 663 if (lock_sk) 664 lock_sock(sk); 665 ret = sock_bindtoindex_locked(sk, ifindex); 666 if (lock_sk) 667 release_sock(sk); 668 669 return ret; 670 } 671 EXPORT_SYMBOL(sock_bindtoindex); 672 673 static int sock_setbindtodevice(struct sock *sk, sockptr_t optval, int optlen) 674 { 675 int ret = -ENOPROTOOPT; 676 #ifdef CONFIG_NETDEVICES 677 struct net *net = sock_net(sk); 678 char devname[IFNAMSIZ]; 679 int index; 680 681 ret = -EINVAL; 682 if (optlen < 0) 683 goto out; 684 685 /* Bind this socket to a particular device like "eth0", 686 * as specified in the passed interface name. If the 687 * name is "" or the option length is zero the socket 688 * is not bound. 689 */ 690 if (optlen > IFNAMSIZ - 1) 691 optlen = IFNAMSIZ - 1; 692 memset(devname, 0, sizeof(devname)); 693 694 ret = -EFAULT; 695 if (copy_from_sockptr(devname, optval, optlen)) 696 goto out; 697 698 index = 0; 699 if (devname[0] != '\0') { 700 struct net_device *dev; 701 702 rcu_read_lock(); 703 dev = dev_get_by_name_rcu(net, devname); 704 if (dev) 705 index = dev->ifindex; 706 rcu_read_unlock(); 707 ret = -ENODEV; 708 if (!dev) 709 goto out; 710 } 711 712 sockopt_lock_sock(sk); 713 ret = sock_bindtoindex_locked(sk, index); 714 sockopt_release_sock(sk); 715 out: 716 #endif 717 718 return ret; 719 } 720 721 static int sock_getbindtodevice(struct sock *sk, sockptr_t optval, 722 sockptr_t optlen, int len) 723 { 724 int ret = -ENOPROTOOPT; 725 #ifdef CONFIG_NETDEVICES 726 int bound_dev_if = READ_ONCE(sk->sk_bound_dev_if); 727 struct net *net = sock_net(sk); 728 char devname[IFNAMSIZ]; 729 730 if (bound_dev_if == 0) { 731 len = 0; 732 goto zero; 733 } 734 735 ret = -EINVAL; 736 if (len < IFNAMSIZ) 737 goto out; 738 739 ret = netdev_get_name(net, devname, bound_dev_if); 740 if (ret) 741 goto out; 742 743 len = strlen(devname) + 1; 744 745 ret = -EFAULT; 746 if (copy_to_sockptr(optval, devname, len)) 747 goto out; 748 749 zero: 750 ret = -EFAULT; 751 if (copy_to_sockptr(optlen, &len, sizeof(int))) 752 goto out; 753 754 ret = 0; 755 756 out: 757 #endif 758 759 return ret; 760 } 761 762 bool sk_mc_loop(const struct sock *sk) 763 { 764 if (dev_recursion_level()) 765 return false; 766 if (!sk) 767 return true; 768 /* IPV6_ADDRFORM can change sk->sk_family under us. */ 769 switch (READ_ONCE(sk->sk_family)) { 770 case AF_INET: 771 return inet_test_bit(MC_LOOP, sk); 772 #if IS_ENABLED(CONFIG_IPV6) 773 case AF_INET6: 774 return inet6_test_bit(MC6_LOOP, sk); 775 #endif 776 } 777 WARN_ON_ONCE(1); 778 return true; 779 } 780 EXPORT_SYMBOL(sk_mc_loop); 781 782 void sock_set_reuseaddr(struct sock *sk) 783 { 784 lock_sock(sk); 785 sk->sk_reuse = SK_CAN_REUSE; 786 release_sock(sk); 787 } 788 EXPORT_SYMBOL(sock_set_reuseaddr); 789 790 void sock_set_reuseport(struct sock *sk) 791 { 792 lock_sock(sk); 793 sk->sk_reuseport = true; 794 release_sock(sk); 795 } 796 EXPORT_SYMBOL(sock_set_reuseport); 797 798 void sock_no_linger(struct sock *sk) 799 { 800 lock_sock(sk); 801 WRITE_ONCE(sk->sk_lingertime, 0); 802 sock_set_flag(sk, SOCK_LINGER); 803 release_sock(sk); 804 } 805 EXPORT_SYMBOL(sock_no_linger); 806 807 void sock_set_priority(struct sock *sk, u32 priority) 808 { 809 WRITE_ONCE(sk->sk_priority, priority); 810 } 811 EXPORT_SYMBOL(sock_set_priority); 812 813 void sock_set_sndtimeo(struct sock *sk, s64 secs) 814 { 815 lock_sock(sk); 816 if (secs && secs < MAX_SCHEDULE_TIMEOUT / HZ - 1) 817 WRITE_ONCE(sk->sk_sndtimeo, secs * HZ); 818 else 819 WRITE_ONCE(sk->sk_sndtimeo, MAX_SCHEDULE_TIMEOUT); 820 release_sock(sk); 821 } 822 EXPORT_SYMBOL(sock_set_sndtimeo); 823 824 static void __sock_set_timestamps(struct sock *sk, bool val, bool new, bool ns) 825 { 826 if (val) { 827 sock_valbool_flag(sk, SOCK_TSTAMP_NEW, new); 828 sock_valbool_flag(sk, SOCK_RCVTSTAMPNS, ns); 829 sock_set_flag(sk, SOCK_RCVTSTAMP); 830 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 831 } else { 832 sock_reset_flag(sk, SOCK_RCVTSTAMP); 833 sock_reset_flag(sk, SOCK_RCVTSTAMPNS); 834 } 835 } 836 837 void sock_enable_timestamps(struct sock *sk) 838 { 839 lock_sock(sk); 840 __sock_set_timestamps(sk, true, false, true); 841 release_sock(sk); 842 } 843 EXPORT_SYMBOL(sock_enable_timestamps); 844 845 void sock_set_timestamp(struct sock *sk, int optname, bool valbool) 846 { 847 switch (optname) { 848 case SO_TIMESTAMP_OLD: 849 __sock_set_timestamps(sk, valbool, false, false); 850 break; 851 case SO_TIMESTAMP_NEW: 852 __sock_set_timestamps(sk, valbool, true, false); 853 break; 854 case SO_TIMESTAMPNS_OLD: 855 __sock_set_timestamps(sk, valbool, false, true); 856 break; 857 case SO_TIMESTAMPNS_NEW: 858 __sock_set_timestamps(sk, valbool, true, true); 859 break; 860 } 861 } 862 863 static int sock_timestamping_bind_phc(struct sock *sk, int phc_index) 864 { 865 struct net *net = sock_net(sk); 866 struct net_device *dev = NULL; 867 bool match = false; 868 int *vclock_index; 869 int i, num; 870 871 if (sk->sk_bound_dev_if) 872 dev = dev_get_by_index(net, sk->sk_bound_dev_if); 873 874 if (!dev) { 875 pr_err("%s: sock not bind to device\n", __func__); 876 return -EOPNOTSUPP; 877 } 878 879 num = ethtool_get_phc_vclocks(dev, &vclock_index); 880 dev_put(dev); 881 882 for (i = 0; i < num; i++) { 883 if (*(vclock_index + i) == phc_index) { 884 match = true; 885 break; 886 } 887 } 888 889 if (num > 0) 890 kfree(vclock_index); 891 892 if (!match) 893 return -EINVAL; 894 895 WRITE_ONCE(sk->sk_bind_phc, phc_index); 896 897 return 0; 898 } 899 900 int sock_set_timestamping(struct sock *sk, int optname, 901 struct so_timestamping timestamping) 902 { 903 int val = timestamping.flags; 904 int ret; 905 906 if (val & ~SOF_TIMESTAMPING_MASK) 907 return -EINVAL; 908 909 if (val & SOF_TIMESTAMPING_OPT_ID_TCP && 910 !(val & SOF_TIMESTAMPING_OPT_ID)) 911 return -EINVAL; 912 913 if (val & SOF_TIMESTAMPING_OPT_ID && 914 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) { 915 if (sk_is_tcp(sk)) { 916 if ((1 << sk->sk_state) & 917 (TCPF_CLOSE | TCPF_LISTEN)) 918 return -EINVAL; 919 if (val & SOF_TIMESTAMPING_OPT_ID_TCP) 920 atomic_set(&sk->sk_tskey, tcp_sk(sk)->write_seq); 921 else 922 atomic_set(&sk->sk_tskey, tcp_sk(sk)->snd_una); 923 } else { 924 atomic_set(&sk->sk_tskey, 0); 925 } 926 } 927 928 if (val & SOF_TIMESTAMPING_OPT_STATS && 929 !(val & SOF_TIMESTAMPING_OPT_TSONLY)) 930 return -EINVAL; 931 932 if (val & SOF_TIMESTAMPING_BIND_PHC) { 933 ret = sock_timestamping_bind_phc(sk, timestamping.bind_phc); 934 if (ret) 935 return ret; 936 } 937 938 WRITE_ONCE(sk->sk_tsflags, val); 939 sock_valbool_flag(sk, SOCK_TSTAMP_NEW, optname == SO_TIMESTAMPING_NEW); 940 941 if (val & SOF_TIMESTAMPING_RX_SOFTWARE) 942 sock_enable_timestamp(sk, 943 SOCK_TIMESTAMPING_RX_SOFTWARE); 944 else 945 sock_disable_timestamp(sk, 946 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE)); 947 return 0; 948 } 949 950 void sock_set_keepalive(struct sock *sk) 951 { 952 lock_sock(sk); 953 if (sk->sk_prot->keepalive) 954 sk->sk_prot->keepalive(sk, true); 955 sock_valbool_flag(sk, SOCK_KEEPOPEN, true); 956 release_sock(sk); 957 } 958 EXPORT_SYMBOL(sock_set_keepalive); 959 960 static void __sock_set_rcvbuf(struct sock *sk, int val) 961 { 962 /* Ensure val * 2 fits into an int, to prevent max_t() from treating it 963 * as a negative value. 964 */ 965 val = min_t(int, val, INT_MAX / 2); 966 sk->sk_userlocks |= SOCK_RCVBUF_LOCK; 967 968 /* We double it on the way in to account for "struct sk_buff" etc. 969 * overhead. Applications assume that the SO_RCVBUF setting they make 970 * will allow that much actual data to be received on that socket. 971 * 972 * Applications are unaware that "struct sk_buff" and other overheads 973 * allocate from the receive buffer during socket buffer allocation. 974 * 975 * And after considering the possible alternatives, returning the value 976 * we actually used in getsockopt is the most desirable behavior. 977 */ 978 WRITE_ONCE(sk->sk_rcvbuf, max_t(int, val * 2, SOCK_MIN_RCVBUF)); 979 } 980 981 void sock_set_rcvbuf(struct sock *sk, int val) 982 { 983 lock_sock(sk); 984 __sock_set_rcvbuf(sk, val); 985 release_sock(sk); 986 } 987 EXPORT_SYMBOL(sock_set_rcvbuf); 988 989 static void __sock_set_mark(struct sock *sk, u32 val) 990 { 991 if (val != sk->sk_mark) { 992 WRITE_ONCE(sk->sk_mark, val); 993 sk_dst_reset(sk); 994 } 995 } 996 997 void sock_set_mark(struct sock *sk, u32 val) 998 { 999 lock_sock(sk); 1000 __sock_set_mark(sk, val); 1001 release_sock(sk); 1002 } 1003 EXPORT_SYMBOL(sock_set_mark); 1004 1005 static void sock_release_reserved_memory(struct sock *sk, int bytes) 1006 { 1007 /* Round down bytes to multiple of pages */ 1008 bytes = round_down(bytes, PAGE_SIZE); 1009 1010 WARN_ON(bytes > sk->sk_reserved_mem); 1011 WRITE_ONCE(sk->sk_reserved_mem, sk->sk_reserved_mem - bytes); 1012 sk_mem_reclaim(sk); 1013 } 1014 1015 static int sock_reserve_memory(struct sock *sk, int bytes) 1016 { 1017 long allocated; 1018 bool charged; 1019 int pages; 1020 1021 if (!mem_cgroup_sockets_enabled || !sk->sk_memcg || !sk_has_account(sk)) 1022 return -EOPNOTSUPP; 1023 1024 if (!bytes) 1025 return 0; 1026 1027 pages = sk_mem_pages(bytes); 1028 1029 /* pre-charge to memcg */ 1030 charged = mem_cgroup_charge_skmem(sk->sk_memcg, pages, 1031 GFP_KERNEL | __GFP_RETRY_MAYFAIL); 1032 if (!charged) 1033 return -ENOMEM; 1034 1035 /* pre-charge to forward_alloc */ 1036 sk_memory_allocated_add(sk, pages); 1037 allocated = sk_memory_allocated(sk); 1038 /* If the system goes into memory pressure with this 1039 * precharge, give up and return error. 1040 */ 1041 if (allocated > sk_prot_mem_limits(sk, 1)) { 1042 sk_memory_allocated_sub(sk, pages); 1043 mem_cgroup_uncharge_skmem(sk->sk_memcg, pages); 1044 return -ENOMEM; 1045 } 1046 sk_forward_alloc_add(sk, pages << PAGE_SHIFT); 1047 1048 WRITE_ONCE(sk->sk_reserved_mem, 1049 sk->sk_reserved_mem + (pages << PAGE_SHIFT)); 1050 1051 return 0; 1052 } 1053 1054 void sockopt_lock_sock(struct sock *sk) 1055 { 1056 /* When current->bpf_ctx is set, the setsockopt is called from 1057 * a bpf prog. bpf has ensured the sk lock has been 1058 * acquired before calling setsockopt(). 1059 */ 1060 if (has_current_bpf_ctx()) 1061 return; 1062 1063 lock_sock(sk); 1064 } 1065 EXPORT_SYMBOL(sockopt_lock_sock); 1066 1067 void sockopt_release_sock(struct sock *sk) 1068 { 1069 if (has_current_bpf_ctx()) 1070 return; 1071 1072 release_sock(sk); 1073 } 1074 EXPORT_SYMBOL(sockopt_release_sock); 1075 1076 bool sockopt_ns_capable(struct user_namespace *ns, int cap) 1077 { 1078 return has_current_bpf_ctx() || ns_capable(ns, cap); 1079 } 1080 EXPORT_SYMBOL(sockopt_ns_capable); 1081 1082 bool sockopt_capable(int cap) 1083 { 1084 return has_current_bpf_ctx() || capable(cap); 1085 } 1086 EXPORT_SYMBOL(sockopt_capable); 1087 1088 /* 1089 * This is meant for all protocols to use and covers goings on 1090 * at the socket level. Everything here is generic. 1091 */ 1092 1093 int sk_setsockopt(struct sock *sk, int level, int optname, 1094 sockptr_t optval, unsigned int optlen) 1095 { 1096 struct so_timestamping timestamping; 1097 struct socket *sock = sk->sk_socket; 1098 struct sock_txtime sk_txtime; 1099 int val; 1100 int valbool; 1101 struct linger ling; 1102 int ret = 0; 1103 1104 /* 1105 * Options without arguments 1106 */ 1107 1108 if (optname == SO_BINDTODEVICE) 1109 return sock_setbindtodevice(sk, optval, optlen); 1110 1111 if (optlen < sizeof(int)) 1112 return -EINVAL; 1113 1114 if (copy_from_sockptr(&val, optval, sizeof(val))) 1115 return -EFAULT; 1116 1117 valbool = val ? 1 : 0; 1118 1119 /* handle options which do not require locking the socket. */ 1120 switch (optname) { 1121 case SO_PRIORITY: 1122 if ((val >= 0 && val <= 6) || 1123 sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) || 1124 sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1125 sock_set_priority(sk, val); 1126 return 0; 1127 } 1128 return -EPERM; 1129 case SO_PASSSEC: 1130 assign_bit(SOCK_PASSSEC, &sock->flags, valbool); 1131 return 0; 1132 case SO_PASSCRED: 1133 assign_bit(SOCK_PASSCRED, &sock->flags, valbool); 1134 return 0; 1135 case SO_PASSPIDFD: 1136 assign_bit(SOCK_PASSPIDFD, &sock->flags, valbool); 1137 return 0; 1138 case SO_TYPE: 1139 case SO_PROTOCOL: 1140 case SO_DOMAIN: 1141 case SO_ERROR: 1142 return -ENOPROTOOPT; 1143 #ifdef CONFIG_NET_RX_BUSY_POLL 1144 case SO_BUSY_POLL: 1145 if (val < 0) 1146 return -EINVAL; 1147 WRITE_ONCE(sk->sk_ll_usec, val); 1148 return 0; 1149 case SO_PREFER_BUSY_POLL: 1150 if (valbool && !sockopt_capable(CAP_NET_ADMIN)) 1151 return -EPERM; 1152 WRITE_ONCE(sk->sk_prefer_busy_poll, valbool); 1153 return 0; 1154 case SO_BUSY_POLL_BUDGET: 1155 if (val > READ_ONCE(sk->sk_busy_poll_budget) && 1156 !sockopt_capable(CAP_NET_ADMIN)) 1157 return -EPERM; 1158 if (val < 0 || val > U16_MAX) 1159 return -EINVAL; 1160 WRITE_ONCE(sk->sk_busy_poll_budget, val); 1161 return 0; 1162 #endif 1163 case SO_MAX_PACING_RATE: 1164 { 1165 unsigned long ulval = (val == ~0U) ? ~0UL : (unsigned int)val; 1166 unsigned long pacing_rate; 1167 1168 if (sizeof(ulval) != sizeof(val) && 1169 optlen >= sizeof(ulval) && 1170 copy_from_sockptr(&ulval, optval, sizeof(ulval))) { 1171 return -EFAULT; 1172 } 1173 if (ulval != ~0UL) 1174 cmpxchg(&sk->sk_pacing_status, 1175 SK_PACING_NONE, 1176 SK_PACING_NEEDED); 1177 /* Pairs with READ_ONCE() from sk_getsockopt() */ 1178 WRITE_ONCE(sk->sk_max_pacing_rate, ulval); 1179 pacing_rate = READ_ONCE(sk->sk_pacing_rate); 1180 if (ulval < pacing_rate) 1181 WRITE_ONCE(sk->sk_pacing_rate, ulval); 1182 return 0; 1183 } 1184 case SO_TXREHASH: 1185 if (val < -1 || val > 1) 1186 return -EINVAL; 1187 if ((u8)val == SOCK_TXREHASH_DEFAULT) 1188 val = READ_ONCE(sock_net(sk)->core.sysctl_txrehash); 1189 /* Paired with READ_ONCE() in tcp_rtx_synack() 1190 * and sk_getsockopt(). 1191 */ 1192 WRITE_ONCE(sk->sk_txrehash, (u8)val); 1193 return 0; 1194 } 1195 1196 sockopt_lock_sock(sk); 1197 1198 switch (optname) { 1199 case SO_DEBUG: 1200 if (val && !sockopt_capable(CAP_NET_ADMIN)) 1201 ret = -EACCES; 1202 else 1203 sock_valbool_flag(sk, SOCK_DBG, valbool); 1204 break; 1205 case SO_REUSEADDR: 1206 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE); 1207 break; 1208 case SO_REUSEPORT: 1209 sk->sk_reuseport = valbool; 1210 break; 1211 case SO_DONTROUTE: 1212 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool); 1213 sk_dst_reset(sk); 1214 break; 1215 case SO_BROADCAST: 1216 sock_valbool_flag(sk, SOCK_BROADCAST, valbool); 1217 break; 1218 case SO_SNDBUF: 1219 /* Don't error on this BSD doesn't and if you think 1220 * about it this is right. Otherwise apps have to 1221 * play 'guess the biggest size' games. RCVBUF/SNDBUF 1222 * are treated in BSD as hints 1223 */ 1224 val = min_t(u32, val, READ_ONCE(sysctl_wmem_max)); 1225 set_sndbuf: 1226 /* Ensure val * 2 fits into an int, to prevent max_t() 1227 * from treating it as a negative value. 1228 */ 1229 val = min_t(int, val, INT_MAX / 2); 1230 sk->sk_userlocks |= SOCK_SNDBUF_LOCK; 1231 WRITE_ONCE(sk->sk_sndbuf, 1232 max_t(int, val * 2, SOCK_MIN_SNDBUF)); 1233 /* Wake up sending tasks if we upped the value. */ 1234 sk->sk_write_space(sk); 1235 break; 1236 1237 case SO_SNDBUFFORCE: 1238 if (!sockopt_capable(CAP_NET_ADMIN)) { 1239 ret = -EPERM; 1240 break; 1241 } 1242 1243 /* No negative values (to prevent underflow, as val will be 1244 * multiplied by 2). 1245 */ 1246 if (val < 0) 1247 val = 0; 1248 goto set_sndbuf; 1249 1250 case SO_RCVBUF: 1251 /* Don't error on this BSD doesn't and if you think 1252 * about it this is right. Otherwise apps have to 1253 * play 'guess the biggest size' games. RCVBUF/SNDBUF 1254 * are treated in BSD as hints 1255 */ 1256 __sock_set_rcvbuf(sk, min_t(u32, val, READ_ONCE(sysctl_rmem_max))); 1257 break; 1258 1259 case SO_RCVBUFFORCE: 1260 if (!sockopt_capable(CAP_NET_ADMIN)) { 1261 ret = -EPERM; 1262 break; 1263 } 1264 1265 /* No negative values (to prevent underflow, as val will be 1266 * multiplied by 2). 1267 */ 1268 __sock_set_rcvbuf(sk, max(val, 0)); 1269 break; 1270 1271 case SO_KEEPALIVE: 1272 if (sk->sk_prot->keepalive) 1273 sk->sk_prot->keepalive(sk, valbool); 1274 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool); 1275 break; 1276 1277 case SO_OOBINLINE: 1278 sock_valbool_flag(sk, SOCK_URGINLINE, valbool); 1279 break; 1280 1281 case SO_NO_CHECK: 1282 sk->sk_no_check_tx = valbool; 1283 break; 1284 1285 case SO_LINGER: 1286 if (optlen < sizeof(ling)) { 1287 ret = -EINVAL; /* 1003.1g */ 1288 break; 1289 } 1290 if (copy_from_sockptr(&ling, optval, sizeof(ling))) { 1291 ret = -EFAULT; 1292 break; 1293 } 1294 if (!ling.l_onoff) { 1295 sock_reset_flag(sk, SOCK_LINGER); 1296 } else { 1297 unsigned long t_sec = ling.l_linger; 1298 1299 if (t_sec >= MAX_SCHEDULE_TIMEOUT / HZ) 1300 WRITE_ONCE(sk->sk_lingertime, MAX_SCHEDULE_TIMEOUT); 1301 else 1302 WRITE_ONCE(sk->sk_lingertime, t_sec * HZ); 1303 sock_set_flag(sk, SOCK_LINGER); 1304 } 1305 break; 1306 1307 case SO_BSDCOMPAT: 1308 break; 1309 1310 case SO_TIMESTAMP_OLD: 1311 case SO_TIMESTAMP_NEW: 1312 case SO_TIMESTAMPNS_OLD: 1313 case SO_TIMESTAMPNS_NEW: 1314 sock_set_timestamp(sk, optname, valbool); 1315 break; 1316 1317 case SO_TIMESTAMPING_NEW: 1318 case SO_TIMESTAMPING_OLD: 1319 if (optlen == sizeof(timestamping)) { 1320 if (copy_from_sockptr(×tamping, optval, 1321 sizeof(timestamping))) { 1322 ret = -EFAULT; 1323 break; 1324 } 1325 } else { 1326 memset(×tamping, 0, sizeof(timestamping)); 1327 timestamping.flags = val; 1328 } 1329 ret = sock_set_timestamping(sk, optname, timestamping); 1330 break; 1331 1332 case SO_RCVLOWAT: 1333 { 1334 int (*set_rcvlowat)(struct sock *sk, int val) = NULL; 1335 1336 if (val < 0) 1337 val = INT_MAX; 1338 if (sock) 1339 set_rcvlowat = READ_ONCE(sock->ops)->set_rcvlowat; 1340 if (set_rcvlowat) 1341 ret = set_rcvlowat(sk, val); 1342 else 1343 WRITE_ONCE(sk->sk_rcvlowat, val ? : 1); 1344 break; 1345 } 1346 case SO_RCVTIMEO_OLD: 1347 case SO_RCVTIMEO_NEW: 1348 ret = sock_set_timeout(&sk->sk_rcvtimeo, optval, 1349 optlen, optname == SO_RCVTIMEO_OLD); 1350 break; 1351 1352 case SO_SNDTIMEO_OLD: 1353 case SO_SNDTIMEO_NEW: 1354 ret = sock_set_timeout(&sk->sk_sndtimeo, optval, 1355 optlen, optname == SO_SNDTIMEO_OLD); 1356 break; 1357 1358 case SO_ATTACH_FILTER: { 1359 struct sock_fprog fprog; 1360 1361 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen); 1362 if (!ret) 1363 ret = sk_attach_filter(&fprog, sk); 1364 break; 1365 } 1366 case SO_ATTACH_BPF: 1367 ret = -EINVAL; 1368 if (optlen == sizeof(u32)) { 1369 u32 ufd; 1370 1371 ret = -EFAULT; 1372 if (copy_from_sockptr(&ufd, optval, sizeof(ufd))) 1373 break; 1374 1375 ret = sk_attach_bpf(ufd, sk); 1376 } 1377 break; 1378 1379 case SO_ATTACH_REUSEPORT_CBPF: { 1380 struct sock_fprog fprog; 1381 1382 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen); 1383 if (!ret) 1384 ret = sk_reuseport_attach_filter(&fprog, sk); 1385 break; 1386 } 1387 case SO_ATTACH_REUSEPORT_EBPF: 1388 ret = -EINVAL; 1389 if (optlen == sizeof(u32)) { 1390 u32 ufd; 1391 1392 ret = -EFAULT; 1393 if (copy_from_sockptr(&ufd, optval, sizeof(ufd))) 1394 break; 1395 1396 ret = sk_reuseport_attach_bpf(ufd, sk); 1397 } 1398 break; 1399 1400 case SO_DETACH_REUSEPORT_BPF: 1401 ret = reuseport_detach_prog(sk); 1402 break; 1403 1404 case SO_DETACH_FILTER: 1405 ret = sk_detach_filter(sk); 1406 break; 1407 1408 case SO_LOCK_FILTER: 1409 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool) 1410 ret = -EPERM; 1411 else 1412 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool); 1413 break; 1414 1415 case SO_MARK: 1416 if (!sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && 1417 !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1418 ret = -EPERM; 1419 break; 1420 } 1421 1422 __sock_set_mark(sk, val); 1423 break; 1424 case SO_RCVMARK: 1425 sock_valbool_flag(sk, SOCK_RCVMARK, valbool); 1426 break; 1427 1428 case SO_RXQ_OVFL: 1429 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool); 1430 break; 1431 1432 case SO_WIFI_STATUS: 1433 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool); 1434 break; 1435 1436 case SO_PEEK_OFF: 1437 { 1438 int (*set_peek_off)(struct sock *sk, int val); 1439 1440 set_peek_off = READ_ONCE(sock->ops)->set_peek_off; 1441 if (set_peek_off) 1442 ret = set_peek_off(sk, val); 1443 else 1444 ret = -EOPNOTSUPP; 1445 break; 1446 } 1447 1448 case SO_NOFCS: 1449 sock_valbool_flag(sk, SOCK_NOFCS, valbool); 1450 break; 1451 1452 case SO_SELECT_ERR_QUEUE: 1453 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool); 1454 break; 1455 1456 1457 case SO_INCOMING_CPU: 1458 reuseport_update_incoming_cpu(sk, val); 1459 break; 1460 1461 case SO_CNX_ADVICE: 1462 if (val == 1) 1463 dst_negative_advice(sk); 1464 break; 1465 1466 case SO_ZEROCOPY: 1467 if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) { 1468 if (!(sk_is_tcp(sk) || 1469 (sk->sk_type == SOCK_DGRAM && 1470 sk->sk_protocol == IPPROTO_UDP))) 1471 ret = -EOPNOTSUPP; 1472 } else if (sk->sk_family != PF_RDS) { 1473 ret = -EOPNOTSUPP; 1474 } 1475 if (!ret) { 1476 if (val < 0 || val > 1) 1477 ret = -EINVAL; 1478 else 1479 sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool); 1480 } 1481 break; 1482 1483 case SO_TXTIME: 1484 if (optlen != sizeof(struct sock_txtime)) { 1485 ret = -EINVAL; 1486 break; 1487 } else if (copy_from_sockptr(&sk_txtime, optval, 1488 sizeof(struct sock_txtime))) { 1489 ret = -EFAULT; 1490 break; 1491 } else if (sk_txtime.flags & ~SOF_TXTIME_FLAGS_MASK) { 1492 ret = -EINVAL; 1493 break; 1494 } 1495 /* CLOCK_MONOTONIC is only used by sch_fq, and this packet 1496 * scheduler has enough safe guards. 1497 */ 1498 if (sk_txtime.clockid != CLOCK_MONOTONIC && 1499 !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1500 ret = -EPERM; 1501 break; 1502 } 1503 sock_valbool_flag(sk, SOCK_TXTIME, true); 1504 sk->sk_clockid = sk_txtime.clockid; 1505 sk->sk_txtime_deadline_mode = 1506 !!(sk_txtime.flags & SOF_TXTIME_DEADLINE_MODE); 1507 sk->sk_txtime_report_errors = 1508 !!(sk_txtime.flags & SOF_TXTIME_REPORT_ERRORS); 1509 break; 1510 1511 case SO_BINDTOIFINDEX: 1512 ret = sock_bindtoindex_locked(sk, val); 1513 break; 1514 1515 case SO_BUF_LOCK: 1516 if (val & ~SOCK_BUF_LOCK_MASK) { 1517 ret = -EINVAL; 1518 break; 1519 } 1520 sk->sk_userlocks = val | (sk->sk_userlocks & 1521 ~SOCK_BUF_LOCK_MASK); 1522 break; 1523 1524 case SO_RESERVE_MEM: 1525 { 1526 int delta; 1527 1528 if (val < 0) { 1529 ret = -EINVAL; 1530 break; 1531 } 1532 1533 delta = val - sk->sk_reserved_mem; 1534 if (delta < 0) 1535 sock_release_reserved_memory(sk, -delta); 1536 else 1537 ret = sock_reserve_memory(sk, delta); 1538 break; 1539 } 1540 1541 default: 1542 ret = -ENOPROTOOPT; 1543 break; 1544 } 1545 sockopt_release_sock(sk); 1546 return ret; 1547 } 1548 1549 int sock_setsockopt(struct socket *sock, int level, int optname, 1550 sockptr_t optval, unsigned int optlen) 1551 { 1552 return sk_setsockopt(sock->sk, level, optname, 1553 optval, optlen); 1554 } 1555 EXPORT_SYMBOL(sock_setsockopt); 1556 1557 static const struct cred *sk_get_peer_cred(struct sock *sk) 1558 { 1559 const struct cred *cred; 1560 1561 spin_lock(&sk->sk_peer_lock); 1562 cred = get_cred(sk->sk_peer_cred); 1563 spin_unlock(&sk->sk_peer_lock); 1564 1565 return cred; 1566 } 1567 1568 static void cred_to_ucred(struct pid *pid, const struct cred *cred, 1569 struct ucred *ucred) 1570 { 1571 ucred->pid = pid_vnr(pid); 1572 ucred->uid = ucred->gid = -1; 1573 if (cred) { 1574 struct user_namespace *current_ns = current_user_ns(); 1575 1576 ucred->uid = from_kuid_munged(current_ns, cred->euid); 1577 ucred->gid = from_kgid_munged(current_ns, cred->egid); 1578 } 1579 } 1580 1581 static int groups_to_user(sockptr_t dst, const struct group_info *src) 1582 { 1583 struct user_namespace *user_ns = current_user_ns(); 1584 int i; 1585 1586 for (i = 0; i < src->ngroups; i++) { 1587 gid_t gid = from_kgid_munged(user_ns, src->gid[i]); 1588 1589 if (copy_to_sockptr_offset(dst, i * sizeof(gid), &gid, sizeof(gid))) 1590 return -EFAULT; 1591 } 1592 1593 return 0; 1594 } 1595 1596 int sk_getsockopt(struct sock *sk, int level, int optname, 1597 sockptr_t optval, sockptr_t optlen) 1598 { 1599 struct socket *sock = sk->sk_socket; 1600 1601 union { 1602 int val; 1603 u64 val64; 1604 unsigned long ulval; 1605 struct linger ling; 1606 struct old_timeval32 tm32; 1607 struct __kernel_old_timeval tm; 1608 struct __kernel_sock_timeval stm; 1609 struct sock_txtime txtime; 1610 struct so_timestamping timestamping; 1611 } v; 1612 1613 int lv = sizeof(int); 1614 int len; 1615 1616 if (copy_from_sockptr(&len, optlen, sizeof(int))) 1617 return -EFAULT; 1618 if (len < 0) 1619 return -EINVAL; 1620 1621 memset(&v, 0, sizeof(v)); 1622 1623 switch (optname) { 1624 case SO_DEBUG: 1625 v.val = sock_flag(sk, SOCK_DBG); 1626 break; 1627 1628 case SO_DONTROUTE: 1629 v.val = sock_flag(sk, SOCK_LOCALROUTE); 1630 break; 1631 1632 case SO_BROADCAST: 1633 v.val = sock_flag(sk, SOCK_BROADCAST); 1634 break; 1635 1636 case SO_SNDBUF: 1637 v.val = READ_ONCE(sk->sk_sndbuf); 1638 break; 1639 1640 case SO_RCVBUF: 1641 v.val = READ_ONCE(sk->sk_rcvbuf); 1642 break; 1643 1644 case SO_REUSEADDR: 1645 v.val = sk->sk_reuse; 1646 break; 1647 1648 case SO_REUSEPORT: 1649 v.val = sk->sk_reuseport; 1650 break; 1651 1652 case SO_KEEPALIVE: 1653 v.val = sock_flag(sk, SOCK_KEEPOPEN); 1654 break; 1655 1656 case SO_TYPE: 1657 v.val = sk->sk_type; 1658 break; 1659 1660 case SO_PROTOCOL: 1661 v.val = sk->sk_protocol; 1662 break; 1663 1664 case SO_DOMAIN: 1665 v.val = sk->sk_family; 1666 break; 1667 1668 case SO_ERROR: 1669 v.val = -sock_error(sk); 1670 if (v.val == 0) 1671 v.val = xchg(&sk->sk_err_soft, 0); 1672 break; 1673 1674 case SO_OOBINLINE: 1675 v.val = sock_flag(sk, SOCK_URGINLINE); 1676 break; 1677 1678 case SO_NO_CHECK: 1679 v.val = sk->sk_no_check_tx; 1680 break; 1681 1682 case SO_PRIORITY: 1683 v.val = READ_ONCE(sk->sk_priority); 1684 break; 1685 1686 case SO_LINGER: 1687 lv = sizeof(v.ling); 1688 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER); 1689 v.ling.l_linger = READ_ONCE(sk->sk_lingertime) / HZ; 1690 break; 1691 1692 case SO_BSDCOMPAT: 1693 break; 1694 1695 case SO_TIMESTAMP_OLD: 1696 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && 1697 !sock_flag(sk, SOCK_TSTAMP_NEW) && 1698 !sock_flag(sk, SOCK_RCVTSTAMPNS); 1699 break; 1700 1701 case SO_TIMESTAMPNS_OLD: 1702 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && !sock_flag(sk, SOCK_TSTAMP_NEW); 1703 break; 1704 1705 case SO_TIMESTAMP_NEW: 1706 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && sock_flag(sk, SOCK_TSTAMP_NEW); 1707 break; 1708 1709 case SO_TIMESTAMPNS_NEW: 1710 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && sock_flag(sk, SOCK_TSTAMP_NEW); 1711 break; 1712 1713 case SO_TIMESTAMPING_OLD: 1714 lv = sizeof(v.timestamping); 1715 v.timestamping.flags = READ_ONCE(sk->sk_tsflags); 1716 v.timestamping.bind_phc = READ_ONCE(sk->sk_bind_phc); 1717 break; 1718 1719 case SO_RCVTIMEO_OLD: 1720 case SO_RCVTIMEO_NEW: 1721 lv = sock_get_timeout(READ_ONCE(sk->sk_rcvtimeo), &v, 1722 SO_RCVTIMEO_OLD == optname); 1723 break; 1724 1725 case SO_SNDTIMEO_OLD: 1726 case SO_SNDTIMEO_NEW: 1727 lv = sock_get_timeout(READ_ONCE(sk->sk_sndtimeo), &v, 1728 SO_SNDTIMEO_OLD == optname); 1729 break; 1730 1731 case SO_RCVLOWAT: 1732 v.val = READ_ONCE(sk->sk_rcvlowat); 1733 break; 1734 1735 case SO_SNDLOWAT: 1736 v.val = 1; 1737 break; 1738 1739 case SO_PASSCRED: 1740 v.val = !!test_bit(SOCK_PASSCRED, &sock->flags); 1741 break; 1742 1743 case SO_PASSPIDFD: 1744 v.val = !!test_bit(SOCK_PASSPIDFD, &sock->flags); 1745 break; 1746 1747 case SO_PEERCRED: 1748 { 1749 struct ucred peercred; 1750 if (len > sizeof(peercred)) 1751 len = sizeof(peercred); 1752 1753 spin_lock(&sk->sk_peer_lock); 1754 cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred); 1755 spin_unlock(&sk->sk_peer_lock); 1756 1757 if (copy_to_sockptr(optval, &peercred, len)) 1758 return -EFAULT; 1759 goto lenout; 1760 } 1761 1762 case SO_PEERPIDFD: 1763 { 1764 struct pid *peer_pid; 1765 struct file *pidfd_file = NULL; 1766 int pidfd; 1767 1768 if (len > sizeof(pidfd)) 1769 len = sizeof(pidfd); 1770 1771 spin_lock(&sk->sk_peer_lock); 1772 peer_pid = get_pid(sk->sk_peer_pid); 1773 spin_unlock(&sk->sk_peer_lock); 1774 1775 if (!peer_pid) 1776 return -ENODATA; 1777 1778 pidfd = pidfd_prepare(peer_pid, 0, &pidfd_file); 1779 put_pid(peer_pid); 1780 if (pidfd < 0) 1781 return pidfd; 1782 1783 if (copy_to_sockptr(optval, &pidfd, len) || 1784 copy_to_sockptr(optlen, &len, sizeof(int))) { 1785 put_unused_fd(pidfd); 1786 fput(pidfd_file); 1787 1788 return -EFAULT; 1789 } 1790 1791 fd_install(pidfd, pidfd_file); 1792 return 0; 1793 } 1794 1795 case SO_PEERGROUPS: 1796 { 1797 const struct cred *cred; 1798 int ret, n; 1799 1800 cred = sk_get_peer_cred(sk); 1801 if (!cred) 1802 return -ENODATA; 1803 1804 n = cred->group_info->ngroups; 1805 if (len < n * sizeof(gid_t)) { 1806 len = n * sizeof(gid_t); 1807 put_cred(cred); 1808 return copy_to_sockptr(optlen, &len, sizeof(int)) ? -EFAULT : -ERANGE; 1809 } 1810 len = n * sizeof(gid_t); 1811 1812 ret = groups_to_user(optval, cred->group_info); 1813 put_cred(cred); 1814 if (ret) 1815 return ret; 1816 goto lenout; 1817 } 1818 1819 case SO_PEERNAME: 1820 { 1821 struct sockaddr_storage address; 1822 1823 lv = READ_ONCE(sock->ops)->getname(sock, (struct sockaddr *)&address, 2); 1824 if (lv < 0) 1825 return -ENOTCONN; 1826 if (lv < len) 1827 return -EINVAL; 1828 if (copy_to_sockptr(optval, &address, len)) 1829 return -EFAULT; 1830 goto lenout; 1831 } 1832 1833 /* Dubious BSD thing... Probably nobody even uses it, but 1834 * the UNIX standard wants it for whatever reason... -DaveM 1835 */ 1836 case SO_ACCEPTCONN: 1837 v.val = sk->sk_state == TCP_LISTEN; 1838 break; 1839 1840 case SO_PASSSEC: 1841 v.val = !!test_bit(SOCK_PASSSEC, &sock->flags); 1842 break; 1843 1844 case SO_PEERSEC: 1845 return security_socket_getpeersec_stream(sock, 1846 optval, optlen, len); 1847 1848 case SO_MARK: 1849 v.val = READ_ONCE(sk->sk_mark); 1850 break; 1851 1852 case SO_RCVMARK: 1853 v.val = sock_flag(sk, SOCK_RCVMARK); 1854 break; 1855 1856 case SO_RXQ_OVFL: 1857 v.val = sock_flag(sk, SOCK_RXQ_OVFL); 1858 break; 1859 1860 case SO_WIFI_STATUS: 1861 v.val = sock_flag(sk, SOCK_WIFI_STATUS); 1862 break; 1863 1864 case SO_PEEK_OFF: 1865 if (!READ_ONCE(sock->ops)->set_peek_off) 1866 return -EOPNOTSUPP; 1867 1868 v.val = READ_ONCE(sk->sk_peek_off); 1869 break; 1870 case SO_NOFCS: 1871 v.val = sock_flag(sk, SOCK_NOFCS); 1872 break; 1873 1874 case SO_BINDTODEVICE: 1875 return sock_getbindtodevice(sk, optval, optlen, len); 1876 1877 case SO_GET_FILTER: 1878 len = sk_get_filter(sk, optval, len); 1879 if (len < 0) 1880 return len; 1881 1882 goto lenout; 1883 1884 case SO_LOCK_FILTER: 1885 v.val = sock_flag(sk, SOCK_FILTER_LOCKED); 1886 break; 1887 1888 case SO_BPF_EXTENSIONS: 1889 v.val = bpf_tell_extensions(); 1890 break; 1891 1892 case SO_SELECT_ERR_QUEUE: 1893 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE); 1894 break; 1895 1896 #ifdef CONFIG_NET_RX_BUSY_POLL 1897 case SO_BUSY_POLL: 1898 v.val = READ_ONCE(sk->sk_ll_usec); 1899 break; 1900 case SO_PREFER_BUSY_POLL: 1901 v.val = READ_ONCE(sk->sk_prefer_busy_poll); 1902 break; 1903 #endif 1904 1905 case SO_MAX_PACING_RATE: 1906 /* The READ_ONCE() pair with the WRITE_ONCE() in sk_setsockopt() */ 1907 if (sizeof(v.ulval) != sizeof(v.val) && len >= sizeof(v.ulval)) { 1908 lv = sizeof(v.ulval); 1909 v.ulval = READ_ONCE(sk->sk_max_pacing_rate); 1910 } else { 1911 /* 32bit version */ 1912 v.val = min_t(unsigned long, ~0U, 1913 READ_ONCE(sk->sk_max_pacing_rate)); 1914 } 1915 break; 1916 1917 case SO_INCOMING_CPU: 1918 v.val = READ_ONCE(sk->sk_incoming_cpu); 1919 break; 1920 1921 case SO_MEMINFO: 1922 { 1923 u32 meminfo[SK_MEMINFO_VARS]; 1924 1925 sk_get_meminfo(sk, meminfo); 1926 1927 len = min_t(unsigned int, len, sizeof(meminfo)); 1928 if (copy_to_sockptr(optval, &meminfo, len)) 1929 return -EFAULT; 1930 1931 goto lenout; 1932 } 1933 1934 #ifdef CONFIG_NET_RX_BUSY_POLL 1935 case SO_INCOMING_NAPI_ID: 1936 v.val = READ_ONCE(sk->sk_napi_id); 1937 1938 /* aggregate non-NAPI IDs down to 0 */ 1939 if (v.val < MIN_NAPI_ID) 1940 v.val = 0; 1941 1942 break; 1943 #endif 1944 1945 case SO_COOKIE: 1946 lv = sizeof(u64); 1947 if (len < lv) 1948 return -EINVAL; 1949 v.val64 = sock_gen_cookie(sk); 1950 break; 1951 1952 case SO_ZEROCOPY: 1953 v.val = sock_flag(sk, SOCK_ZEROCOPY); 1954 break; 1955 1956 case SO_TXTIME: 1957 lv = sizeof(v.txtime); 1958 v.txtime.clockid = sk->sk_clockid; 1959 v.txtime.flags |= sk->sk_txtime_deadline_mode ? 1960 SOF_TXTIME_DEADLINE_MODE : 0; 1961 v.txtime.flags |= sk->sk_txtime_report_errors ? 1962 SOF_TXTIME_REPORT_ERRORS : 0; 1963 break; 1964 1965 case SO_BINDTOIFINDEX: 1966 v.val = READ_ONCE(sk->sk_bound_dev_if); 1967 break; 1968 1969 case SO_NETNS_COOKIE: 1970 lv = sizeof(u64); 1971 if (len != lv) 1972 return -EINVAL; 1973 v.val64 = sock_net(sk)->net_cookie; 1974 break; 1975 1976 case SO_BUF_LOCK: 1977 v.val = sk->sk_userlocks & SOCK_BUF_LOCK_MASK; 1978 break; 1979 1980 case SO_RESERVE_MEM: 1981 v.val = READ_ONCE(sk->sk_reserved_mem); 1982 break; 1983 1984 case SO_TXREHASH: 1985 /* Paired with WRITE_ONCE() in sk_setsockopt() */ 1986 v.val = READ_ONCE(sk->sk_txrehash); 1987 break; 1988 1989 default: 1990 /* We implement the SO_SNDLOWAT etc to not be settable 1991 * (1003.1g 7). 1992 */ 1993 return -ENOPROTOOPT; 1994 } 1995 1996 if (len > lv) 1997 len = lv; 1998 if (copy_to_sockptr(optval, &v, len)) 1999 return -EFAULT; 2000 lenout: 2001 if (copy_to_sockptr(optlen, &len, sizeof(int))) 2002 return -EFAULT; 2003 return 0; 2004 } 2005 2006 /* 2007 * Initialize an sk_lock. 2008 * 2009 * (We also register the sk_lock with the lock validator.) 2010 */ 2011 static inline void sock_lock_init(struct sock *sk) 2012 { 2013 if (sk->sk_kern_sock) 2014 sock_lock_init_class_and_name( 2015 sk, 2016 af_family_kern_slock_key_strings[sk->sk_family], 2017 af_family_kern_slock_keys + sk->sk_family, 2018 af_family_kern_key_strings[sk->sk_family], 2019 af_family_kern_keys + sk->sk_family); 2020 else 2021 sock_lock_init_class_and_name( 2022 sk, 2023 af_family_slock_key_strings[sk->sk_family], 2024 af_family_slock_keys + sk->sk_family, 2025 af_family_key_strings[sk->sk_family], 2026 af_family_keys + sk->sk_family); 2027 } 2028 2029 /* 2030 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet, 2031 * even temporarly, because of RCU lookups. sk_node should also be left as is. 2032 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end 2033 */ 2034 static void sock_copy(struct sock *nsk, const struct sock *osk) 2035 { 2036 const struct proto *prot = READ_ONCE(osk->sk_prot); 2037 #ifdef CONFIG_SECURITY_NETWORK 2038 void *sptr = nsk->sk_security; 2039 #endif 2040 2041 /* If we move sk_tx_queue_mapping out of the private section, 2042 * we must check if sk_tx_queue_clear() is called after 2043 * sock_copy() in sk_clone_lock(). 2044 */ 2045 BUILD_BUG_ON(offsetof(struct sock, sk_tx_queue_mapping) < 2046 offsetof(struct sock, sk_dontcopy_begin) || 2047 offsetof(struct sock, sk_tx_queue_mapping) >= 2048 offsetof(struct sock, sk_dontcopy_end)); 2049 2050 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin)); 2051 2052 memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end, 2053 prot->obj_size - offsetof(struct sock, sk_dontcopy_end)); 2054 2055 #ifdef CONFIG_SECURITY_NETWORK 2056 nsk->sk_security = sptr; 2057 security_sk_clone(osk, nsk); 2058 #endif 2059 } 2060 2061 static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority, 2062 int family) 2063 { 2064 struct sock *sk; 2065 struct kmem_cache *slab; 2066 2067 slab = prot->slab; 2068 if (slab != NULL) { 2069 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO); 2070 if (!sk) 2071 return sk; 2072 if (want_init_on_alloc(priority)) 2073 sk_prot_clear_nulls(sk, prot->obj_size); 2074 } else 2075 sk = kmalloc(prot->obj_size, priority); 2076 2077 if (sk != NULL) { 2078 if (security_sk_alloc(sk, family, priority)) 2079 goto out_free; 2080 2081 if (!try_module_get(prot->owner)) 2082 goto out_free_sec; 2083 } 2084 2085 return sk; 2086 2087 out_free_sec: 2088 security_sk_free(sk); 2089 out_free: 2090 if (slab != NULL) 2091 kmem_cache_free(slab, sk); 2092 else 2093 kfree(sk); 2094 return NULL; 2095 } 2096 2097 static void sk_prot_free(struct proto *prot, struct sock *sk) 2098 { 2099 struct kmem_cache *slab; 2100 struct module *owner; 2101 2102 owner = prot->owner; 2103 slab = prot->slab; 2104 2105 cgroup_sk_free(&sk->sk_cgrp_data); 2106 mem_cgroup_sk_free(sk); 2107 security_sk_free(sk); 2108 if (slab != NULL) 2109 kmem_cache_free(slab, sk); 2110 else 2111 kfree(sk); 2112 module_put(owner); 2113 } 2114 2115 /** 2116 * sk_alloc - All socket objects are allocated here 2117 * @net: the applicable net namespace 2118 * @family: protocol family 2119 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 2120 * @prot: struct proto associated with this new sock instance 2121 * @kern: is this to be a kernel socket? 2122 */ 2123 struct sock *sk_alloc(struct net *net, int family, gfp_t priority, 2124 struct proto *prot, int kern) 2125 { 2126 struct sock *sk; 2127 2128 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family); 2129 if (sk) { 2130 sk->sk_family = family; 2131 /* 2132 * See comment in struct sock definition to understand 2133 * why we need sk_prot_creator -acme 2134 */ 2135 sk->sk_prot = sk->sk_prot_creator = prot; 2136 sk->sk_kern_sock = kern; 2137 sock_lock_init(sk); 2138 sk->sk_net_refcnt = kern ? 0 : 1; 2139 if (likely(sk->sk_net_refcnt)) { 2140 get_net_track(net, &sk->ns_tracker, priority); 2141 sock_inuse_add(net, 1); 2142 } else { 2143 __netns_tracker_alloc(net, &sk->ns_tracker, 2144 false, priority); 2145 } 2146 2147 sock_net_set(sk, net); 2148 refcount_set(&sk->sk_wmem_alloc, 1); 2149 2150 mem_cgroup_sk_alloc(sk); 2151 cgroup_sk_alloc(&sk->sk_cgrp_data); 2152 sock_update_classid(&sk->sk_cgrp_data); 2153 sock_update_netprioidx(&sk->sk_cgrp_data); 2154 sk_tx_queue_clear(sk); 2155 } 2156 2157 return sk; 2158 } 2159 EXPORT_SYMBOL(sk_alloc); 2160 2161 /* Sockets having SOCK_RCU_FREE will call this function after one RCU 2162 * grace period. This is the case for UDP sockets and TCP listeners. 2163 */ 2164 static void __sk_destruct(struct rcu_head *head) 2165 { 2166 struct sock *sk = container_of(head, struct sock, sk_rcu); 2167 struct sk_filter *filter; 2168 2169 if (sk->sk_destruct) 2170 sk->sk_destruct(sk); 2171 2172 filter = rcu_dereference_check(sk->sk_filter, 2173 refcount_read(&sk->sk_wmem_alloc) == 0); 2174 if (filter) { 2175 sk_filter_uncharge(sk, filter); 2176 RCU_INIT_POINTER(sk->sk_filter, NULL); 2177 } 2178 2179 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP); 2180 2181 #ifdef CONFIG_BPF_SYSCALL 2182 bpf_sk_storage_free(sk); 2183 #endif 2184 2185 if (atomic_read(&sk->sk_omem_alloc)) 2186 pr_debug("%s: optmem leakage (%d bytes) detected\n", 2187 __func__, atomic_read(&sk->sk_omem_alloc)); 2188 2189 if (sk->sk_frag.page) { 2190 put_page(sk->sk_frag.page); 2191 sk->sk_frag.page = NULL; 2192 } 2193 2194 /* We do not need to acquire sk->sk_peer_lock, we are the last user. */ 2195 put_cred(sk->sk_peer_cred); 2196 put_pid(sk->sk_peer_pid); 2197 2198 if (likely(sk->sk_net_refcnt)) 2199 put_net_track(sock_net(sk), &sk->ns_tracker); 2200 else 2201 __netns_tracker_free(sock_net(sk), &sk->ns_tracker, false); 2202 2203 sk_prot_free(sk->sk_prot_creator, sk); 2204 } 2205 2206 void sk_destruct(struct sock *sk) 2207 { 2208 bool use_call_rcu = sock_flag(sk, SOCK_RCU_FREE); 2209 2210 if (rcu_access_pointer(sk->sk_reuseport_cb)) { 2211 reuseport_detach_sock(sk); 2212 use_call_rcu = true; 2213 } 2214 2215 if (use_call_rcu) 2216 call_rcu(&sk->sk_rcu, __sk_destruct); 2217 else 2218 __sk_destruct(&sk->sk_rcu); 2219 } 2220 2221 static void __sk_free(struct sock *sk) 2222 { 2223 if (likely(sk->sk_net_refcnt)) 2224 sock_inuse_add(sock_net(sk), -1); 2225 2226 if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk))) 2227 sock_diag_broadcast_destroy(sk); 2228 else 2229 sk_destruct(sk); 2230 } 2231 2232 void sk_free(struct sock *sk) 2233 { 2234 /* 2235 * We subtract one from sk_wmem_alloc and can know if 2236 * some packets are still in some tx queue. 2237 * If not null, sock_wfree() will call __sk_free(sk) later 2238 */ 2239 if (refcount_dec_and_test(&sk->sk_wmem_alloc)) 2240 __sk_free(sk); 2241 } 2242 EXPORT_SYMBOL(sk_free); 2243 2244 static void sk_init_common(struct sock *sk) 2245 { 2246 skb_queue_head_init(&sk->sk_receive_queue); 2247 skb_queue_head_init(&sk->sk_write_queue); 2248 skb_queue_head_init(&sk->sk_error_queue); 2249 2250 rwlock_init(&sk->sk_callback_lock); 2251 lockdep_set_class_and_name(&sk->sk_receive_queue.lock, 2252 af_rlock_keys + sk->sk_family, 2253 af_family_rlock_key_strings[sk->sk_family]); 2254 lockdep_set_class_and_name(&sk->sk_write_queue.lock, 2255 af_wlock_keys + sk->sk_family, 2256 af_family_wlock_key_strings[sk->sk_family]); 2257 lockdep_set_class_and_name(&sk->sk_error_queue.lock, 2258 af_elock_keys + sk->sk_family, 2259 af_family_elock_key_strings[sk->sk_family]); 2260 lockdep_set_class_and_name(&sk->sk_callback_lock, 2261 af_callback_keys + sk->sk_family, 2262 af_family_clock_key_strings[sk->sk_family]); 2263 } 2264 2265 /** 2266 * sk_clone_lock - clone a socket, and lock its clone 2267 * @sk: the socket to clone 2268 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 2269 * 2270 * Caller must unlock socket even in error path (bh_unlock_sock(newsk)) 2271 */ 2272 struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority) 2273 { 2274 struct proto *prot = READ_ONCE(sk->sk_prot); 2275 struct sk_filter *filter; 2276 bool is_charged = true; 2277 struct sock *newsk; 2278 2279 newsk = sk_prot_alloc(prot, priority, sk->sk_family); 2280 if (!newsk) 2281 goto out; 2282 2283 sock_copy(newsk, sk); 2284 2285 newsk->sk_prot_creator = prot; 2286 2287 /* SANITY */ 2288 if (likely(newsk->sk_net_refcnt)) { 2289 get_net_track(sock_net(newsk), &newsk->ns_tracker, priority); 2290 sock_inuse_add(sock_net(newsk), 1); 2291 } else { 2292 /* Kernel sockets are not elevating the struct net refcount. 2293 * Instead, use a tracker to more easily detect if a layer 2294 * is not properly dismantling its kernel sockets at netns 2295 * destroy time. 2296 */ 2297 __netns_tracker_alloc(sock_net(newsk), &newsk->ns_tracker, 2298 false, priority); 2299 } 2300 sk_node_init(&newsk->sk_node); 2301 sock_lock_init(newsk); 2302 bh_lock_sock(newsk); 2303 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL; 2304 newsk->sk_backlog.len = 0; 2305 2306 atomic_set(&newsk->sk_rmem_alloc, 0); 2307 2308 /* sk_wmem_alloc set to one (see sk_free() and sock_wfree()) */ 2309 refcount_set(&newsk->sk_wmem_alloc, 1); 2310 2311 atomic_set(&newsk->sk_omem_alloc, 0); 2312 sk_init_common(newsk); 2313 2314 newsk->sk_dst_cache = NULL; 2315 newsk->sk_dst_pending_confirm = 0; 2316 newsk->sk_wmem_queued = 0; 2317 newsk->sk_forward_alloc = 0; 2318 newsk->sk_reserved_mem = 0; 2319 atomic_set(&newsk->sk_drops, 0); 2320 newsk->sk_send_head = NULL; 2321 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK; 2322 atomic_set(&newsk->sk_zckey, 0); 2323 2324 sock_reset_flag(newsk, SOCK_DONE); 2325 2326 /* sk->sk_memcg will be populated at accept() time */ 2327 newsk->sk_memcg = NULL; 2328 2329 cgroup_sk_clone(&newsk->sk_cgrp_data); 2330 2331 rcu_read_lock(); 2332 filter = rcu_dereference(sk->sk_filter); 2333 if (filter != NULL) 2334 /* though it's an empty new sock, the charging may fail 2335 * if sysctl_optmem_max was changed between creation of 2336 * original socket and cloning 2337 */ 2338 is_charged = sk_filter_charge(newsk, filter); 2339 RCU_INIT_POINTER(newsk->sk_filter, filter); 2340 rcu_read_unlock(); 2341 2342 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) { 2343 /* We need to make sure that we don't uncharge the new 2344 * socket if we couldn't charge it in the first place 2345 * as otherwise we uncharge the parent's filter. 2346 */ 2347 if (!is_charged) 2348 RCU_INIT_POINTER(newsk->sk_filter, NULL); 2349 sk_free_unlock_clone(newsk); 2350 newsk = NULL; 2351 goto out; 2352 } 2353 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL); 2354 2355 if (bpf_sk_storage_clone(sk, newsk)) { 2356 sk_free_unlock_clone(newsk); 2357 newsk = NULL; 2358 goto out; 2359 } 2360 2361 /* Clear sk_user_data if parent had the pointer tagged 2362 * as not suitable for copying when cloning. 2363 */ 2364 if (sk_user_data_is_nocopy(newsk)) 2365 newsk->sk_user_data = NULL; 2366 2367 newsk->sk_err = 0; 2368 newsk->sk_err_soft = 0; 2369 newsk->sk_priority = 0; 2370 newsk->sk_incoming_cpu = raw_smp_processor_id(); 2371 2372 /* Before updating sk_refcnt, we must commit prior changes to memory 2373 * (Documentation/RCU/rculist_nulls.rst for details) 2374 */ 2375 smp_wmb(); 2376 refcount_set(&newsk->sk_refcnt, 2); 2377 2378 sk_set_socket(newsk, NULL); 2379 sk_tx_queue_clear(newsk); 2380 RCU_INIT_POINTER(newsk->sk_wq, NULL); 2381 2382 if (newsk->sk_prot->sockets_allocated) 2383 sk_sockets_allocated_inc(newsk); 2384 2385 if (sock_needs_netstamp(sk) && newsk->sk_flags & SK_FLAGS_TIMESTAMP) 2386 net_enable_timestamp(); 2387 out: 2388 return newsk; 2389 } 2390 EXPORT_SYMBOL_GPL(sk_clone_lock); 2391 2392 void sk_free_unlock_clone(struct sock *sk) 2393 { 2394 /* It is still raw copy of parent, so invalidate 2395 * destructor and make plain sk_free() */ 2396 sk->sk_destruct = NULL; 2397 bh_unlock_sock(sk); 2398 sk_free(sk); 2399 } 2400 EXPORT_SYMBOL_GPL(sk_free_unlock_clone); 2401 2402 static u32 sk_dst_gso_max_size(struct sock *sk, struct dst_entry *dst) 2403 { 2404 bool is_ipv6 = false; 2405 u32 max_size; 2406 2407 #if IS_ENABLED(CONFIG_IPV6) 2408 is_ipv6 = (sk->sk_family == AF_INET6 && 2409 !ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)); 2410 #endif 2411 /* pairs with the WRITE_ONCE() in netif_set_gso(_ipv4)_max_size() */ 2412 max_size = is_ipv6 ? READ_ONCE(dst->dev->gso_max_size) : 2413 READ_ONCE(dst->dev->gso_ipv4_max_size); 2414 if (max_size > GSO_LEGACY_MAX_SIZE && !sk_is_tcp(sk)) 2415 max_size = GSO_LEGACY_MAX_SIZE; 2416 2417 return max_size - (MAX_TCP_HEADER + 1); 2418 } 2419 2420 void sk_setup_caps(struct sock *sk, struct dst_entry *dst) 2421 { 2422 u32 max_segs = 1; 2423 2424 sk->sk_route_caps = dst->dev->features; 2425 if (sk_is_tcp(sk)) 2426 sk->sk_route_caps |= NETIF_F_GSO; 2427 if (sk->sk_route_caps & NETIF_F_GSO) 2428 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE; 2429 if (unlikely(sk->sk_gso_disabled)) 2430 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 2431 if (sk_can_gso(sk)) { 2432 if (dst->header_len && !xfrm_dst_offload_ok(dst)) { 2433 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 2434 } else { 2435 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM; 2436 sk->sk_gso_max_size = sk_dst_gso_max_size(sk, dst); 2437 /* pairs with the WRITE_ONCE() in netif_set_gso_max_segs() */ 2438 max_segs = max_t(u32, READ_ONCE(dst->dev->gso_max_segs), 1); 2439 } 2440 } 2441 sk->sk_gso_max_segs = max_segs; 2442 sk_dst_set(sk, dst); 2443 } 2444 EXPORT_SYMBOL_GPL(sk_setup_caps); 2445 2446 /* 2447 * Simple resource managers for sockets. 2448 */ 2449 2450 2451 /* 2452 * Write buffer destructor automatically called from kfree_skb. 2453 */ 2454 void sock_wfree(struct sk_buff *skb) 2455 { 2456 struct sock *sk = skb->sk; 2457 unsigned int len = skb->truesize; 2458 bool free; 2459 2460 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) { 2461 if (sock_flag(sk, SOCK_RCU_FREE) && 2462 sk->sk_write_space == sock_def_write_space) { 2463 rcu_read_lock(); 2464 free = refcount_sub_and_test(len, &sk->sk_wmem_alloc); 2465 sock_def_write_space_wfree(sk); 2466 rcu_read_unlock(); 2467 if (unlikely(free)) 2468 __sk_free(sk); 2469 return; 2470 } 2471 2472 /* 2473 * Keep a reference on sk_wmem_alloc, this will be released 2474 * after sk_write_space() call 2475 */ 2476 WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc)); 2477 sk->sk_write_space(sk); 2478 len = 1; 2479 } 2480 /* 2481 * if sk_wmem_alloc reaches 0, we must finish what sk_free() 2482 * could not do because of in-flight packets 2483 */ 2484 if (refcount_sub_and_test(len, &sk->sk_wmem_alloc)) 2485 __sk_free(sk); 2486 } 2487 EXPORT_SYMBOL(sock_wfree); 2488 2489 /* This variant of sock_wfree() is used by TCP, 2490 * since it sets SOCK_USE_WRITE_QUEUE. 2491 */ 2492 void __sock_wfree(struct sk_buff *skb) 2493 { 2494 struct sock *sk = skb->sk; 2495 2496 if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)) 2497 __sk_free(sk); 2498 } 2499 2500 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk) 2501 { 2502 skb_orphan(skb); 2503 skb->sk = sk; 2504 #ifdef CONFIG_INET 2505 if (unlikely(!sk_fullsock(sk))) { 2506 skb->destructor = sock_edemux; 2507 sock_hold(sk); 2508 return; 2509 } 2510 #endif 2511 skb->destructor = sock_wfree; 2512 skb_set_hash_from_sk(skb, sk); 2513 /* 2514 * We used to take a refcount on sk, but following operation 2515 * is enough to guarantee sk_free() wont free this sock until 2516 * all in-flight packets are completed 2517 */ 2518 refcount_add(skb->truesize, &sk->sk_wmem_alloc); 2519 } 2520 EXPORT_SYMBOL(skb_set_owner_w); 2521 2522 static bool can_skb_orphan_partial(const struct sk_buff *skb) 2523 { 2524 #ifdef CONFIG_TLS_DEVICE 2525 /* Drivers depend on in-order delivery for crypto offload, 2526 * partial orphan breaks out-of-order-OK logic. 2527 */ 2528 if (skb->decrypted) 2529 return false; 2530 #endif 2531 return (skb->destructor == sock_wfree || 2532 (IS_ENABLED(CONFIG_INET) && skb->destructor == tcp_wfree)); 2533 } 2534 2535 /* This helper is used by netem, as it can hold packets in its 2536 * delay queue. We want to allow the owner socket to send more 2537 * packets, as if they were already TX completed by a typical driver. 2538 * But we also want to keep skb->sk set because some packet schedulers 2539 * rely on it (sch_fq for example). 2540 */ 2541 void skb_orphan_partial(struct sk_buff *skb) 2542 { 2543 if (skb_is_tcp_pure_ack(skb)) 2544 return; 2545 2546 if (can_skb_orphan_partial(skb) && skb_set_owner_sk_safe(skb, skb->sk)) 2547 return; 2548 2549 skb_orphan(skb); 2550 } 2551 EXPORT_SYMBOL(skb_orphan_partial); 2552 2553 /* 2554 * Read buffer destructor automatically called from kfree_skb. 2555 */ 2556 void sock_rfree(struct sk_buff *skb) 2557 { 2558 struct sock *sk = skb->sk; 2559 unsigned int len = skb->truesize; 2560 2561 atomic_sub(len, &sk->sk_rmem_alloc); 2562 sk_mem_uncharge(sk, len); 2563 } 2564 EXPORT_SYMBOL(sock_rfree); 2565 2566 /* 2567 * Buffer destructor for skbs that are not used directly in read or write 2568 * path, e.g. for error handler skbs. Automatically called from kfree_skb. 2569 */ 2570 void sock_efree(struct sk_buff *skb) 2571 { 2572 sock_put(skb->sk); 2573 } 2574 EXPORT_SYMBOL(sock_efree); 2575 2576 /* Buffer destructor for prefetch/receive path where reference count may 2577 * not be held, e.g. for listen sockets. 2578 */ 2579 #ifdef CONFIG_INET 2580 void sock_pfree(struct sk_buff *skb) 2581 { 2582 if (sk_is_refcounted(skb->sk)) 2583 sock_gen_put(skb->sk); 2584 } 2585 EXPORT_SYMBOL(sock_pfree); 2586 #endif /* CONFIG_INET */ 2587 2588 kuid_t sock_i_uid(struct sock *sk) 2589 { 2590 kuid_t uid; 2591 2592 read_lock_bh(&sk->sk_callback_lock); 2593 uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID; 2594 read_unlock_bh(&sk->sk_callback_lock); 2595 return uid; 2596 } 2597 EXPORT_SYMBOL(sock_i_uid); 2598 2599 unsigned long __sock_i_ino(struct sock *sk) 2600 { 2601 unsigned long ino; 2602 2603 read_lock(&sk->sk_callback_lock); 2604 ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0; 2605 read_unlock(&sk->sk_callback_lock); 2606 return ino; 2607 } 2608 EXPORT_SYMBOL(__sock_i_ino); 2609 2610 unsigned long sock_i_ino(struct sock *sk) 2611 { 2612 unsigned long ino; 2613 2614 local_bh_disable(); 2615 ino = __sock_i_ino(sk); 2616 local_bh_enable(); 2617 return ino; 2618 } 2619 EXPORT_SYMBOL(sock_i_ino); 2620 2621 /* 2622 * Allocate a skb from the socket's send buffer. 2623 */ 2624 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force, 2625 gfp_t priority) 2626 { 2627 if (force || 2628 refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) { 2629 struct sk_buff *skb = alloc_skb(size, priority); 2630 2631 if (skb) { 2632 skb_set_owner_w(skb, sk); 2633 return skb; 2634 } 2635 } 2636 return NULL; 2637 } 2638 EXPORT_SYMBOL(sock_wmalloc); 2639 2640 static void sock_ofree(struct sk_buff *skb) 2641 { 2642 struct sock *sk = skb->sk; 2643 2644 atomic_sub(skb->truesize, &sk->sk_omem_alloc); 2645 } 2646 2647 struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size, 2648 gfp_t priority) 2649 { 2650 struct sk_buff *skb; 2651 2652 /* small safe race: SKB_TRUESIZE may differ from final skb->truesize */ 2653 if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) > 2654 READ_ONCE(sysctl_optmem_max)) 2655 return NULL; 2656 2657 skb = alloc_skb(size, priority); 2658 if (!skb) 2659 return NULL; 2660 2661 atomic_add(skb->truesize, &sk->sk_omem_alloc); 2662 skb->sk = sk; 2663 skb->destructor = sock_ofree; 2664 return skb; 2665 } 2666 2667 /* 2668 * Allocate a memory block from the socket's option memory buffer. 2669 */ 2670 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority) 2671 { 2672 int optmem_max = READ_ONCE(sysctl_optmem_max); 2673 2674 if ((unsigned int)size <= optmem_max && 2675 atomic_read(&sk->sk_omem_alloc) + size < optmem_max) { 2676 void *mem; 2677 /* First do the add, to avoid the race if kmalloc 2678 * might sleep. 2679 */ 2680 atomic_add(size, &sk->sk_omem_alloc); 2681 mem = kmalloc(size, priority); 2682 if (mem) 2683 return mem; 2684 atomic_sub(size, &sk->sk_omem_alloc); 2685 } 2686 return NULL; 2687 } 2688 EXPORT_SYMBOL(sock_kmalloc); 2689 2690 /* Free an option memory block. Note, we actually want the inline 2691 * here as this allows gcc to detect the nullify and fold away the 2692 * condition entirely. 2693 */ 2694 static inline void __sock_kfree_s(struct sock *sk, void *mem, int size, 2695 const bool nullify) 2696 { 2697 if (WARN_ON_ONCE(!mem)) 2698 return; 2699 if (nullify) 2700 kfree_sensitive(mem); 2701 else 2702 kfree(mem); 2703 atomic_sub(size, &sk->sk_omem_alloc); 2704 } 2705 2706 void sock_kfree_s(struct sock *sk, void *mem, int size) 2707 { 2708 __sock_kfree_s(sk, mem, size, false); 2709 } 2710 EXPORT_SYMBOL(sock_kfree_s); 2711 2712 void sock_kzfree_s(struct sock *sk, void *mem, int size) 2713 { 2714 __sock_kfree_s(sk, mem, size, true); 2715 } 2716 EXPORT_SYMBOL(sock_kzfree_s); 2717 2718 /* It is almost wait_for_tcp_memory minus release_sock/lock_sock. 2719 I think, these locks should be removed for datagram sockets. 2720 */ 2721 static long sock_wait_for_wmem(struct sock *sk, long timeo) 2722 { 2723 DEFINE_WAIT(wait); 2724 2725 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 2726 for (;;) { 2727 if (!timeo) 2728 break; 2729 if (signal_pending(current)) 2730 break; 2731 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2732 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); 2733 if (refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) 2734 break; 2735 if (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN) 2736 break; 2737 if (READ_ONCE(sk->sk_err)) 2738 break; 2739 timeo = schedule_timeout(timeo); 2740 } 2741 finish_wait(sk_sleep(sk), &wait); 2742 return timeo; 2743 } 2744 2745 2746 /* 2747 * Generic send/receive buffer handlers 2748 */ 2749 2750 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len, 2751 unsigned long data_len, int noblock, 2752 int *errcode, int max_page_order) 2753 { 2754 struct sk_buff *skb; 2755 long timeo; 2756 int err; 2757 2758 timeo = sock_sndtimeo(sk, noblock); 2759 for (;;) { 2760 err = sock_error(sk); 2761 if (err != 0) 2762 goto failure; 2763 2764 err = -EPIPE; 2765 if (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN) 2766 goto failure; 2767 2768 if (sk_wmem_alloc_get(sk) < READ_ONCE(sk->sk_sndbuf)) 2769 break; 2770 2771 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 2772 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2773 err = -EAGAIN; 2774 if (!timeo) 2775 goto failure; 2776 if (signal_pending(current)) 2777 goto interrupted; 2778 timeo = sock_wait_for_wmem(sk, timeo); 2779 } 2780 skb = alloc_skb_with_frags(header_len, data_len, max_page_order, 2781 errcode, sk->sk_allocation); 2782 if (skb) 2783 skb_set_owner_w(skb, sk); 2784 return skb; 2785 2786 interrupted: 2787 err = sock_intr_errno(timeo); 2788 failure: 2789 *errcode = err; 2790 return NULL; 2791 } 2792 EXPORT_SYMBOL(sock_alloc_send_pskb); 2793 2794 int __sock_cmsg_send(struct sock *sk, struct cmsghdr *cmsg, 2795 struct sockcm_cookie *sockc) 2796 { 2797 u32 tsflags; 2798 2799 switch (cmsg->cmsg_type) { 2800 case SO_MARK: 2801 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && 2802 !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 2803 return -EPERM; 2804 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 2805 return -EINVAL; 2806 sockc->mark = *(u32 *)CMSG_DATA(cmsg); 2807 break; 2808 case SO_TIMESTAMPING_OLD: 2809 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 2810 return -EINVAL; 2811 2812 tsflags = *(u32 *)CMSG_DATA(cmsg); 2813 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK) 2814 return -EINVAL; 2815 2816 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK; 2817 sockc->tsflags |= tsflags; 2818 break; 2819 case SCM_TXTIME: 2820 if (!sock_flag(sk, SOCK_TXTIME)) 2821 return -EINVAL; 2822 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64))) 2823 return -EINVAL; 2824 sockc->transmit_time = get_unaligned((u64 *)CMSG_DATA(cmsg)); 2825 break; 2826 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */ 2827 case SCM_RIGHTS: 2828 case SCM_CREDENTIALS: 2829 break; 2830 default: 2831 return -EINVAL; 2832 } 2833 return 0; 2834 } 2835 EXPORT_SYMBOL(__sock_cmsg_send); 2836 2837 int sock_cmsg_send(struct sock *sk, struct msghdr *msg, 2838 struct sockcm_cookie *sockc) 2839 { 2840 struct cmsghdr *cmsg; 2841 int ret; 2842 2843 for_each_cmsghdr(cmsg, msg) { 2844 if (!CMSG_OK(msg, cmsg)) 2845 return -EINVAL; 2846 if (cmsg->cmsg_level != SOL_SOCKET) 2847 continue; 2848 ret = __sock_cmsg_send(sk, cmsg, sockc); 2849 if (ret) 2850 return ret; 2851 } 2852 return 0; 2853 } 2854 EXPORT_SYMBOL(sock_cmsg_send); 2855 2856 static void sk_enter_memory_pressure(struct sock *sk) 2857 { 2858 if (!sk->sk_prot->enter_memory_pressure) 2859 return; 2860 2861 sk->sk_prot->enter_memory_pressure(sk); 2862 } 2863 2864 static void sk_leave_memory_pressure(struct sock *sk) 2865 { 2866 if (sk->sk_prot->leave_memory_pressure) { 2867 INDIRECT_CALL_INET_1(sk->sk_prot->leave_memory_pressure, 2868 tcp_leave_memory_pressure, sk); 2869 } else { 2870 unsigned long *memory_pressure = sk->sk_prot->memory_pressure; 2871 2872 if (memory_pressure && READ_ONCE(*memory_pressure)) 2873 WRITE_ONCE(*memory_pressure, 0); 2874 } 2875 } 2876 2877 DEFINE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key); 2878 2879 /** 2880 * skb_page_frag_refill - check that a page_frag contains enough room 2881 * @sz: minimum size of the fragment we want to get 2882 * @pfrag: pointer to page_frag 2883 * @gfp: priority for memory allocation 2884 * 2885 * Note: While this allocator tries to use high order pages, there is 2886 * no guarantee that allocations succeed. Therefore, @sz MUST be 2887 * less or equal than PAGE_SIZE. 2888 */ 2889 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp) 2890 { 2891 if (pfrag->page) { 2892 if (page_ref_count(pfrag->page) == 1) { 2893 pfrag->offset = 0; 2894 return true; 2895 } 2896 if (pfrag->offset + sz <= pfrag->size) 2897 return true; 2898 put_page(pfrag->page); 2899 } 2900 2901 pfrag->offset = 0; 2902 if (SKB_FRAG_PAGE_ORDER && 2903 !static_branch_unlikely(&net_high_order_alloc_disable_key)) { 2904 /* Avoid direct reclaim but allow kswapd to wake */ 2905 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) | 2906 __GFP_COMP | __GFP_NOWARN | 2907 __GFP_NORETRY, 2908 SKB_FRAG_PAGE_ORDER); 2909 if (likely(pfrag->page)) { 2910 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER; 2911 return true; 2912 } 2913 } 2914 pfrag->page = alloc_page(gfp); 2915 if (likely(pfrag->page)) { 2916 pfrag->size = PAGE_SIZE; 2917 return true; 2918 } 2919 return false; 2920 } 2921 EXPORT_SYMBOL(skb_page_frag_refill); 2922 2923 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 2924 { 2925 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation))) 2926 return true; 2927 2928 sk_enter_memory_pressure(sk); 2929 sk_stream_moderate_sndbuf(sk); 2930 return false; 2931 } 2932 EXPORT_SYMBOL(sk_page_frag_refill); 2933 2934 void __lock_sock(struct sock *sk) 2935 __releases(&sk->sk_lock.slock) 2936 __acquires(&sk->sk_lock.slock) 2937 { 2938 DEFINE_WAIT(wait); 2939 2940 for (;;) { 2941 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait, 2942 TASK_UNINTERRUPTIBLE); 2943 spin_unlock_bh(&sk->sk_lock.slock); 2944 schedule(); 2945 spin_lock_bh(&sk->sk_lock.slock); 2946 if (!sock_owned_by_user(sk)) 2947 break; 2948 } 2949 finish_wait(&sk->sk_lock.wq, &wait); 2950 } 2951 2952 void __release_sock(struct sock *sk) 2953 __releases(&sk->sk_lock.slock) 2954 __acquires(&sk->sk_lock.slock) 2955 { 2956 struct sk_buff *skb, *next; 2957 2958 while ((skb = sk->sk_backlog.head) != NULL) { 2959 sk->sk_backlog.head = sk->sk_backlog.tail = NULL; 2960 2961 spin_unlock_bh(&sk->sk_lock.slock); 2962 2963 do { 2964 next = skb->next; 2965 prefetch(next); 2966 DEBUG_NET_WARN_ON_ONCE(skb_dst_is_noref(skb)); 2967 skb_mark_not_on_list(skb); 2968 sk_backlog_rcv(sk, skb); 2969 2970 cond_resched(); 2971 2972 skb = next; 2973 } while (skb != NULL); 2974 2975 spin_lock_bh(&sk->sk_lock.slock); 2976 } 2977 2978 /* 2979 * Doing the zeroing here guarantee we can not loop forever 2980 * while a wild producer attempts to flood us. 2981 */ 2982 sk->sk_backlog.len = 0; 2983 } 2984 2985 void __sk_flush_backlog(struct sock *sk) 2986 { 2987 spin_lock_bh(&sk->sk_lock.slock); 2988 __release_sock(sk); 2989 2990 if (sk->sk_prot->release_cb) 2991 INDIRECT_CALL_INET_1(sk->sk_prot->release_cb, 2992 tcp_release_cb, sk); 2993 2994 spin_unlock_bh(&sk->sk_lock.slock); 2995 } 2996 EXPORT_SYMBOL_GPL(__sk_flush_backlog); 2997 2998 /** 2999 * sk_wait_data - wait for data to arrive at sk_receive_queue 3000 * @sk: sock to wait on 3001 * @timeo: for how long 3002 * @skb: last skb seen on sk_receive_queue 3003 * 3004 * Now socket state including sk->sk_err is changed only under lock, 3005 * hence we may omit checks after joining wait queue. 3006 * We check receive queue before schedule() only as optimization; 3007 * it is very likely that release_sock() added new data. 3008 */ 3009 int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb) 3010 { 3011 DEFINE_WAIT_FUNC(wait, woken_wake_function); 3012 int rc; 3013 3014 add_wait_queue(sk_sleep(sk), &wait); 3015 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 3016 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait); 3017 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 3018 remove_wait_queue(sk_sleep(sk), &wait); 3019 return rc; 3020 } 3021 EXPORT_SYMBOL(sk_wait_data); 3022 3023 /** 3024 * __sk_mem_raise_allocated - increase memory_allocated 3025 * @sk: socket 3026 * @size: memory size to allocate 3027 * @amt: pages to allocate 3028 * @kind: allocation type 3029 * 3030 * Similar to __sk_mem_schedule(), but does not update sk_forward_alloc. 3031 * 3032 * Unlike the globally shared limits among the sockets under same protocol, 3033 * consuming the budget of a memcg won't have direct effect on other ones. 3034 * So be optimistic about memcg's tolerance, and leave the callers to decide 3035 * whether or not to raise allocated through sk_under_memory_pressure() or 3036 * its variants. 3037 */ 3038 int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind) 3039 { 3040 struct mem_cgroup *memcg = mem_cgroup_sockets_enabled ? sk->sk_memcg : NULL; 3041 struct proto *prot = sk->sk_prot; 3042 bool charged = false; 3043 long allocated; 3044 3045 sk_memory_allocated_add(sk, amt); 3046 allocated = sk_memory_allocated(sk); 3047 3048 if (memcg) { 3049 if (!mem_cgroup_charge_skmem(memcg, amt, gfp_memcg_charge())) 3050 goto suppress_allocation; 3051 charged = true; 3052 } 3053 3054 /* Under limit. */ 3055 if (allocated <= sk_prot_mem_limits(sk, 0)) { 3056 sk_leave_memory_pressure(sk); 3057 return 1; 3058 } 3059 3060 /* Under pressure. */ 3061 if (allocated > sk_prot_mem_limits(sk, 1)) 3062 sk_enter_memory_pressure(sk); 3063 3064 /* Over hard limit. */ 3065 if (allocated > sk_prot_mem_limits(sk, 2)) 3066 goto suppress_allocation; 3067 3068 /* Guarantee minimum buffer size under pressure (either global 3069 * or memcg) to make sure features described in RFC 7323 (TCP 3070 * Extensions for High Performance) work properly. 3071 * 3072 * This rule does NOT stand when exceeds global or memcg's hard 3073 * limit, or else a DoS attack can be taken place by spawning 3074 * lots of sockets whose usage are under minimum buffer size. 3075 */ 3076 if (kind == SK_MEM_RECV) { 3077 if (atomic_read(&sk->sk_rmem_alloc) < sk_get_rmem0(sk, prot)) 3078 return 1; 3079 3080 } else { /* SK_MEM_SEND */ 3081 int wmem0 = sk_get_wmem0(sk, prot); 3082 3083 if (sk->sk_type == SOCK_STREAM) { 3084 if (sk->sk_wmem_queued < wmem0) 3085 return 1; 3086 } else if (refcount_read(&sk->sk_wmem_alloc) < wmem0) { 3087 return 1; 3088 } 3089 } 3090 3091 if (sk_has_memory_pressure(sk)) { 3092 u64 alloc; 3093 3094 /* The following 'average' heuristic is within the 3095 * scope of global accounting, so it only makes 3096 * sense for global memory pressure. 3097 */ 3098 if (!sk_under_global_memory_pressure(sk)) 3099 return 1; 3100 3101 /* Try to be fair among all the sockets under global 3102 * pressure by allowing the ones that below average 3103 * usage to raise. 3104 */ 3105 alloc = sk_sockets_allocated_read_positive(sk); 3106 if (sk_prot_mem_limits(sk, 2) > alloc * 3107 sk_mem_pages(sk->sk_wmem_queued + 3108 atomic_read(&sk->sk_rmem_alloc) + 3109 sk->sk_forward_alloc)) 3110 return 1; 3111 } 3112 3113 suppress_allocation: 3114 3115 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) { 3116 sk_stream_moderate_sndbuf(sk); 3117 3118 /* Fail only if socket is _under_ its sndbuf. 3119 * In this case we cannot block, so that we have to fail. 3120 */ 3121 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf) { 3122 /* Force charge with __GFP_NOFAIL */ 3123 if (memcg && !charged) { 3124 mem_cgroup_charge_skmem(memcg, amt, 3125 gfp_memcg_charge() | __GFP_NOFAIL); 3126 } 3127 return 1; 3128 } 3129 } 3130 3131 if (kind == SK_MEM_SEND || (kind == SK_MEM_RECV && charged)) 3132 trace_sock_exceed_buf_limit(sk, prot, allocated, kind); 3133 3134 sk_memory_allocated_sub(sk, amt); 3135 3136 if (charged) 3137 mem_cgroup_uncharge_skmem(memcg, amt); 3138 3139 return 0; 3140 } 3141 3142 /** 3143 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated 3144 * @sk: socket 3145 * @size: memory size to allocate 3146 * @kind: allocation type 3147 * 3148 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means 3149 * rmem allocation. This function assumes that protocols which have 3150 * memory_pressure use sk_wmem_queued as write buffer accounting. 3151 */ 3152 int __sk_mem_schedule(struct sock *sk, int size, int kind) 3153 { 3154 int ret, amt = sk_mem_pages(size); 3155 3156 sk_forward_alloc_add(sk, amt << PAGE_SHIFT); 3157 ret = __sk_mem_raise_allocated(sk, size, amt, kind); 3158 if (!ret) 3159 sk_forward_alloc_add(sk, -(amt << PAGE_SHIFT)); 3160 return ret; 3161 } 3162 EXPORT_SYMBOL(__sk_mem_schedule); 3163 3164 /** 3165 * __sk_mem_reduce_allocated - reclaim memory_allocated 3166 * @sk: socket 3167 * @amount: number of quanta 3168 * 3169 * Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc 3170 */ 3171 void __sk_mem_reduce_allocated(struct sock *sk, int amount) 3172 { 3173 sk_memory_allocated_sub(sk, amount); 3174 3175 if (mem_cgroup_sockets_enabled && sk->sk_memcg) 3176 mem_cgroup_uncharge_skmem(sk->sk_memcg, amount); 3177 3178 if (sk_under_global_memory_pressure(sk) && 3179 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0))) 3180 sk_leave_memory_pressure(sk); 3181 } 3182 3183 /** 3184 * __sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated 3185 * @sk: socket 3186 * @amount: number of bytes (rounded down to a PAGE_SIZE multiple) 3187 */ 3188 void __sk_mem_reclaim(struct sock *sk, int amount) 3189 { 3190 amount >>= PAGE_SHIFT; 3191 sk_forward_alloc_add(sk, -(amount << PAGE_SHIFT)); 3192 __sk_mem_reduce_allocated(sk, amount); 3193 } 3194 EXPORT_SYMBOL(__sk_mem_reclaim); 3195 3196 int sk_set_peek_off(struct sock *sk, int val) 3197 { 3198 WRITE_ONCE(sk->sk_peek_off, val); 3199 return 0; 3200 } 3201 EXPORT_SYMBOL_GPL(sk_set_peek_off); 3202 3203 /* 3204 * Set of default routines for initialising struct proto_ops when 3205 * the protocol does not support a particular function. In certain 3206 * cases where it makes no sense for a protocol to have a "do nothing" 3207 * function, some default processing is provided. 3208 */ 3209 3210 int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len) 3211 { 3212 return -EOPNOTSUPP; 3213 } 3214 EXPORT_SYMBOL(sock_no_bind); 3215 3216 int sock_no_connect(struct socket *sock, struct sockaddr *saddr, 3217 int len, int flags) 3218 { 3219 return -EOPNOTSUPP; 3220 } 3221 EXPORT_SYMBOL(sock_no_connect); 3222 3223 int sock_no_socketpair(struct socket *sock1, struct socket *sock2) 3224 { 3225 return -EOPNOTSUPP; 3226 } 3227 EXPORT_SYMBOL(sock_no_socketpair); 3228 3229 int sock_no_accept(struct socket *sock, struct socket *newsock, int flags, 3230 bool kern) 3231 { 3232 return -EOPNOTSUPP; 3233 } 3234 EXPORT_SYMBOL(sock_no_accept); 3235 3236 int sock_no_getname(struct socket *sock, struct sockaddr *saddr, 3237 int peer) 3238 { 3239 return -EOPNOTSUPP; 3240 } 3241 EXPORT_SYMBOL(sock_no_getname); 3242 3243 int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) 3244 { 3245 return -EOPNOTSUPP; 3246 } 3247 EXPORT_SYMBOL(sock_no_ioctl); 3248 3249 int sock_no_listen(struct socket *sock, int backlog) 3250 { 3251 return -EOPNOTSUPP; 3252 } 3253 EXPORT_SYMBOL(sock_no_listen); 3254 3255 int sock_no_shutdown(struct socket *sock, int how) 3256 { 3257 return -EOPNOTSUPP; 3258 } 3259 EXPORT_SYMBOL(sock_no_shutdown); 3260 3261 int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len) 3262 { 3263 return -EOPNOTSUPP; 3264 } 3265 EXPORT_SYMBOL(sock_no_sendmsg); 3266 3267 int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len) 3268 { 3269 return -EOPNOTSUPP; 3270 } 3271 EXPORT_SYMBOL(sock_no_sendmsg_locked); 3272 3273 int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len, 3274 int flags) 3275 { 3276 return -EOPNOTSUPP; 3277 } 3278 EXPORT_SYMBOL(sock_no_recvmsg); 3279 3280 int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma) 3281 { 3282 /* Mirror missing mmap method error code */ 3283 return -ENODEV; 3284 } 3285 EXPORT_SYMBOL(sock_no_mmap); 3286 3287 /* 3288 * When a file is received (via SCM_RIGHTS, etc), we must bump the 3289 * various sock-based usage counts. 3290 */ 3291 void __receive_sock(struct file *file) 3292 { 3293 struct socket *sock; 3294 3295 sock = sock_from_file(file); 3296 if (sock) { 3297 sock_update_netprioidx(&sock->sk->sk_cgrp_data); 3298 sock_update_classid(&sock->sk->sk_cgrp_data); 3299 } 3300 } 3301 3302 /* 3303 * Default Socket Callbacks 3304 */ 3305 3306 static void sock_def_wakeup(struct sock *sk) 3307 { 3308 struct socket_wq *wq; 3309 3310 rcu_read_lock(); 3311 wq = rcu_dereference(sk->sk_wq); 3312 if (skwq_has_sleeper(wq)) 3313 wake_up_interruptible_all(&wq->wait); 3314 rcu_read_unlock(); 3315 } 3316 3317 static void sock_def_error_report(struct sock *sk) 3318 { 3319 struct socket_wq *wq; 3320 3321 rcu_read_lock(); 3322 wq = rcu_dereference(sk->sk_wq); 3323 if (skwq_has_sleeper(wq)) 3324 wake_up_interruptible_poll(&wq->wait, EPOLLERR); 3325 sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR); 3326 rcu_read_unlock(); 3327 } 3328 3329 void sock_def_readable(struct sock *sk) 3330 { 3331 struct socket_wq *wq; 3332 3333 trace_sk_data_ready(sk); 3334 3335 rcu_read_lock(); 3336 wq = rcu_dereference(sk->sk_wq); 3337 if (skwq_has_sleeper(wq)) 3338 wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI | 3339 EPOLLRDNORM | EPOLLRDBAND); 3340 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 3341 rcu_read_unlock(); 3342 } 3343 3344 static void sock_def_write_space(struct sock *sk) 3345 { 3346 struct socket_wq *wq; 3347 3348 rcu_read_lock(); 3349 3350 /* Do not wake up a writer until he can make "significant" 3351 * progress. --DaveM 3352 */ 3353 if (sock_writeable(sk)) { 3354 wq = rcu_dereference(sk->sk_wq); 3355 if (skwq_has_sleeper(wq)) 3356 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 3357 EPOLLWRNORM | EPOLLWRBAND); 3358 3359 /* Should agree with poll, otherwise some programs break */ 3360 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); 3361 } 3362 3363 rcu_read_unlock(); 3364 } 3365 3366 /* An optimised version of sock_def_write_space(), should only be called 3367 * for SOCK_RCU_FREE sockets under RCU read section and after putting 3368 * ->sk_wmem_alloc. 3369 */ 3370 static void sock_def_write_space_wfree(struct sock *sk) 3371 { 3372 /* Do not wake up a writer until he can make "significant" 3373 * progress. --DaveM 3374 */ 3375 if (sock_writeable(sk)) { 3376 struct socket_wq *wq = rcu_dereference(sk->sk_wq); 3377 3378 /* rely on refcount_sub from sock_wfree() */ 3379 smp_mb__after_atomic(); 3380 if (wq && waitqueue_active(&wq->wait)) 3381 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 3382 EPOLLWRNORM | EPOLLWRBAND); 3383 3384 /* Should agree with poll, otherwise some programs break */ 3385 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); 3386 } 3387 } 3388 3389 static void sock_def_destruct(struct sock *sk) 3390 { 3391 } 3392 3393 void sk_send_sigurg(struct sock *sk) 3394 { 3395 if (sk->sk_socket && sk->sk_socket->file) 3396 if (send_sigurg(&sk->sk_socket->file->f_owner)) 3397 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI); 3398 } 3399 EXPORT_SYMBOL(sk_send_sigurg); 3400 3401 void sk_reset_timer(struct sock *sk, struct timer_list* timer, 3402 unsigned long expires) 3403 { 3404 if (!mod_timer(timer, expires)) 3405 sock_hold(sk); 3406 } 3407 EXPORT_SYMBOL(sk_reset_timer); 3408 3409 void sk_stop_timer(struct sock *sk, struct timer_list* timer) 3410 { 3411 if (del_timer(timer)) 3412 __sock_put(sk); 3413 } 3414 EXPORT_SYMBOL(sk_stop_timer); 3415 3416 void sk_stop_timer_sync(struct sock *sk, struct timer_list *timer) 3417 { 3418 if (del_timer_sync(timer)) 3419 __sock_put(sk); 3420 } 3421 EXPORT_SYMBOL(sk_stop_timer_sync); 3422 3423 void sock_init_data_uid(struct socket *sock, struct sock *sk, kuid_t uid) 3424 { 3425 sk_init_common(sk); 3426 sk->sk_send_head = NULL; 3427 3428 timer_setup(&sk->sk_timer, NULL, 0); 3429 3430 sk->sk_allocation = GFP_KERNEL; 3431 sk->sk_rcvbuf = READ_ONCE(sysctl_rmem_default); 3432 sk->sk_sndbuf = READ_ONCE(sysctl_wmem_default); 3433 sk->sk_state = TCP_CLOSE; 3434 sk->sk_use_task_frag = true; 3435 sk_set_socket(sk, sock); 3436 3437 sock_set_flag(sk, SOCK_ZAPPED); 3438 3439 if (sock) { 3440 sk->sk_type = sock->type; 3441 RCU_INIT_POINTER(sk->sk_wq, &sock->wq); 3442 sock->sk = sk; 3443 } else { 3444 RCU_INIT_POINTER(sk->sk_wq, NULL); 3445 } 3446 sk->sk_uid = uid; 3447 3448 rwlock_init(&sk->sk_callback_lock); 3449 if (sk->sk_kern_sock) 3450 lockdep_set_class_and_name( 3451 &sk->sk_callback_lock, 3452 af_kern_callback_keys + sk->sk_family, 3453 af_family_kern_clock_key_strings[sk->sk_family]); 3454 else 3455 lockdep_set_class_and_name( 3456 &sk->sk_callback_lock, 3457 af_callback_keys + sk->sk_family, 3458 af_family_clock_key_strings[sk->sk_family]); 3459 3460 sk->sk_state_change = sock_def_wakeup; 3461 sk->sk_data_ready = sock_def_readable; 3462 sk->sk_write_space = sock_def_write_space; 3463 sk->sk_error_report = sock_def_error_report; 3464 sk->sk_destruct = sock_def_destruct; 3465 3466 sk->sk_frag.page = NULL; 3467 sk->sk_frag.offset = 0; 3468 sk->sk_peek_off = -1; 3469 3470 sk->sk_peer_pid = NULL; 3471 sk->sk_peer_cred = NULL; 3472 spin_lock_init(&sk->sk_peer_lock); 3473 3474 sk->sk_write_pending = 0; 3475 sk->sk_rcvlowat = 1; 3476 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT; 3477 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT; 3478 3479 sk->sk_stamp = SK_DEFAULT_STAMP; 3480 #if BITS_PER_LONG==32 3481 seqlock_init(&sk->sk_stamp_seq); 3482 #endif 3483 atomic_set(&sk->sk_zckey, 0); 3484 3485 #ifdef CONFIG_NET_RX_BUSY_POLL 3486 sk->sk_napi_id = 0; 3487 sk->sk_ll_usec = READ_ONCE(sysctl_net_busy_read); 3488 #endif 3489 3490 sk->sk_max_pacing_rate = ~0UL; 3491 sk->sk_pacing_rate = ~0UL; 3492 WRITE_ONCE(sk->sk_pacing_shift, 10); 3493 sk->sk_incoming_cpu = -1; 3494 3495 sk_rx_queue_clear(sk); 3496 /* 3497 * Before updating sk_refcnt, we must commit prior changes to memory 3498 * (Documentation/RCU/rculist_nulls.rst for details) 3499 */ 3500 smp_wmb(); 3501 refcount_set(&sk->sk_refcnt, 1); 3502 atomic_set(&sk->sk_drops, 0); 3503 } 3504 EXPORT_SYMBOL(sock_init_data_uid); 3505 3506 void sock_init_data(struct socket *sock, struct sock *sk) 3507 { 3508 kuid_t uid = sock ? 3509 SOCK_INODE(sock)->i_uid : 3510 make_kuid(sock_net(sk)->user_ns, 0); 3511 3512 sock_init_data_uid(sock, sk, uid); 3513 } 3514 EXPORT_SYMBOL(sock_init_data); 3515 3516 void lock_sock_nested(struct sock *sk, int subclass) 3517 { 3518 /* The sk_lock has mutex_lock() semantics here. */ 3519 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_); 3520 3521 might_sleep(); 3522 spin_lock_bh(&sk->sk_lock.slock); 3523 if (sock_owned_by_user_nocheck(sk)) 3524 __lock_sock(sk); 3525 sk->sk_lock.owned = 1; 3526 spin_unlock_bh(&sk->sk_lock.slock); 3527 } 3528 EXPORT_SYMBOL(lock_sock_nested); 3529 3530 void release_sock(struct sock *sk) 3531 { 3532 spin_lock_bh(&sk->sk_lock.slock); 3533 if (sk->sk_backlog.tail) 3534 __release_sock(sk); 3535 3536 if (sk->sk_prot->release_cb) 3537 INDIRECT_CALL_INET_1(sk->sk_prot->release_cb, 3538 tcp_release_cb, sk); 3539 3540 sock_release_ownership(sk); 3541 if (waitqueue_active(&sk->sk_lock.wq)) 3542 wake_up(&sk->sk_lock.wq); 3543 spin_unlock_bh(&sk->sk_lock.slock); 3544 } 3545 EXPORT_SYMBOL(release_sock); 3546 3547 bool __lock_sock_fast(struct sock *sk) __acquires(&sk->sk_lock.slock) 3548 { 3549 might_sleep(); 3550 spin_lock_bh(&sk->sk_lock.slock); 3551 3552 if (!sock_owned_by_user_nocheck(sk)) { 3553 /* 3554 * Fast path return with bottom halves disabled and 3555 * sock::sk_lock.slock held. 3556 * 3557 * The 'mutex' is not contended and holding 3558 * sock::sk_lock.slock prevents all other lockers to 3559 * proceed so the corresponding unlock_sock_fast() can 3560 * avoid the slow path of release_sock() completely and 3561 * just release slock. 3562 * 3563 * From a semantical POV this is equivalent to 'acquiring' 3564 * the 'mutex', hence the corresponding lockdep 3565 * mutex_release() has to happen in the fast path of 3566 * unlock_sock_fast(). 3567 */ 3568 return false; 3569 } 3570 3571 __lock_sock(sk); 3572 sk->sk_lock.owned = 1; 3573 __acquire(&sk->sk_lock.slock); 3574 spin_unlock_bh(&sk->sk_lock.slock); 3575 return true; 3576 } 3577 EXPORT_SYMBOL(__lock_sock_fast); 3578 3579 int sock_gettstamp(struct socket *sock, void __user *userstamp, 3580 bool timeval, bool time32) 3581 { 3582 struct sock *sk = sock->sk; 3583 struct timespec64 ts; 3584 3585 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 3586 ts = ktime_to_timespec64(sock_read_timestamp(sk)); 3587 if (ts.tv_sec == -1) 3588 return -ENOENT; 3589 if (ts.tv_sec == 0) { 3590 ktime_t kt = ktime_get_real(); 3591 sock_write_timestamp(sk, kt); 3592 ts = ktime_to_timespec64(kt); 3593 } 3594 3595 if (timeval) 3596 ts.tv_nsec /= 1000; 3597 3598 #ifdef CONFIG_COMPAT_32BIT_TIME 3599 if (time32) 3600 return put_old_timespec32(&ts, userstamp); 3601 #endif 3602 #ifdef CONFIG_SPARC64 3603 /* beware of padding in sparc64 timeval */ 3604 if (timeval && !in_compat_syscall()) { 3605 struct __kernel_old_timeval __user tv = { 3606 .tv_sec = ts.tv_sec, 3607 .tv_usec = ts.tv_nsec, 3608 }; 3609 if (copy_to_user(userstamp, &tv, sizeof(tv))) 3610 return -EFAULT; 3611 return 0; 3612 } 3613 #endif 3614 return put_timespec64(&ts, userstamp); 3615 } 3616 EXPORT_SYMBOL(sock_gettstamp); 3617 3618 void sock_enable_timestamp(struct sock *sk, enum sock_flags flag) 3619 { 3620 if (!sock_flag(sk, flag)) { 3621 unsigned long previous_flags = sk->sk_flags; 3622 3623 sock_set_flag(sk, flag); 3624 /* 3625 * we just set one of the two flags which require net 3626 * time stamping, but time stamping might have been on 3627 * already because of the other one 3628 */ 3629 if (sock_needs_netstamp(sk) && 3630 !(previous_flags & SK_FLAGS_TIMESTAMP)) 3631 net_enable_timestamp(); 3632 } 3633 } 3634 3635 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, 3636 int level, int type) 3637 { 3638 struct sock_exterr_skb *serr; 3639 struct sk_buff *skb; 3640 int copied, err; 3641 3642 err = -EAGAIN; 3643 skb = sock_dequeue_err_skb(sk); 3644 if (skb == NULL) 3645 goto out; 3646 3647 copied = skb->len; 3648 if (copied > len) { 3649 msg->msg_flags |= MSG_TRUNC; 3650 copied = len; 3651 } 3652 err = skb_copy_datagram_msg(skb, 0, msg, copied); 3653 if (err) 3654 goto out_free_skb; 3655 3656 sock_recv_timestamp(msg, sk, skb); 3657 3658 serr = SKB_EXT_ERR(skb); 3659 put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee); 3660 3661 msg->msg_flags |= MSG_ERRQUEUE; 3662 err = copied; 3663 3664 out_free_skb: 3665 kfree_skb(skb); 3666 out: 3667 return err; 3668 } 3669 EXPORT_SYMBOL(sock_recv_errqueue); 3670 3671 /* 3672 * Get a socket option on an socket. 3673 * 3674 * FIX: POSIX 1003.1g is very ambiguous here. It states that 3675 * asynchronous errors should be reported by getsockopt. We assume 3676 * this means if you specify SO_ERROR (otherwise whats the point of it). 3677 */ 3678 int sock_common_getsockopt(struct socket *sock, int level, int optname, 3679 char __user *optval, int __user *optlen) 3680 { 3681 struct sock *sk = sock->sk; 3682 3683 /* IPV6_ADDRFORM can change sk->sk_prot under us. */ 3684 return READ_ONCE(sk->sk_prot)->getsockopt(sk, level, optname, optval, optlen); 3685 } 3686 EXPORT_SYMBOL(sock_common_getsockopt); 3687 3688 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, 3689 int flags) 3690 { 3691 struct sock *sk = sock->sk; 3692 int addr_len = 0; 3693 int err; 3694 3695 err = sk->sk_prot->recvmsg(sk, msg, size, flags, &addr_len); 3696 if (err >= 0) 3697 msg->msg_namelen = addr_len; 3698 return err; 3699 } 3700 EXPORT_SYMBOL(sock_common_recvmsg); 3701 3702 /* 3703 * Set socket options on an inet socket. 3704 */ 3705 int sock_common_setsockopt(struct socket *sock, int level, int optname, 3706 sockptr_t optval, unsigned int optlen) 3707 { 3708 struct sock *sk = sock->sk; 3709 3710 /* IPV6_ADDRFORM can change sk->sk_prot under us. */ 3711 return READ_ONCE(sk->sk_prot)->setsockopt(sk, level, optname, optval, optlen); 3712 } 3713 EXPORT_SYMBOL(sock_common_setsockopt); 3714 3715 void sk_common_release(struct sock *sk) 3716 { 3717 if (sk->sk_prot->destroy) 3718 sk->sk_prot->destroy(sk); 3719 3720 /* 3721 * Observation: when sk_common_release is called, processes have 3722 * no access to socket. But net still has. 3723 * Step one, detach it from networking: 3724 * 3725 * A. Remove from hash tables. 3726 */ 3727 3728 sk->sk_prot->unhash(sk); 3729 3730 /* 3731 * In this point socket cannot receive new packets, but it is possible 3732 * that some packets are in flight because some CPU runs receiver and 3733 * did hash table lookup before we unhashed socket. They will achieve 3734 * receive queue and will be purged by socket destructor. 3735 * 3736 * Also we still have packets pending on receive queue and probably, 3737 * our own packets waiting in device queues. sock_destroy will drain 3738 * receive queue, but transmitted packets will delay socket destruction 3739 * until the last reference will be released. 3740 */ 3741 3742 sock_orphan(sk); 3743 3744 xfrm_sk_free_policy(sk); 3745 3746 sock_put(sk); 3747 } 3748 EXPORT_SYMBOL(sk_common_release); 3749 3750 void sk_get_meminfo(const struct sock *sk, u32 *mem) 3751 { 3752 memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS); 3753 3754 mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk); 3755 mem[SK_MEMINFO_RCVBUF] = READ_ONCE(sk->sk_rcvbuf); 3756 mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk); 3757 mem[SK_MEMINFO_SNDBUF] = READ_ONCE(sk->sk_sndbuf); 3758 mem[SK_MEMINFO_FWD_ALLOC] = sk_forward_alloc_get(sk); 3759 mem[SK_MEMINFO_WMEM_QUEUED] = READ_ONCE(sk->sk_wmem_queued); 3760 mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc); 3761 mem[SK_MEMINFO_BACKLOG] = READ_ONCE(sk->sk_backlog.len); 3762 mem[SK_MEMINFO_DROPS] = atomic_read(&sk->sk_drops); 3763 } 3764 3765 #ifdef CONFIG_PROC_FS 3766 static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR); 3767 3768 int sock_prot_inuse_get(struct net *net, struct proto *prot) 3769 { 3770 int cpu, idx = prot->inuse_idx; 3771 int res = 0; 3772 3773 for_each_possible_cpu(cpu) 3774 res += per_cpu_ptr(net->core.prot_inuse, cpu)->val[idx]; 3775 3776 return res >= 0 ? res : 0; 3777 } 3778 EXPORT_SYMBOL_GPL(sock_prot_inuse_get); 3779 3780 int sock_inuse_get(struct net *net) 3781 { 3782 int cpu, res = 0; 3783 3784 for_each_possible_cpu(cpu) 3785 res += per_cpu_ptr(net->core.prot_inuse, cpu)->all; 3786 3787 return res; 3788 } 3789 3790 EXPORT_SYMBOL_GPL(sock_inuse_get); 3791 3792 static int __net_init sock_inuse_init_net(struct net *net) 3793 { 3794 net->core.prot_inuse = alloc_percpu(struct prot_inuse); 3795 if (net->core.prot_inuse == NULL) 3796 return -ENOMEM; 3797 return 0; 3798 } 3799 3800 static void __net_exit sock_inuse_exit_net(struct net *net) 3801 { 3802 free_percpu(net->core.prot_inuse); 3803 } 3804 3805 static struct pernet_operations net_inuse_ops = { 3806 .init = sock_inuse_init_net, 3807 .exit = sock_inuse_exit_net, 3808 }; 3809 3810 static __init int net_inuse_init(void) 3811 { 3812 if (register_pernet_subsys(&net_inuse_ops)) 3813 panic("Cannot initialize net inuse counters"); 3814 3815 return 0; 3816 } 3817 3818 core_initcall(net_inuse_init); 3819 3820 static int assign_proto_idx(struct proto *prot) 3821 { 3822 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR); 3823 3824 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) { 3825 pr_err("PROTO_INUSE_NR exhausted\n"); 3826 return -ENOSPC; 3827 } 3828 3829 set_bit(prot->inuse_idx, proto_inuse_idx); 3830 return 0; 3831 } 3832 3833 static void release_proto_idx(struct proto *prot) 3834 { 3835 if (prot->inuse_idx != PROTO_INUSE_NR - 1) 3836 clear_bit(prot->inuse_idx, proto_inuse_idx); 3837 } 3838 #else 3839 static inline int assign_proto_idx(struct proto *prot) 3840 { 3841 return 0; 3842 } 3843 3844 static inline void release_proto_idx(struct proto *prot) 3845 { 3846 } 3847 3848 #endif 3849 3850 static void tw_prot_cleanup(struct timewait_sock_ops *twsk_prot) 3851 { 3852 if (!twsk_prot) 3853 return; 3854 kfree(twsk_prot->twsk_slab_name); 3855 twsk_prot->twsk_slab_name = NULL; 3856 kmem_cache_destroy(twsk_prot->twsk_slab); 3857 twsk_prot->twsk_slab = NULL; 3858 } 3859 3860 static int tw_prot_init(const struct proto *prot) 3861 { 3862 struct timewait_sock_ops *twsk_prot = prot->twsk_prot; 3863 3864 if (!twsk_prot) 3865 return 0; 3866 3867 twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", 3868 prot->name); 3869 if (!twsk_prot->twsk_slab_name) 3870 return -ENOMEM; 3871 3872 twsk_prot->twsk_slab = 3873 kmem_cache_create(twsk_prot->twsk_slab_name, 3874 twsk_prot->twsk_obj_size, 0, 3875 SLAB_ACCOUNT | prot->slab_flags, 3876 NULL); 3877 if (!twsk_prot->twsk_slab) { 3878 pr_crit("%s: Can't create timewait sock SLAB cache!\n", 3879 prot->name); 3880 return -ENOMEM; 3881 } 3882 3883 return 0; 3884 } 3885 3886 static void req_prot_cleanup(struct request_sock_ops *rsk_prot) 3887 { 3888 if (!rsk_prot) 3889 return; 3890 kfree(rsk_prot->slab_name); 3891 rsk_prot->slab_name = NULL; 3892 kmem_cache_destroy(rsk_prot->slab); 3893 rsk_prot->slab = NULL; 3894 } 3895 3896 static int req_prot_init(const struct proto *prot) 3897 { 3898 struct request_sock_ops *rsk_prot = prot->rsk_prot; 3899 3900 if (!rsk_prot) 3901 return 0; 3902 3903 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s", 3904 prot->name); 3905 if (!rsk_prot->slab_name) 3906 return -ENOMEM; 3907 3908 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name, 3909 rsk_prot->obj_size, 0, 3910 SLAB_ACCOUNT | prot->slab_flags, 3911 NULL); 3912 3913 if (!rsk_prot->slab) { 3914 pr_crit("%s: Can't create request sock SLAB cache!\n", 3915 prot->name); 3916 return -ENOMEM; 3917 } 3918 return 0; 3919 } 3920 3921 int proto_register(struct proto *prot, int alloc_slab) 3922 { 3923 int ret = -ENOBUFS; 3924 3925 if (prot->memory_allocated && !prot->sysctl_mem) { 3926 pr_err("%s: missing sysctl_mem\n", prot->name); 3927 return -EINVAL; 3928 } 3929 if (prot->memory_allocated && !prot->per_cpu_fw_alloc) { 3930 pr_err("%s: missing per_cpu_fw_alloc\n", prot->name); 3931 return -EINVAL; 3932 } 3933 if (alloc_slab) { 3934 prot->slab = kmem_cache_create_usercopy(prot->name, 3935 prot->obj_size, 0, 3936 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT | 3937 prot->slab_flags, 3938 prot->useroffset, prot->usersize, 3939 NULL); 3940 3941 if (prot->slab == NULL) { 3942 pr_crit("%s: Can't create sock SLAB cache!\n", 3943 prot->name); 3944 goto out; 3945 } 3946 3947 if (req_prot_init(prot)) 3948 goto out_free_request_sock_slab; 3949 3950 if (tw_prot_init(prot)) 3951 goto out_free_timewait_sock_slab; 3952 } 3953 3954 mutex_lock(&proto_list_mutex); 3955 ret = assign_proto_idx(prot); 3956 if (ret) { 3957 mutex_unlock(&proto_list_mutex); 3958 goto out_free_timewait_sock_slab; 3959 } 3960 list_add(&prot->node, &proto_list); 3961 mutex_unlock(&proto_list_mutex); 3962 return ret; 3963 3964 out_free_timewait_sock_slab: 3965 if (alloc_slab) 3966 tw_prot_cleanup(prot->twsk_prot); 3967 out_free_request_sock_slab: 3968 if (alloc_slab) { 3969 req_prot_cleanup(prot->rsk_prot); 3970 3971 kmem_cache_destroy(prot->slab); 3972 prot->slab = NULL; 3973 } 3974 out: 3975 return ret; 3976 } 3977 EXPORT_SYMBOL(proto_register); 3978 3979 void proto_unregister(struct proto *prot) 3980 { 3981 mutex_lock(&proto_list_mutex); 3982 release_proto_idx(prot); 3983 list_del(&prot->node); 3984 mutex_unlock(&proto_list_mutex); 3985 3986 kmem_cache_destroy(prot->slab); 3987 prot->slab = NULL; 3988 3989 req_prot_cleanup(prot->rsk_prot); 3990 tw_prot_cleanup(prot->twsk_prot); 3991 } 3992 EXPORT_SYMBOL(proto_unregister); 3993 3994 int sock_load_diag_module(int family, int protocol) 3995 { 3996 if (!protocol) { 3997 if (!sock_is_registered(family)) 3998 return -ENOENT; 3999 4000 return request_module("net-pf-%d-proto-%d-type-%d", PF_NETLINK, 4001 NETLINK_SOCK_DIAG, family); 4002 } 4003 4004 #ifdef CONFIG_INET 4005 if (family == AF_INET && 4006 protocol != IPPROTO_RAW && 4007 protocol < MAX_INET_PROTOS && 4008 !rcu_access_pointer(inet_protos[protocol])) 4009 return -ENOENT; 4010 #endif 4011 4012 return request_module("net-pf-%d-proto-%d-type-%d-%d", PF_NETLINK, 4013 NETLINK_SOCK_DIAG, family, protocol); 4014 } 4015 EXPORT_SYMBOL(sock_load_diag_module); 4016 4017 #ifdef CONFIG_PROC_FS 4018 static void *proto_seq_start(struct seq_file *seq, loff_t *pos) 4019 __acquires(proto_list_mutex) 4020 { 4021 mutex_lock(&proto_list_mutex); 4022 return seq_list_start_head(&proto_list, *pos); 4023 } 4024 4025 static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos) 4026 { 4027 return seq_list_next(v, &proto_list, pos); 4028 } 4029 4030 static void proto_seq_stop(struct seq_file *seq, void *v) 4031 __releases(proto_list_mutex) 4032 { 4033 mutex_unlock(&proto_list_mutex); 4034 } 4035 4036 static char proto_method_implemented(const void *method) 4037 { 4038 return method == NULL ? 'n' : 'y'; 4039 } 4040 static long sock_prot_memory_allocated(struct proto *proto) 4041 { 4042 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L; 4043 } 4044 4045 static const char *sock_prot_memory_pressure(struct proto *proto) 4046 { 4047 return proto->memory_pressure != NULL ? 4048 proto_memory_pressure(proto) ? "yes" : "no" : "NI"; 4049 } 4050 4051 static void proto_seq_printf(struct seq_file *seq, struct proto *proto) 4052 { 4053 4054 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s " 4055 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n", 4056 proto->name, 4057 proto->obj_size, 4058 sock_prot_inuse_get(seq_file_net(seq), proto), 4059 sock_prot_memory_allocated(proto), 4060 sock_prot_memory_pressure(proto), 4061 proto->max_header, 4062 proto->slab == NULL ? "no" : "yes", 4063 module_name(proto->owner), 4064 proto_method_implemented(proto->close), 4065 proto_method_implemented(proto->connect), 4066 proto_method_implemented(proto->disconnect), 4067 proto_method_implemented(proto->accept), 4068 proto_method_implemented(proto->ioctl), 4069 proto_method_implemented(proto->init), 4070 proto_method_implemented(proto->destroy), 4071 proto_method_implemented(proto->shutdown), 4072 proto_method_implemented(proto->setsockopt), 4073 proto_method_implemented(proto->getsockopt), 4074 proto_method_implemented(proto->sendmsg), 4075 proto_method_implemented(proto->recvmsg), 4076 proto_method_implemented(proto->bind), 4077 proto_method_implemented(proto->backlog_rcv), 4078 proto_method_implemented(proto->hash), 4079 proto_method_implemented(proto->unhash), 4080 proto_method_implemented(proto->get_port), 4081 proto_method_implemented(proto->enter_memory_pressure)); 4082 } 4083 4084 static int proto_seq_show(struct seq_file *seq, void *v) 4085 { 4086 if (v == &proto_list) 4087 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s", 4088 "protocol", 4089 "size", 4090 "sockets", 4091 "memory", 4092 "press", 4093 "maxhdr", 4094 "slab", 4095 "module", 4096 "cl co di ac io in de sh ss gs se re bi br ha uh gp em\n"); 4097 else 4098 proto_seq_printf(seq, list_entry(v, struct proto, node)); 4099 return 0; 4100 } 4101 4102 static const struct seq_operations proto_seq_ops = { 4103 .start = proto_seq_start, 4104 .next = proto_seq_next, 4105 .stop = proto_seq_stop, 4106 .show = proto_seq_show, 4107 }; 4108 4109 static __net_init int proto_init_net(struct net *net) 4110 { 4111 if (!proc_create_net("protocols", 0444, net->proc_net, &proto_seq_ops, 4112 sizeof(struct seq_net_private))) 4113 return -ENOMEM; 4114 4115 return 0; 4116 } 4117 4118 static __net_exit void proto_exit_net(struct net *net) 4119 { 4120 remove_proc_entry("protocols", net->proc_net); 4121 } 4122 4123 4124 static __net_initdata struct pernet_operations proto_net_ops = { 4125 .init = proto_init_net, 4126 .exit = proto_exit_net, 4127 }; 4128 4129 static int __init proto_init(void) 4130 { 4131 return register_pernet_subsys(&proto_net_ops); 4132 } 4133 4134 subsys_initcall(proto_init); 4135 4136 #endif /* PROC_FS */ 4137 4138 #ifdef CONFIG_NET_RX_BUSY_POLL 4139 bool sk_busy_loop_end(void *p, unsigned long start_time) 4140 { 4141 struct sock *sk = p; 4142 4143 return !skb_queue_empty_lockless(&sk->sk_receive_queue) || 4144 sk_busy_loop_timeout(sk, start_time); 4145 } 4146 EXPORT_SYMBOL(sk_busy_loop_end); 4147 #endif /* CONFIG_NET_RX_BUSY_POLL */ 4148 4149 int sock_bind_add(struct sock *sk, struct sockaddr *addr, int addr_len) 4150 { 4151 if (!sk->sk_prot->bind_add) 4152 return -EOPNOTSUPP; 4153 return sk->sk_prot->bind_add(sk, addr, addr_len); 4154 } 4155 EXPORT_SYMBOL(sock_bind_add); 4156 4157 /* Copy 'size' bytes from userspace and return `size` back to userspace */ 4158 int sock_ioctl_inout(struct sock *sk, unsigned int cmd, 4159 void __user *arg, void *karg, size_t size) 4160 { 4161 int ret; 4162 4163 if (copy_from_user(karg, arg, size)) 4164 return -EFAULT; 4165 4166 ret = READ_ONCE(sk->sk_prot)->ioctl(sk, cmd, karg); 4167 if (ret) 4168 return ret; 4169 4170 if (copy_to_user(arg, karg, size)) 4171 return -EFAULT; 4172 4173 return 0; 4174 } 4175 EXPORT_SYMBOL(sock_ioctl_inout); 4176 4177 /* This is the most common ioctl prep function, where the result (4 bytes) is 4178 * copied back to userspace if the ioctl() returns successfully. No input is 4179 * copied from userspace as input argument. 4180 */ 4181 static int sock_ioctl_out(struct sock *sk, unsigned int cmd, void __user *arg) 4182 { 4183 int ret, karg = 0; 4184 4185 ret = READ_ONCE(sk->sk_prot)->ioctl(sk, cmd, &karg); 4186 if (ret) 4187 return ret; 4188 4189 return put_user(karg, (int __user *)arg); 4190 } 4191 4192 /* A wrapper around sock ioctls, which copies the data from userspace 4193 * (depending on the protocol/ioctl), and copies back the result to userspace. 4194 * The main motivation for this function is to pass kernel memory to the 4195 * protocol ioctl callbacks, instead of userspace memory. 4196 */ 4197 int sk_ioctl(struct sock *sk, unsigned int cmd, void __user *arg) 4198 { 4199 int rc = 1; 4200 4201 if (sk->sk_type == SOCK_RAW && sk->sk_family == AF_INET) 4202 rc = ipmr_sk_ioctl(sk, cmd, arg); 4203 else if (sk->sk_type == SOCK_RAW && sk->sk_family == AF_INET6) 4204 rc = ip6mr_sk_ioctl(sk, cmd, arg); 4205 else if (sk_is_phonet(sk)) 4206 rc = phonet_sk_ioctl(sk, cmd, arg); 4207 4208 /* If ioctl was processed, returns its value */ 4209 if (rc <= 0) 4210 return rc; 4211 4212 /* Otherwise call the default handler */ 4213 return sock_ioctl_out(sk, cmd, arg); 4214 } 4215 EXPORT_SYMBOL(sk_ioctl); 4216