1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * Implementation of the Transmission Control Protocol(TCP). 8 * 9 * Authors: Ross Biro 10 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 11 * Mark Evans, <evansmp@uhura.aston.ac.uk> 12 * Corey Minyard <wf-rch!minyard@relay.EU.net> 13 * Florian La Roche, <flla@stud.uni-sb.de> 14 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu> 15 * Linus Torvalds, <torvalds@cs.helsinki.fi> 16 * Alan Cox, <gw4pts@gw4pts.ampr.org> 17 * Matthew Dillon, <dillon@apollo.west.oic.com> 18 * Arnt Gulbrandsen, <agulbra@nvg.unit.no> 19 * Jorge Cwik, <jorge@laser.satlink.net> 20 * 21 * Fixes: 22 * Alan Cox : Numerous verify_area() calls 23 * Alan Cox : Set the ACK bit on a reset 24 * Alan Cox : Stopped it crashing if it closed while 25 * sk->inuse=1 and was trying to connect 26 * (tcp_err()). 27 * Alan Cox : All icmp error handling was broken 28 * pointers passed where wrong and the 29 * socket was looked up backwards. Nobody 30 * tested any icmp error code obviously. 31 * Alan Cox : tcp_err() now handled properly. It 32 * wakes people on errors. poll 33 * behaves and the icmp error race 34 * has gone by moving it into sock.c 35 * Alan Cox : tcp_send_reset() fixed to work for 36 * everything not just packets for 37 * unknown sockets. 38 * Alan Cox : tcp option processing. 39 * Alan Cox : Reset tweaked (still not 100%) [Had 40 * syn rule wrong] 41 * Herp Rosmanith : More reset fixes 42 * Alan Cox : No longer acks invalid rst frames. 43 * Acking any kind of RST is right out. 44 * Alan Cox : Sets an ignore me flag on an rst 45 * receive otherwise odd bits of prattle 46 * escape still 47 * Alan Cox : Fixed another acking RST frame bug. 48 * Should stop LAN workplace lockups. 49 * Alan Cox : Some tidyups using the new skb list 50 * facilities 51 * Alan Cox : sk->keepopen now seems to work 52 * Alan Cox : Pulls options out correctly on accepts 53 * Alan Cox : Fixed assorted sk->rqueue->next errors 54 * Alan Cox : PSH doesn't end a TCP read. Switched a 55 * bit to skb ops. 56 * Alan Cox : Tidied tcp_data to avoid a potential 57 * nasty. 58 * Alan Cox : Added some better commenting, as the 59 * tcp is hard to follow 60 * Alan Cox : Removed incorrect check for 20 * psh 61 * Michael O'Reilly : ack < copied bug fix. 62 * Johannes Stille : Misc tcp fixes (not all in yet). 63 * Alan Cox : FIN with no memory -> CRASH 64 * Alan Cox : Added socket option proto entries. 65 * Also added awareness of them to accept. 66 * Alan Cox : Added TCP options (SOL_TCP) 67 * Alan Cox : Switched wakeup calls to callbacks, 68 * so the kernel can layer network 69 * sockets. 70 * Alan Cox : Use ip_tos/ip_ttl settings. 71 * Alan Cox : Handle FIN (more) properly (we hope). 72 * Alan Cox : RST frames sent on unsynchronised 73 * state ack error. 74 * Alan Cox : Put in missing check for SYN bit. 75 * Alan Cox : Added tcp_select_window() aka NET2E 76 * window non shrink trick. 77 * Alan Cox : Added a couple of small NET2E timer 78 * fixes 79 * Charles Hedrick : TCP fixes 80 * Toomas Tamm : TCP window fixes 81 * Alan Cox : Small URG fix to rlogin ^C ack fight 82 * Charles Hedrick : Rewrote most of it to actually work 83 * Linus : Rewrote tcp_read() and URG handling 84 * completely 85 * Gerhard Koerting: Fixed some missing timer handling 86 * Matthew Dillon : Reworked TCP machine states as per RFC 87 * Gerhard Koerting: PC/TCP workarounds 88 * Adam Caldwell : Assorted timer/timing errors 89 * Matthew Dillon : Fixed another RST bug 90 * Alan Cox : Move to kernel side addressing changes. 91 * Alan Cox : Beginning work on TCP fastpathing 92 * (not yet usable) 93 * Arnt Gulbrandsen: Turbocharged tcp_check() routine. 94 * Alan Cox : TCP fast path debugging 95 * Alan Cox : Window clamping 96 * Michael Riepe : Bug in tcp_check() 97 * Matt Dillon : More TCP improvements and RST bug fixes 98 * Matt Dillon : Yet more small nasties remove from the 99 * TCP code (Be very nice to this man if 100 * tcp finally works 100%) 8) 101 * Alan Cox : BSD accept semantics. 102 * Alan Cox : Reset on closedown bug. 103 * Peter De Schrijver : ENOTCONN check missing in tcp_sendto(). 104 * Michael Pall : Handle poll() after URG properly in 105 * all cases. 106 * Michael Pall : Undo the last fix in tcp_read_urg() 107 * (multi URG PUSH broke rlogin). 108 * Michael Pall : Fix the multi URG PUSH problem in 109 * tcp_readable(), poll() after URG 110 * works now. 111 * Michael Pall : recv(...,MSG_OOB) never blocks in the 112 * BSD api. 113 * Alan Cox : Changed the semantics of sk->socket to 114 * fix a race and a signal problem with 115 * accept() and async I/O. 116 * Alan Cox : Relaxed the rules on tcp_sendto(). 117 * Yury Shevchuk : Really fixed accept() blocking problem. 118 * Craig I. Hagan : Allow for BSD compatible TIME_WAIT for 119 * clients/servers which listen in on 120 * fixed ports. 121 * Alan Cox : Cleaned the above up and shrank it to 122 * a sensible code size. 123 * Alan Cox : Self connect lockup fix. 124 * Alan Cox : No connect to multicast. 125 * Ross Biro : Close unaccepted children on master 126 * socket close. 127 * Alan Cox : Reset tracing code. 128 * Alan Cox : Spurious resets on shutdown. 129 * Alan Cox : Giant 15 minute/60 second timer error 130 * Alan Cox : Small whoops in polling before an 131 * accept. 132 * Alan Cox : Kept the state trace facility since 133 * it's handy for debugging. 134 * Alan Cox : More reset handler fixes. 135 * Alan Cox : Started rewriting the code based on 136 * the RFC's for other useful protocol 137 * references see: Comer, KA9Q NOS, and 138 * for a reference on the difference 139 * between specifications and how BSD 140 * works see the 4.4lite source. 141 * A.N.Kuznetsov : Don't time wait on completion of tidy 142 * close. 143 * Linus Torvalds : Fin/Shutdown & copied_seq changes. 144 * Linus Torvalds : Fixed BSD port reuse to work first syn 145 * Alan Cox : Reimplemented timers as per the RFC 146 * and using multiple timers for sanity. 147 * Alan Cox : Small bug fixes, and a lot of new 148 * comments. 149 * Alan Cox : Fixed dual reader crash by locking 150 * the buffers (much like datagram.c) 151 * Alan Cox : Fixed stuck sockets in probe. A probe 152 * now gets fed up of retrying without 153 * (even a no space) answer. 154 * Alan Cox : Extracted closing code better 155 * Alan Cox : Fixed the closing state machine to 156 * resemble the RFC. 157 * Alan Cox : More 'per spec' fixes. 158 * Jorge Cwik : Even faster checksumming. 159 * Alan Cox : tcp_data() doesn't ack illegal PSH 160 * only frames. At least one pc tcp stack 161 * generates them. 162 * Alan Cox : Cache last socket. 163 * Alan Cox : Per route irtt. 164 * Matt Day : poll()->select() match BSD precisely on error 165 * Alan Cox : New buffers 166 * Marc Tamsky : Various sk->prot->retransmits and 167 * sk->retransmits misupdating fixed. 168 * Fixed tcp_write_timeout: stuck close, 169 * and TCP syn retries gets used now. 170 * Mark Yarvis : In tcp_read_wakeup(), don't send an 171 * ack if state is TCP_CLOSED. 172 * Alan Cox : Look up device on a retransmit - routes may 173 * change. Doesn't yet cope with MSS shrink right 174 * but it's a start! 175 * Marc Tamsky : Closing in closing fixes. 176 * Mike Shaver : RFC1122 verifications. 177 * Alan Cox : rcv_saddr errors. 178 * Alan Cox : Block double connect(). 179 * Alan Cox : Small hooks for enSKIP. 180 * Alexey Kuznetsov: Path MTU discovery. 181 * Alan Cox : Support soft errors. 182 * Alan Cox : Fix MTU discovery pathological case 183 * when the remote claims no mtu! 184 * Marc Tamsky : TCP_CLOSE fix. 185 * Colin (G3TNE) : Send a reset on syn ack replies in 186 * window but wrong (fixes NT lpd problems) 187 * Pedro Roque : Better TCP window handling, delayed ack. 188 * Joerg Reuter : No modification of locked buffers in 189 * tcp_do_retransmit() 190 * Eric Schenk : Changed receiver side silly window 191 * avoidance algorithm to BSD style 192 * algorithm. This doubles throughput 193 * against machines running Solaris, 194 * and seems to result in general 195 * improvement. 196 * Stefan Magdalinski : adjusted tcp_readable() to fix FIONREAD 197 * Willy Konynenberg : Transparent proxying support. 198 * Mike McLagan : Routing by source 199 * Keith Owens : Do proper merging with partial SKB's in 200 * tcp_do_sendmsg to avoid burstiness. 201 * Eric Schenk : Fix fast close down bug with 202 * shutdown() followed by close(). 203 * Andi Kleen : Make poll agree with SIGIO 204 * Salvatore Sanfilippo : Support SO_LINGER with linger == 1 and 205 * lingertime == 0 (RFC 793 ABORT Call) 206 * Hirokazu Takahashi : Use copy_from_user() instead of 207 * csum_and_copy_from_user() if possible. 208 * 209 * Description of States: 210 * 211 * TCP_SYN_SENT sent a connection request, waiting for ack 212 * 213 * TCP_SYN_RECV received a connection request, sent ack, 214 * waiting for final ack in three-way handshake. 215 * 216 * TCP_ESTABLISHED connection established 217 * 218 * TCP_FIN_WAIT1 our side has shutdown, waiting to complete 219 * transmission of remaining buffered data 220 * 221 * TCP_FIN_WAIT2 all buffered data sent, waiting for remote 222 * to shutdown 223 * 224 * TCP_CLOSING both sides have shutdown but we still have 225 * data we have to finish sending 226 * 227 * TCP_TIME_WAIT timeout to catch resent junk before entering 228 * closed, can only be entered from FIN_WAIT2 229 * or CLOSING. Required because the other end 230 * may not have gotten our last ACK causing it 231 * to retransmit the data packet (which we ignore) 232 * 233 * TCP_CLOSE_WAIT remote side has shutdown and is waiting for 234 * us to finish writing our data and to shutdown 235 * (we have to close() to move on to LAST_ACK) 236 * 237 * TCP_LAST_ACK out side has shutdown after remote has 238 * shutdown. There may still be data in our 239 * buffer that we have to finish sending 240 * 241 * TCP_CLOSE socket is finished 242 */ 243 244 #define pr_fmt(fmt) "TCP: " fmt 245 246 #include <crypto/hash.h> 247 #include <linux/kernel.h> 248 #include <linux/module.h> 249 #include <linux/types.h> 250 #include <linux/fcntl.h> 251 #include <linux/poll.h> 252 #include <linux/inet_diag.h> 253 #include <linux/init.h> 254 #include <linux/fs.h> 255 #include <linux/skbuff.h> 256 #include <linux/scatterlist.h> 257 #include <linux/splice.h> 258 #include <linux/net.h> 259 #include <linux/socket.h> 260 #include <linux/random.h> 261 #include <linux/memblock.h> 262 #include <linux/highmem.h> 263 #include <linux/cache.h> 264 #include <linux/err.h> 265 #include <linux/time.h> 266 #include <linux/slab.h> 267 #include <linux/errqueue.h> 268 #include <linux/static_key.h> 269 #include <linux/btf.h> 270 271 #include <net/icmp.h> 272 #include <net/inet_common.h> 273 #include <net/inet_ecn.h> 274 #include <net/tcp.h> 275 #include <net/tcp_ecn.h> 276 #include <net/mptcp.h> 277 #include <net/proto_memory.h> 278 #include <net/xfrm.h> 279 #include <net/ip.h> 280 #include <net/psp.h> 281 #include <net/sock.h> 282 #include <net/rstreason.h> 283 284 #include <linux/uaccess.h> 285 #include <asm/ioctls.h> 286 #include <net/busy_poll.h> 287 #include <net/hotdata.h> 288 #include <trace/events/tcp.h> 289 #include <net/rps.h> 290 291 #include "../core/devmem.h" 292 293 /* Track pending CMSGs. */ 294 enum { 295 TCP_CMSG_INQ = 1, 296 TCP_CMSG_TS = 2 297 }; 298 299 DEFINE_PER_CPU(unsigned int, tcp_orphan_count); 300 EXPORT_PER_CPU_SYMBOL_GPL(tcp_orphan_count); 301 302 DEFINE_PER_CPU(u32, tcp_tw_isn); 303 EXPORT_PER_CPU_SYMBOL_GPL(tcp_tw_isn); 304 305 long sysctl_tcp_mem[3] __read_mostly; 306 EXPORT_IPV6_MOD(sysctl_tcp_mem); 307 308 DEFINE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc); 309 EXPORT_PER_CPU_SYMBOL_GPL(tcp_memory_per_cpu_fw_alloc); 310 311 #if IS_ENABLED(CONFIG_SMC) 312 DEFINE_STATIC_KEY_FALSE(tcp_have_smc); 313 EXPORT_SYMBOL(tcp_have_smc); 314 #endif 315 316 /* 317 * Current number of TCP sockets. 318 */ 319 struct percpu_counter tcp_sockets_allocated ____cacheline_aligned_in_smp; 320 EXPORT_IPV6_MOD(tcp_sockets_allocated); 321 322 /* 323 * TCP splice context 324 */ 325 struct tcp_splice_state { 326 struct pipe_inode_info *pipe; 327 size_t len; 328 unsigned int flags; 329 }; 330 331 /* 332 * Pressure flag: try to collapse. 333 * Technical note: it is used by multiple contexts non atomically. 334 * All the __sk_mem_schedule() is of this nature: accounting 335 * is strict, actions are advisory and have some latency. 336 */ 337 unsigned long tcp_memory_pressure __read_mostly; 338 EXPORT_SYMBOL_GPL(tcp_memory_pressure); 339 340 void tcp_enter_memory_pressure(struct sock *sk) 341 { 342 unsigned long val; 343 344 if (READ_ONCE(tcp_memory_pressure)) 345 return; 346 val = jiffies; 347 348 if (!val) 349 val--; 350 if (!cmpxchg(&tcp_memory_pressure, 0, val)) 351 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURES); 352 } 353 EXPORT_IPV6_MOD_GPL(tcp_enter_memory_pressure); 354 355 void tcp_leave_memory_pressure(struct sock *sk) 356 { 357 unsigned long val; 358 359 if (!READ_ONCE(tcp_memory_pressure)) 360 return; 361 val = xchg(&tcp_memory_pressure, 0); 362 if (val) 363 NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURESCHRONO, 364 jiffies_to_msecs(jiffies - val)); 365 } 366 EXPORT_IPV6_MOD_GPL(tcp_leave_memory_pressure); 367 368 /* Convert seconds to retransmits based on initial and max timeout */ 369 static u8 secs_to_retrans(int seconds, int timeout, int rto_max) 370 { 371 u8 res = 0; 372 373 if (seconds > 0) { 374 int period = timeout; 375 376 res = 1; 377 while (seconds > period && res < 255) { 378 res++; 379 timeout <<= 1; 380 if (timeout > rto_max) 381 timeout = rto_max; 382 period += timeout; 383 } 384 } 385 return res; 386 } 387 388 /* Convert retransmits to seconds based on initial and max timeout */ 389 static int retrans_to_secs(u8 retrans, int timeout, int rto_max) 390 { 391 int period = 0; 392 393 if (retrans > 0) { 394 period = timeout; 395 while (--retrans) { 396 timeout <<= 1; 397 if (timeout > rto_max) 398 timeout = rto_max; 399 period += timeout; 400 } 401 } 402 return period; 403 } 404 405 static u64 tcp_compute_delivery_rate(const struct tcp_sock *tp) 406 { 407 u32 rate = READ_ONCE(tp->rate_delivered); 408 u32 intv = READ_ONCE(tp->rate_interval_us); 409 u64 rate64 = 0; 410 411 if (rate && intv) { 412 rate64 = (u64)rate * tp->mss_cache * USEC_PER_SEC; 413 do_div(rate64, intv); 414 } 415 return rate64; 416 } 417 418 #ifdef CONFIG_TCP_MD5SIG 419 void tcp_md5_destruct_sock(struct sock *sk) 420 { 421 struct tcp_sock *tp = tcp_sk(sk); 422 423 if (tp->md5sig_info) { 424 425 tcp_clear_md5_list(sk); 426 kfree(rcu_replace_pointer(tp->md5sig_info, NULL, 1)); 427 static_branch_slow_dec_deferred(&tcp_md5_needed); 428 tcp_md5_release_sigpool(); 429 } 430 } 431 EXPORT_IPV6_MOD_GPL(tcp_md5_destruct_sock); 432 #endif 433 434 /* Address-family independent initialization for a tcp_sock. 435 * 436 * NOTE: A lot of things set to zero explicitly by call to 437 * sk_alloc() so need not be done here. 438 */ 439 void tcp_init_sock(struct sock *sk) 440 { 441 struct inet_connection_sock *icsk = inet_csk(sk); 442 struct tcp_sock *tp = tcp_sk(sk); 443 int rto_min_us, rto_max_ms; 444 445 tp->out_of_order_queue = RB_ROOT; 446 sk->tcp_rtx_queue = RB_ROOT; 447 tcp_init_xmit_timers(sk); 448 INIT_LIST_HEAD(&tp->tsq_node); 449 INIT_LIST_HEAD(&tp->tsorted_sent_queue); 450 451 icsk->icsk_rto = TCP_TIMEOUT_INIT; 452 453 rto_max_ms = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rto_max_ms); 454 icsk->icsk_rto_max = msecs_to_jiffies(rto_max_ms); 455 456 rto_min_us = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rto_min_us); 457 icsk->icsk_rto_min = usecs_to_jiffies(rto_min_us); 458 icsk->icsk_delack_max = TCP_DELACK_MAX; 459 tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT); 460 minmax_reset(&tp->rtt_min, tcp_jiffies32, ~0U); 461 462 /* So many TCP implementations out there (incorrectly) count the 463 * initial SYN frame in their delayed-ACK and congestion control 464 * algorithms that we must have the following bandaid to talk 465 * efficiently to them. -DaveM 466 */ 467 tcp_snd_cwnd_set(tp, TCP_INIT_CWND); 468 469 /* There's a bubble in the pipe until at least the first ACK. */ 470 tp->app_limited = ~0U; 471 tp->rate_app_limited = 1; 472 473 /* See draft-stevens-tcpca-spec-01 for discussion of the 474 * initialization of these values. 475 */ 476 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; 477 tp->snd_cwnd_clamp = ~0; 478 tp->mss_cache = TCP_MSS_DEFAULT; 479 480 tp->reordering = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reordering); 481 tcp_assign_congestion_control(sk); 482 483 tp->tsoffset = 0; 484 tp->rack.reo_wnd_steps = 1; 485 486 sk->sk_write_space = sk_stream_write_space; 487 sock_set_flag(sk, SOCK_USE_WRITE_QUEUE); 488 489 icsk->icsk_sync_mss = tcp_sync_mss; 490 491 WRITE_ONCE(sk->sk_sndbuf, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_wmem[1])); 492 WRITE_ONCE(sk->sk_rcvbuf, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[1])); 493 tcp_scaling_ratio_init(sk); 494 495 set_bit(SOCK_SUPPORT_ZC, &sk->sk_socket->flags); 496 sk_sockets_allocated_inc(sk); 497 xa_init_flags(&sk->sk_user_frags, XA_FLAGS_ALLOC1); 498 } 499 EXPORT_IPV6_MOD(tcp_init_sock); 500 501 static void tcp_tx_timestamp(struct sock *sk, struct sockcm_cookie *sockc) 502 { 503 struct sk_buff *skb = tcp_write_queue_tail(sk); 504 u32 tsflags = sockc->tsflags; 505 506 if (tsflags && skb) { 507 struct skb_shared_info *shinfo = skb_shinfo(skb); 508 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb); 509 510 sock_tx_timestamp(sk, sockc, &shinfo->tx_flags); 511 if (tsflags & SOF_TIMESTAMPING_TX_ACK) 512 tcb->txstamp_ack |= TSTAMP_ACK_SK; 513 if (tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK) 514 shinfo->tskey = TCP_SKB_CB(skb)->seq + skb->len - 1; 515 } 516 517 if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) && 518 SK_BPF_CB_FLAG_TEST(sk, SK_BPF_CB_TX_TIMESTAMPING) && skb) 519 bpf_skops_tx_timestamping(sk, skb, BPF_SOCK_OPS_TSTAMP_SENDMSG_CB); 520 } 521 522 static bool tcp_stream_is_readable(struct sock *sk, int target) 523 { 524 if (tcp_epollin_ready(sk, target)) 525 return true; 526 return sk_is_readable(sk); 527 } 528 529 /* 530 * Wait for a TCP event. 531 * 532 * Note that we don't need to lock the socket, as the upper poll layers 533 * take care of normal races (between the test and the event) and we don't 534 * go look at any of the socket buffers directly. 535 */ 536 __poll_t tcp_poll(struct file *file, struct socket *sock, poll_table *wait) 537 { 538 __poll_t mask; 539 struct sock *sk = sock->sk; 540 const struct tcp_sock *tp = tcp_sk(sk); 541 u8 shutdown; 542 int state; 543 544 sock_poll_wait(file, sock, wait); 545 546 state = inet_sk_state_load(sk); 547 if (state == TCP_LISTEN) 548 return inet_csk_listen_poll(sk); 549 550 /* Socket is not locked. We are protected from async events 551 * by poll logic and correct handling of state changes 552 * made by other threads is impossible in any case. 553 */ 554 555 mask = 0; 556 557 /* 558 * EPOLLHUP is certainly not done right. But poll() doesn't 559 * have a notion of HUP in just one direction, and for a 560 * socket the read side is more interesting. 561 * 562 * Some poll() documentation says that EPOLLHUP is incompatible 563 * with the EPOLLOUT/POLLWR flags, so somebody should check this 564 * all. But careful, it tends to be safer to return too many 565 * bits than too few, and you can easily break real applications 566 * if you don't tell them that something has hung up! 567 * 568 * Check-me. 569 * 570 * Check number 1. EPOLLHUP is _UNMASKABLE_ event (see UNIX98 and 571 * our fs/select.c). It means that after we received EOF, 572 * poll always returns immediately, making impossible poll() on write() 573 * in state CLOSE_WAIT. One solution is evident --- to set EPOLLHUP 574 * if and only if shutdown has been made in both directions. 575 * Actually, it is interesting to look how Solaris and DUX 576 * solve this dilemma. I would prefer, if EPOLLHUP were maskable, 577 * then we could set it on SND_SHUTDOWN. BTW examples given 578 * in Stevens' books assume exactly this behaviour, it explains 579 * why EPOLLHUP is incompatible with EPOLLOUT. --ANK 580 * 581 * NOTE. Check for TCP_CLOSE is added. The goal is to prevent 582 * blocking on fresh not-connected or disconnected socket. --ANK 583 */ 584 shutdown = READ_ONCE(sk->sk_shutdown); 585 if (shutdown == SHUTDOWN_MASK || state == TCP_CLOSE) 586 mask |= EPOLLHUP; 587 if (shutdown & RCV_SHUTDOWN) 588 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 589 590 /* Connected or passive Fast Open socket? */ 591 if (state != TCP_SYN_SENT && 592 (state != TCP_SYN_RECV || rcu_access_pointer(tp->fastopen_rsk))) { 593 int target = sock_rcvlowat(sk, 0, INT_MAX); 594 u16 urg_data = READ_ONCE(tp->urg_data); 595 596 if (unlikely(urg_data) && 597 READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq) && 598 !sock_flag(sk, SOCK_URGINLINE)) 599 target++; 600 601 if (tcp_stream_is_readable(sk, target)) 602 mask |= EPOLLIN | EPOLLRDNORM; 603 604 if (!(shutdown & SEND_SHUTDOWN)) { 605 if (__sk_stream_is_writeable(sk, 1)) { 606 mask |= EPOLLOUT | EPOLLWRNORM; 607 } else { /* send SIGIO later */ 608 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 609 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 610 611 /* Race breaker. If space is freed after 612 * wspace test but before the flags are set, 613 * IO signal will be lost. Memory barrier 614 * pairs with the input side. 615 */ 616 smp_mb__after_atomic(); 617 if (__sk_stream_is_writeable(sk, 1)) 618 mask |= EPOLLOUT | EPOLLWRNORM; 619 } 620 } else 621 mask |= EPOLLOUT | EPOLLWRNORM; 622 623 if (urg_data & TCP_URG_VALID) 624 mask |= EPOLLPRI; 625 } else if (state == TCP_SYN_SENT && 626 inet_test_bit(DEFER_CONNECT, sk)) { 627 /* Active TCP fastopen socket with defer_connect 628 * Return EPOLLOUT so application can call write() 629 * in order for kernel to generate SYN+data 630 */ 631 mask |= EPOLLOUT | EPOLLWRNORM; 632 } 633 /* This barrier is coupled with smp_wmb() in tcp_done_with_error() */ 634 smp_rmb(); 635 if (READ_ONCE(sk->sk_err) || 636 !skb_queue_empty_lockless(&sk->sk_error_queue)) 637 mask |= EPOLLERR; 638 639 return mask; 640 } 641 EXPORT_SYMBOL(tcp_poll); 642 643 int tcp_ioctl(struct sock *sk, int cmd, int *karg) 644 { 645 struct tcp_sock *tp = tcp_sk(sk); 646 int answ; 647 bool slow; 648 649 switch (cmd) { 650 case SIOCINQ: 651 if (sk->sk_state == TCP_LISTEN) 652 return -EINVAL; 653 654 slow = lock_sock_fast(sk); 655 answ = tcp_inq(sk); 656 unlock_sock_fast(sk, slow); 657 break; 658 case SIOCATMARK: 659 answ = READ_ONCE(tp->urg_data) && 660 READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq); 661 break; 662 case SIOCOUTQ: 663 if (sk->sk_state == TCP_LISTEN) 664 return -EINVAL; 665 666 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) 667 answ = 0; 668 else 669 answ = READ_ONCE(tp->write_seq) - tp->snd_una; 670 break; 671 case SIOCOUTQNSD: 672 if (sk->sk_state == TCP_LISTEN) 673 return -EINVAL; 674 675 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) 676 answ = 0; 677 else 678 answ = READ_ONCE(tp->write_seq) - 679 READ_ONCE(tp->snd_nxt); 680 break; 681 default: 682 return -ENOIOCTLCMD; 683 } 684 685 *karg = answ; 686 return 0; 687 } 688 EXPORT_IPV6_MOD(tcp_ioctl); 689 690 void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb) 691 { 692 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH; 693 tp->pushed_seq = tp->write_seq; 694 } 695 696 static inline bool forced_push(const struct tcp_sock *tp) 697 { 698 return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1)); 699 } 700 701 void tcp_skb_entail(struct sock *sk, struct sk_buff *skb) 702 { 703 struct tcp_sock *tp = tcp_sk(sk); 704 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb); 705 706 tcb->seq = tcb->end_seq = tp->write_seq; 707 tcb->tcp_flags = TCPHDR_ACK; 708 __skb_header_release(skb); 709 psp_enqueue_set_decrypted(sk, skb); 710 tcp_add_write_queue_tail(sk, skb); 711 sk_wmem_queued_add(sk, skb->truesize); 712 sk_mem_charge(sk, skb->truesize); 713 if (tp->nonagle & TCP_NAGLE_PUSH) 714 tp->nonagle &= ~TCP_NAGLE_PUSH; 715 716 tcp_slow_start_after_idle_check(sk); 717 } 718 719 static inline void tcp_mark_urg(struct tcp_sock *tp, int flags) 720 { 721 if (flags & MSG_OOB) 722 tp->snd_up = tp->write_seq; 723 } 724 725 /* If a not yet filled skb is pushed, do not send it if 726 * we have data packets in Qdisc or NIC queues : 727 * Because TX completion will happen shortly, it gives a chance 728 * to coalesce future sendmsg() payload into this skb, without 729 * need for a timer, and with no latency trade off. 730 * As packets containing data payload have a bigger truesize 731 * than pure acks (dataless) packets, the last checks prevent 732 * autocorking if we only have an ACK in Qdisc/NIC queues, 733 * or if TX completion was delayed after we processed ACK packet. 734 */ 735 static bool tcp_should_autocork(struct sock *sk, struct sk_buff *skb, 736 int size_goal) 737 { 738 return skb->len < size_goal && 739 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_autocorking) && 740 !tcp_rtx_queue_empty(sk) && 741 refcount_read(&sk->sk_wmem_alloc) > skb->truesize && 742 tcp_skb_can_collapse_to(skb); 743 } 744 745 void tcp_push(struct sock *sk, int flags, int mss_now, 746 int nonagle, int size_goal) 747 { 748 struct tcp_sock *tp = tcp_sk(sk); 749 struct sk_buff *skb; 750 751 skb = tcp_write_queue_tail(sk); 752 if (!skb) 753 return; 754 if (!(flags & MSG_MORE) || forced_push(tp)) 755 tcp_mark_push(tp, skb); 756 757 tcp_mark_urg(tp, flags); 758 759 if (tcp_should_autocork(sk, skb, size_goal)) { 760 761 /* avoid atomic op if TSQ_THROTTLED bit is already set */ 762 if (!test_bit(TSQ_THROTTLED, &sk->sk_tsq_flags)) { 763 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING); 764 set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags); 765 smp_mb__after_atomic(); 766 } 767 /* It is possible TX completion already happened 768 * before we set TSQ_THROTTLED. 769 */ 770 if (refcount_read(&sk->sk_wmem_alloc) > skb->truesize) 771 return; 772 } 773 774 if (flags & MSG_MORE) 775 nonagle = TCP_NAGLE_CORK; 776 777 __tcp_push_pending_frames(sk, mss_now, nonagle); 778 } 779 780 static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb, 781 unsigned int offset, size_t len) 782 { 783 struct tcp_splice_state *tss = rd_desc->arg.data; 784 int ret; 785 786 ret = skb_splice_bits(skb, skb->sk, offset, tss->pipe, 787 min(rd_desc->count, len), tss->flags); 788 if (ret > 0) 789 rd_desc->count -= ret; 790 return ret; 791 } 792 793 static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss) 794 { 795 /* Store TCP splice context information in read_descriptor_t. */ 796 read_descriptor_t rd_desc = { 797 .arg.data = tss, 798 .count = tss->len, 799 }; 800 801 return tcp_read_sock(sk, &rd_desc, tcp_splice_data_recv); 802 } 803 804 /** 805 * tcp_splice_read - splice data from TCP socket to a pipe 806 * @sock: socket to splice from 807 * @ppos: position (not valid) 808 * @pipe: pipe to splice to 809 * @len: number of bytes to splice 810 * @flags: splice modifier flags 811 * 812 * Description: 813 * Will read pages from given socket and fill them into a pipe. 814 * 815 **/ 816 ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos, 817 struct pipe_inode_info *pipe, size_t len, 818 unsigned int flags) 819 { 820 struct sock *sk = sock->sk; 821 struct tcp_splice_state tss = { 822 .pipe = pipe, 823 .len = len, 824 .flags = flags, 825 }; 826 long timeo; 827 ssize_t spliced; 828 int ret; 829 830 sock_rps_record_flow(sk); 831 /* 832 * We can't seek on a socket input 833 */ 834 if (unlikely(*ppos)) 835 return -ESPIPE; 836 837 ret = spliced = 0; 838 839 lock_sock(sk); 840 841 timeo = sock_rcvtimeo(sk, sock->file->f_flags & O_NONBLOCK); 842 while (tss.len) { 843 ret = __tcp_splice_read(sk, &tss); 844 if (ret < 0) 845 break; 846 else if (!ret) { 847 if (spliced) 848 break; 849 if (sock_flag(sk, SOCK_DONE)) 850 break; 851 if (sk->sk_err) { 852 ret = sock_error(sk); 853 break; 854 } 855 if (sk->sk_shutdown & RCV_SHUTDOWN) 856 break; 857 if (sk->sk_state == TCP_CLOSE) { 858 /* 859 * This occurs when user tries to read 860 * from never connected socket. 861 */ 862 ret = -ENOTCONN; 863 break; 864 } 865 if (!timeo) { 866 ret = -EAGAIN; 867 break; 868 } 869 /* if __tcp_splice_read() got nothing while we have 870 * an skb in receive queue, we do not want to loop. 871 * This might happen with URG data. 872 */ 873 if (!skb_queue_empty(&sk->sk_receive_queue)) 874 break; 875 ret = sk_wait_data(sk, &timeo, NULL); 876 if (ret < 0) 877 break; 878 if (signal_pending(current)) { 879 ret = sock_intr_errno(timeo); 880 break; 881 } 882 continue; 883 } 884 tss.len -= ret; 885 spliced += ret; 886 887 if (!tss.len || !timeo) 888 break; 889 release_sock(sk); 890 lock_sock(sk); 891 892 if (sk->sk_err || sk->sk_state == TCP_CLOSE || 893 (sk->sk_shutdown & RCV_SHUTDOWN) || 894 signal_pending(current)) 895 break; 896 } 897 898 release_sock(sk); 899 900 if (spliced) 901 return spliced; 902 903 return ret; 904 } 905 EXPORT_IPV6_MOD(tcp_splice_read); 906 907 struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp, 908 bool force_schedule) 909 { 910 struct sk_buff *skb; 911 912 skb = alloc_skb_fclone(MAX_TCP_HEADER, gfp); 913 if (likely(skb)) { 914 bool mem_scheduled; 915 916 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 917 if (force_schedule) { 918 mem_scheduled = true; 919 sk_forced_mem_schedule(sk, skb->truesize); 920 } else { 921 mem_scheduled = sk_wmem_schedule(sk, skb->truesize); 922 } 923 if (likely(mem_scheduled)) { 924 skb_reserve(skb, MAX_TCP_HEADER); 925 skb->ip_summed = CHECKSUM_PARTIAL; 926 INIT_LIST_HEAD(&skb->tcp_tsorted_anchor); 927 return skb; 928 } 929 __kfree_skb(skb); 930 } else { 931 sk->sk_prot->enter_memory_pressure(sk); 932 sk_stream_moderate_sndbuf(sk); 933 } 934 return NULL; 935 } 936 937 static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now, 938 int large_allowed) 939 { 940 struct tcp_sock *tp = tcp_sk(sk); 941 u32 new_size_goal, size_goal; 942 943 if (!large_allowed) 944 return mss_now; 945 946 /* Note : tcp_tso_autosize() will eventually split this later */ 947 new_size_goal = tcp_bound_to_half_wnd(tp, sk->sk_gso_max_size); 948 949 /* We try hard to avoid divides here */ 950 size_goal = tp->gso_segs * mss_now; 951 if (unlikely(new_size_goal < size_goal || 952 new_size_goal >= size_goal + mss_now)) { 953 tp->gso_segs = min_t(u16, new_size_goal / mss_now, 954 sk->sk_gso_max_segs); 955 size_goal = tp->gso_segs * mss_now; 956 } 957 958 return max(size_goal, mss_now); 959 } 960 961 int tcp_send_mss(struct sock *sk, int *size_goal, int flags) 962 { 963 int mss_now; 964 965 mss_now = tcp_current_mss(sk); 966 *size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB)); 967 968 return mss_now; 969 } 970 971 /* In some cases, sendmsg() could have added an skb to the write queue, 972 * but failed adding payload on it. We need to remove it to consume less 973 * memory, but more importantly be able to generate EPOLLOUT for Edge Trigger 974 * epoll() users. Another reason is that tcp_write_xmit() does not like 975 * finding an empty skb in the write queue. 976 */ 977 void tcp_remove_empty_skb(struct sock *sk) 978 { 979 struct sk_buff *skb = tcp_write_queue_tail(sk); 980 981 if (skb && TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) { 982 tcp_unlink_write_queue(skb, sk); 983 if (tcp_write_queue_empty(sk)) 984 tcp_chrono_stop(sk, TCP_CHRONO_BUSY); 985 tcp_wmem_free_skb(sk, skb); 986 } 987 } 988 989 /* skb changing from pure zc to mixed, must charge zc */ 990 static int tcp_downgrade_zcopy_pure(struct sock *sk, struct sk_buff *skb) 991 { 992 if (unlikely(skb_zcopy_pure(skb))) { 993 u32 extra = skb->truesize - 994 SKB_TRUESIZE(skb_end_offset(skb)); 995 996 if (!sk_wmem_schedule(sk, extra)) 997 return -ENOMEM; 998 999 sk_mem_charge(sk, extra); 1000 skb_shinfo(skb)->flags &= ~SKBFL_PURE_ZEROCOPY; 1001 } 1002 return 0; 1003 } 1004 1005 1006 int tcp_wmem_schedule(struct sock *sk, int copy) 1007 { 1008 int left; 1009 1010 if (likely(sk_wmem_schedule(sk, copy))) 1011 return copy; 1012 1013 /* We could be in trouble if we have nothing queued. 1014 * Use whatever is left in sk->sk_forward_alloc and tcp_wmem[0] 1015 * to guarantee some progress. 1016 */ 1017 left = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_wmem[0]) - sk->sk_wmem_queued; 1018 if (left > 0) 1019 sk_forced_mem_schedule(sk, min(left, copy)); 1020 return min(copy, sk->sk_forward_alloc); 1021 } 1022 1023 void tcp_free_fastopen_req(struct tcp_sock *tp) 1024 { 1025 if (tp->fastopen_req) { 1026 kfree(tp->fastopen_req); 1027 tp->fastopen_req = NULL; 1028 } 1029 } 1030 1031 int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied, 1032 size_t size, struct ubuf_info *uarg) 1033 { 1034 struct tcp_sock *tp = tcp_sk(sk); 1035 struct inet_sock *inet = inet_sk(sk); 1036 struct sockaddr *uaddr = msg->msg_name; 1037 int err, flags; 1038 1039 if (!(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen) & 1040 TFO_CLIENT_ENABLE) || 1041 (uaddr && msg->msg_namelen >= sizeof(uaddr->sa_family) && 1042 uaddr->sa_family == AF_UNSPEC)) 1043 return -EOPNOTSUPP; 1044 if (tp->fastopen_req) 1045 return -EALREADY; /* Another Fast Open is in progress */ 1046 1047 tp->fastopen_req = kzalloc(sizeof(struct tcp_fastopen_request), 1048 sk->sk_allocation); 1049 if (unlikely(!tp->fastopen_req)) 1050 return -ENOBUFS; 1051 tp->fastopen_req->data = msg; 1052 tp->fastopen_req->size = size; 1053 tp->fastopen_req->uarg = uarg; 1054 1055 if (inet_test_bit(DEFER_CONNECT, sk)) { 1056 err = tcp_connect(sk); 1057 /* Same failure procedure as in tcp_v4/6_connect */ 1058 if (err) { 1059 tcp_set_state(sk, TCP_CLOSE); 1060 inet->inet_dport = 0; 1061 sk->sk_route_caps = 0; 1062 } 1063 } 1064 flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0; 1065 err = __inet_stream_connect(sk->sk_socket, uaddr, 1066 msg->msg_namelen, flags, 1); 1067 /* fastopen_req could already be freed in __inet_stream_connect 1068 * if the connection times out or gets rst 1069 */ 1070 if (tp->fastopen_req) { 1071 *copied = tp->fastopen_req->copied; 1072 tcp_free_fastopen_req(tp); 1073 inet_clear_bit(DEFER_CONNECT, sk); 1074 } 1075 return err; 1076 } 1077 1078 int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) 1079 { 1080 struct net_devmem_dmabuf_binding *binding = NULL; 1081 struct tcp_sock *tp = tcp_sk(sk); 1082 struct ubuf_info *uarg = NULL; 1083 struct sk_buff *skb; 1084 struct sockcm_cookie sockc; 1085 int flags, err, copied = 0; 1086 int mss_now = 0, size_goal, copied_syn = 0; 1087 int process_backlog = 0; 1088 int sockc_err = 0; 1089 int zc = 0; 1090 long timeo; 1091 1092 flags = msg->msg_flags; 1093 1094 sockc = (struct sockcm_cookie){ .tsflags = READ_ONCE(sk->sk_tsflags) }; 1095 if (msg->msg_controllen) { 1096 sockc_err = sock_cmsg_send(sk, msg, &sockc); 1097 /* Don't return error until MSG_FASTOPEN has been processed; 1098 * that may succeed even if the cmsg is invalid. 1099 */ 1100 } 1101 1102 if ((flags & MSG_ZEROCOPY) && size) { 1103 if (msg->msg_ubuf) { 1104 uarg = msg->msg_ubuf; 1105 if (sk->sk_route_caps & NETIF_F_SG) 1106 zc = MSG_ZEROCOPY; 1107 } else if (sock_flag(sk, SOCK_ZEROCOPY)) { 1108 skb = tcp_write_queue_tail(sk); 1109 uarg = msg_zerocopy_realloc(sk, size, skb_zcopy(skb), 1110 !sockc_err && sockc.dmabuf_id); 1111 if (!uarg) { 1112 err = -ENOBUFS; 1113 goto out_err; 1114 } 1115 if (sk->sk_route_caps & NETIF_F_SG) 1116 zc = MSG_ZEROCOPY; 1117 else 1118 uarg_to_msgzc(uarg)->zerocopy = 0; 1119 1120 if (!sockc_err && sockc.dmabuf_id) { 1121 binding = net_devmem_get_binding(sk, sockc.dmabuf_id); 1122 if (IS_ERR(binding)) { 1123 err = PTR_ERR(binding); 1124 binding = NULL; 1125 goto out_err; 1126 } 1127 } 1128 } 1129 } else if (unlikely(msg->msg_flags & MSG_SPLICE_PAGES) && size) { 1130 if (sk->sk_route_caps & NETIF_F_SG) 1131 zc = MSG_SPLICE_PAGES; 1132 } 1133 1134 if (!sockc_err && sockc.dmabuf_id && 1135 (!(flags & MSG_ZEROCOPY) || !sock_flag(sk, SOCK_ZEROCOPY))) { 1136 err = -EINVAL; 1137 goto out_err; 1138 } 1139 1140 if (unlikely(flags & MSG_FASTOPEN || 1141 inet_test_bit(DEFER_CONNECT, sk)) && 1142 !tp->repair) { 1143 err = tcp_sendmsg_fastopen(sk, msg, &copied_syn, size, uarg); 1144 if (err == -EINPROGRESS && copied_syn > 0) 1145 goto out; 1146 else if (err) 1147 goto out_err; 1148 } 1149 1150 timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); 1151 1152 tcp_rate_check_app_limited(sk); /* is sending application-limited? */ 1153 1154 /* Wait for a connection to finish. One exception is TCP Fast Open 1155 * (passive side) where data is allowed to be sent before a connection 1156 * is fully established. 1157 */ 1158 if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) && 1159 !tcp_passive_fastopen(sk)) { 1160 err = sk_stream_wait_connect(sk, &timeo); 1161 if (err != 0) 1162 goto do_error; 1163 } 1164 1165 if (unlikely(tp->repair)) { 1166 if (tp->repair_queue == TCP_RECV_QUEUE) { 1167 copied = tcp_send_rcvq(sk, msg, size); 1168 goto out_nopush; 1169 } 1170 1171 err = -EINVAL; 1172 if (tp->repair_queue == TCP_NO_QUEUE) 1173 goto out_err; 1174 1175 /* 'common' sending to sendq */ 1176 } 1177 1178 if (sockc_err) { 1179 err = sockc_err; 1180 goto out_err; 1181 } 1182 1183 /* This should be in poll */ 1184 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 1185 1186 /* Ok commence sending. */ 1187 copied = 0; 1188 1189 restart: 1190 mss_now = tcp_send_mss(sk, &size_goal, flags); 1191 1192 err = -EPIPE; 1193 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) 1194 goto do_error; 1195 1196 while (msg_data_left(msg)) { 1197 int copy = 0; 1198 1199 skb = tcp_write_queue_tail(sk); 1200 if (skb) 1201 copy = size_goal - skb->len; 1202 1203 trace_tcp_sendmsg_locked(sk, msg, skb, size_goal); 1204 1205 if (copy <= 0 || !tcp_skb_can_collapse_to(skb)) { 1206 bool first_skb; 1207 1208 new_segment: 1209 if (!sk_stream_memory_free(sk)) 1210 goto wait_for_space; 1211 1212 if (unlikely(process_backlog >= 16)) { 1213 process_backlog = 0; 1214 if (sk_flush_backlog(sk)) 1215 goto restart; 1216 } 1217 first_skb = tcp_rtx_and_write_queues_empty(sk); 1218 skb = tcp_stream_alloc_skb(sk, sk->sk_allocation, 1219 first_skb); 1220 if (!skb) 1221 goto wait_for_space; 1222 1223 process_backlog++; 1224 1225 #ifdef CONFIG_SKB_DECRYPTED 1226 skb->decrypted = !!(flags & MSG_SENDPAGE_DECRYPTED); 1227 #endif 1228 tcp_skb_entail(sk, skb); 1229 copy = size_goal; 1230 1231 /* All packets are restored as if they have 1232 * already been sent. skb_mstamp_ns isn't set to 1233 * avoid wrong rtt estimation. 1234 */ 1235 if (tp->repair) 1236 TCP_SKB_CB(skb)->sacked |= TCPCB_REPAIRED; 1237 } 1238 1239 /* Try to append data to the end of skb. */ 1240 if (copy > msg_data_left(msg)) 1241 copy = msg_data_left(msg); 1242 1243 if (zc == 0) { 1244 bool merge = true; 1245 int i = skb_shinfo(skb)->nr_frags; 1246 struct page_frag *pfrag = sk_page_frag(sk); 1247 1248 if (!sk_page_frag_refill(sk, pfrag)) 1249 goto wait_for_space; 1250 1251 if (!skb_can_coalesce(skb, i, pfrag->page, 1252 pfrag->offset)) { 1253 if (i >= READ_ONCE(net_hotdata.sysctl_max_skb_frags)) { 1254 tcp_mark_push(tp, skb); 1255 goto new_segment; 1256 } 1257 merge = false; 1258 } 1259 1260 copy = min_t(int, copy, pfrag->size - pfrag->offset); 1261 1262 if (unlikely(skb_zcopy_pure(skb) || skb_zcopy_managed(skb))) { 1263 if (tcp_downgrade_zcopy_pure(sk, skb)) 1264 goto wait_for_space; 1265 skb_zcopy_downgrade_managed(skb); 1266 } 1267 1268 copy = tcp_wmem_schedule(sk, copy); 1269 if (!copy) 1270 goto wait_for_space; 1271 1272 err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb, 1273 pfrag->page, 1274 pfrag->offset, 1275 copy); 1276 if (err) 1277 goto do_error; 1278 1279 /* Update the skb. */ 1280 if (merge) { 1281 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 1282 } else { 1283 skb_fill_page_desc(skb, i, pfrag->page, 1284 pfrag->offset, copy); 1285 page_ref_inc(pfrag->page); 1286 } 1287 pfrag->offset += copy; 1288 } else if (zc == MSG_ZEROCOPY) { 1289 /* First append to a fragless skb builds initial 1290 * pure zerocopy skb 1291 */ 1292 if (!skb->len) 1293 skb_shinfo(skb)->flags |= SKBFL_PURE_ZEROCOPY; 1294 1295 if (!skb_zcopy_pure(skb)) { 1296 copy = tcp_wmem_schedule(sk, copy); 1297 if (!copy) 1298 goto wait_for_space; 1299 } 1300 1301 err = skb_zerocopy_iter_stream(sk, skb, msg, copy, uarg, 1302 binding); 1303 if (err == -EMSGSIZE || err == -EEXIST) { 1304 tcp_mark_push(tp, skb); 1305 goto new_segment; 1306 } 1307 if (err < 0) 1308 goto do_error; 1309 copy = err; 1310 } else if (zc == MSG_SPLICE_PAGES) { 1311 /* Splice in data if we can; copy if we can't. */ 1312 if (tcp_downgrade_zcopy_pure(sk, skb)) 1313 goto wait_for_space; 1314 copy = tcp_wmem_schedule(sk, copy); 1315 if (!copy) 1316 goto wait_for_space; 1317 1318 err = skb_splice_from_iter(skb, &msg->msg_iter, copy); 1319 if (err < 0) { 1320 if (err == -EMSGSIZE) { 1321 tcp_mark_push(tp, skb); 1322 goto new_segment; 1323 } 1324 goto do_error; 1325 } 1326 copy = err; 1327 1328 if (!(flags & MSG_NO_SHARED_FRAGS)) 1329 skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG; 1330 1331 sk_wmem_queued_add(sk, copy); 1332 sk_mem_charge(sk, copy); 1333 } 1334 1335 if (!copied) 1336 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH; 1337 1338 WRITE_ONCE(tp->write_seq, tp->write_seq + copy); 1339 TCP_SKB_CB(skb)->end_seq += copy; 1340 tcp_skb_pcount_set(skb, 0); 1341 1342 copied += copy; 1343 if (!msg_data_left(msg)) { 1344 if (unlikely(flags & MSG_EOR)) 1345 TCP_SKB_CB(skb)->eor = 1; 1346 goto out; 1347 } 1348 1349 if (skb->len < size_goal || (flags & MSG_OOB) || unlikely(tp->repair)) 1350 continue; 1351 1352 if (forced_push(tp)) { 1353 tcp_mark_push(tp, skb); 1354 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH); 1355 } else if (skb == tcp_send_head(sk)) 1356 tcp_push_one(sk, mss_now); 1357 continue; 1358 1359 wait_for_space: 1360 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 1361 tcp_remove_empty_skb(sk); 1362 if (copied) 1363 tcp_push(sk, flags & ~MSG_MORE, mss_now, 1364 TCP_NAGLE_PUSH, size_goal); 1365 1366 err = sk_stream_wait_memory(sk, &timeo); 1367 if (err != 0) 1368 goto do_error; 1369 1370 mss_now = tcp_send_mss(sk, &size_goal, flags); 1371 } 1372 1373 out: 1374 if (copied) { 1375 tcp_tx_timestamp(sk, &sockc); 1376 tcp_push(sk, flags, mss_now, tp->nonagle, size_goal); 1377 } 1378 out_nopush: 1379 /* msg->msg_ubuf is pinned by the caller so we don't take extra refs */ 1380 if (uarg && !msg->msg_ubuf) 1381 net_zcopy_put(uarg); 1382 if (binding) 1383 net_devmem_dmabuf_binding_put(binding); 1384 return copied + copied_syn; 1385 1386 do_error: 1387 tcp_remove_empty_skb(sk); 1388 1389 if (copied + copied_syn) 1390 goto out; 1391 out_err: 1392 /* msg->msg_ubuf is pinned by the caller so we don't take extra refs */ 1393 if (uarg && !msg->msg_ubuf) 1394 net_zcopy_put_abort(uarg, true); 1395 err = sk_stream_error(sk, flags, err); 1396 /* make sure we wake any epoll edge trigger waiter */ 1397 if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) { 1398 sk->sk_write_space(sk); 1399 tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED); 1400 } 1401 if (binding) 1402 net_devmem_dmabuf_binding_put(binding); 1403 1404 return err; 1405 } 1406 EXPORT_SYMBOL_GPL(tcp_sendmsg_locked); 1407 1408 int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) 1409 { 1410 int ret; 1411 1412 lock_sock(sk); 1413 ret = tcp_sendmsg_locked(sk, msg, size); 1414 release_sock(sk); 1415 1416 return ret; 1417 } 1418 EXPORT_SYMBOL(tcp_sendmsg); 1419 1420 void tcp_splice_eof(struct socket *sock) 1421 { 1422 struct sock *sk = sock->sk; 1423 struct tcp_sock *tp = tcp_sk(sk); 1424 int mss_now, size_goal; 1425 1426 if (!tcp_write_queue_tail(sk)) 1427 return; 1428 1429 lock_sock(sk); 1430 mss_now = tcp_send_mss(sk, &size_goal, 0); 1431 tcp_push(sk, 0, mss_now, tp->nonagle, size_goal); 1432 release_sock(sk); 1433 } 1434 EXPORT_IPV6_MOD_GPL(tcp_splice_eof); 1435 1436 /* 1437 * Handle reading urgent data. BSD has very simple semantics for 1438 * this, no blocking and very strange errors 8) 1439 */ 1440 1441 static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags) 1442 { 1443 struct tcp_sock *tp = tcp_sk(sk); 1444 1445 /* No URG data to read. */ 1446 if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data || 1447 tp->urg_data == TCP_URG_READ) 1448 return -EINVAL; /* Yes this is right ! */ 1449 1450 if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE)) 1451 return -ENOTCONN; 1452 1453 if (tp->urg_data & TCP_URG_VALID) { 1454 int err = 0; 1455 char c = tp->urg_data; 1456 1457 if (!(flags & MSG_PEEK)) 1458 WRITE_ONCE(tp->urg_data, TCP_URG_READ); 1459 1460 /* Read urgent data. */ 1461 msg->msg_flags |= MSG_OOB; 1462 1463 if (len > 0) { 1464 if (!(flags & MSG_TRUNC)) 1465 err = memcpy_to_msg(msg, &c, 1); 1466 len = 1; 1467 } else 1468 msg->msg_flags |= MSG_TRUNC; 1469 1470 return err ? -EFAULT : len; 1471 } 1472 1473 if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN)) 1474 return 0; 1475 1476 /* Fixed the recv(..., MSG_OOB) behaviour. BSD docs and 1477 * the available implementations agree in this case: 1478 * this call should never block, independent of the 1479 * blocking state of the socket. 1480 * Mike <pall@rz.uni-karlsruhe.de> 1481 */ 1482 return -EAGAIN; 1483 } 1484 1485 static int tcp_peek_sndq(struct sock *sk, struct msghdr *msg, int len) 1486 { 1487 struct sk_buff *skb; 1488 int copied = 0, err = 0; 1489 1490 skb_rbtree_walk(skb, &sk->tcp_rtx_queue) { 1491 err = skb_copy_datagram_msg(skb, 0, msg, skb->len); 1492 if (err) 1493 return err; 1494 copied += skb->len; 1495 } 1496 1497 skb_queue_walk(&sk->sk_write_queue, skb) { 1498 err = skb_copy_datagram_msg(skb, 0, msg, skb->len); 1499 if (err) 1500 break; 1501 1502 copied += skb->len; 1503 } 1504 1505 return err ?: copied; 1506 } 1507 1508 /* Clean up the receive buffer for full frames taken by the user, 1509 * then send an ACK if necessary. COPIED is the number of bytes 1510 * tcp_recvmsg has given to the user so far, it speeds up the 1511 * calculation of whether or not we must ACK for the sake of 1512 * a window update. 1513 */ 1514 void __tcp_cleanup_rbuf(struct sock *sk, int copied) 1515 { 1516 struct tcp_sock *tp = tcp_sk(sk); 1517 bool time_to_ack = false; 1518 1519 if (inet_csk_ack_scheduled(sk)) { 1520 const struct inet_connection_sock *icsk = inet_csk(sk); 1521 1522 if (/* Once-per-two-segments ACK was not sent by tcp_input.c */ 1523 tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss || 1524 /* 1525 * If this read emptied read buffer, we send ACK, if 1526 * connection is not bidirectional, user drained 1527 * receive buffer and there was a small segment 1528 * in queue. 1529 */ 1530 (copied > 0 && 1531 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) || 1532 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) && 1533 !inet_csk_in_pingpong_mode(sk))) && 1534 !atomic_read(&sk->sk_rmem_alloc))) 1535 time_to_ack = true; 1536 } 1537 1538 /* We send an ACK if we can now advertise a non-zero window 1539 * which has been raised "significantly". 1540 * 1541 * Even if window raised up to infinity, do not send window open ACK 1542 * in states, where we will not receive more. It is useless. 1543 */ 1544 if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) { 1545 __u32 rcv_window_now = tcp_receive_window(tp); 1546 1547 /* Optimize, __tcp_select_window() is not cheap. */ 1548 if (2*rcv_window_now <= tp->window_clamp) { 1549 __u32 new_window = __tcp_select_window(sk); 1550 1551 /* Send ACK now, if this read freed lots of space 1552 * in our buffer. Certainly, new_window is new window. 1553 * We can advertise it now, if it is not less than current one. 1554 * "Lots" means "at least twice" here. 1555 */ 1556 if (new_window && new_window >= 2 * rcv_window_now) 1557 time_to_ack = true; 1558 } 1559 } 1560 if (time_to_ack) 1561 tcp_send_ack(sk); 1562 } 1563 1564 void tcp_cleanup_rbuf(struct sock *sk, int copied) 1565 { 1566 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue); 1567 struct tcp_sock *tp = tcp_sk(sk); 1568 1569 WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq), 1570 "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n", 1571 tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt); 1572 __tcp_cleanup_rbuf(sk, copied); 1573 } 1574 1575 static void tcp_eat_recv_skb(struct sock *sk, struct sk_buff *skb) 1576 { 1577 __skb_unlink(skb, &sk->sk_receive_queue); 1578 if (likely(skb->destructor == sock_rfree)) { 1579 sock_rfree(skb); 1580 skb->destructor = NULL; 1581 skb->sk = NULL; 1582 return skb_attempt_defer_free(skb); 1583 } 1584 __kfree_skb(skb); 1585 } 1586 1587 struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off) 1588 { 1589 struct sk_buff *skb; 1590 u32 offset; 1591 1592 while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) { 1593 offset = seq - TCP_SKB_CB(skb)->seq; 1594 if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) { 1595 pr_err_once("%s: found a SYN, please report !\n", __func__); 1596 offset--; 1597 } 1598 if (offset < skb->len || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) { 1599 *off = offset; 1600 return skb; 1601 } 1602 /* This looks weird, but this can happen if TCP collapsing 1603 * splitted a fat GRO packet, while we released socket lock 1604 * in skb_splice_bits() 1605 */ 1606 tcp_eat_recv_skb(sk, skb); 1607 } 1608 return NULL; 1609 } 1610 EXPORT_SYMBOL(tcp_recv_skb); 1611 1612 /* 1613 * This routine provides an alternative to tcp_recvmsg() for routines 1614 * that would like to handle copying from skbuffs directly in 'sendfile' 1615 * fashion. 1616 * Note: 1617 * - It is assumed that the socket was locked by the caller. 1618 * - The routine does not block. 1619 * - At present, there is no support for reading OOB data 1620 * or for 'peeking' the socket using this routine 1621 * (although both would be easy to implement). 1622 */ 1623 static int __tcp_read_sock(struct sock *sk, read_descriptor_t *desc, 1624 sk_read_actor_t recv_actor, bool noack, 1625 u32 *copied_seq) 1626 { 1627 struct sk_buff *skb; 1628 struct tcp_sock *tp = tcp_sk(sk); 1629 u32 seq = *copied_seq; 1630 u32 offset; 1631 int copied = 0; 1632 1633 if (sk->sk_state == TCP_LISTEN) 1634 return -ENOTCONN; 1635 while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) { 1636 if (offset < skb->len) { 1637 int used; 1638 size_t len; 1639 1640 len = skb->len - offset; 1641 /* Stop reading if we hit a patch of urgent data */ 1642 if (unlikely(tp->urg_data)) { 1643 u32 urg_offset = tp->urg_seq - seq; 1644 if (urg_offset < len) 1645 len = urg_offset; 1646 if (!len) 1647 break; 1648 } 1649 used = recv_actor(desc, skb, offset, len); 1650 if (used <= 0) { 1651 if (!copied) 1652 copied = used; 1653 break; 1654 } 1655 if (WARN_ON_ONCE(used > len)) 1656 used = len; 1657 seq += used; 1658 copied += used; 1659 offset += used; 1660 1661 /* If recv_actor drops the lock (e.g. TCP splice 1662 * receive) the skb pointer might be invalid when 1663 * getting here: tcp_collapse might have deleted it 1664 * while aggregating skbs from the socket queue. 1665 */ 1666 skb = tcp_recv_skb(sk, seq - 1, &offset); 1667 if (!skb) 1668 break; 1669 /* TCP coalescing might have appended data to the skb. 1670 * Try to splice more frags 1671 */ 1672 if (offset + 1 != skb->len) 1673 continue; 1674 } 1675 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) { 1676 tcp_eat_recv_skb(sk, skb); 1677 ++seq; 1678 break; 1679 } 1680 tcp_eat_recv_skb(sk, skb); 1681 if (!desc->count) 1682 break; 1683 WRITE_ONCE(*copied_seq, seq); 1684 } 1685 WRITE_ONCE(*copied_seq, seq); 1686 1687 if (noack) 1688 goto out; 1689 1690 tcp_rcv_space_adjust(sk); 1691 1692 /* Clean up data we have read: This will do ACK frames. */ 1693 if (copied > 0) { 1694 tcp_recv_skb(sk, seq, &offset); 1695 tcp_cleanup_rbuf(sk, copied); 1696 } 1697 out: 1698 return copied; 1699 } 1700 1701 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, 1702 sk_read_actor_t recv_actor) 1703 { 1704 return __tcp_read_sock(sk, desc, recv_actor, false, 1705 &tcp_sk(sk)->copied_seq); 1706 } 1707 EXPORT_SYMBOL(tcp_read_sock); 1708 1709 int tcp_read_sock_noack(struct sock *sk, read_descriptor_t *desc, 1710 sk_read_actor_t recv_actor, bool noack, 1711 u32 *copied_seq) 1712 { 1713 return __tcp_read_sock(sk, desc, recv_actor, noack, copied_seq); 1714 } 1715 1716 int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor) 1717 { 1718 struct sk_buff *skb; 1719 int copied = 0; 1720 1721 if (sk->sk_state == TCP_LISTEN) 1722 return -ENOTCONN; 1723 1724 while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) { 1725 u8 tcp_flags; 1726 int used; 1727 1728 __skb_unlink(skb, &sk->sk_receive_queue); 1729 WARN_ON_ONCE(!skb_set_owner_sk_safe(skb, sk)); 1730 tcp_flags = TCP_SKB_CB(skb)->tcp_flags; 1731 used = recv_actor(sk, skb); 1732 if (used < 0) { 1733 if (!copied) 1734 copied = used; 1735 break; 1736 } 1737 copied += used; 1738 1739 if (tcp_flags & TCPHDR_FIN) 1740 break; 1741 } 1742 return copied; 1743 } 1744 EXPORT_IPV6_MOD(tcp_read_skb); 1745 1746 void tcp_read_done(struct sock *sk, size_t len) 1747 { 1748 struct tcp_sock *tp = tcp_sk(sk); 1749 u32 seq = tp->copied_seq; 1750 struct sk_buff *skb; 1751 size_t left; 1752 u32 offset; 1753 1754 if (sk->sk_state == TCP_LISTEN) 1755 return; 1756 1757 left = len; 1758 while (left && (skb = tcp_recv_skb(sk, seq, &offset)) != NULL) { 1759 int used; 1760 1761 used = min_t(size_t, skb->len - offset, left); 1762 seq += used; 1763 left -= used; 1764 1765 if (skb->len > offset + used) 1766 break; 1767 1768 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) { 1769 tcp_eat_recv_skb(sk, skb); 1770 ++seq; 1771 break; 1772 } 1773 tcp_eat_recv_skb(sk, skb); 1774 } 1775 WRITE_ONCE(tp->copied_seq, seq); 1776 1777 tcp_rcv_space_adjust(sk); 1778 1779 /* Clean up data we have read: This will do ACK frames. */ 1780 if (left != len) 1781 tcp_cleanup_rbuf(sk, len - left); 1782 } 1783 EXPORT_SYMBOL(tcp_read_done); 1784 1785 int tcp_peek_len(struct socket *sock) 1786 { 1787 return tcp_inq(sock->sk); 1788 } 1789 EXPORT_IPV6_MOD(tcp_peek_len); 1790 1791 /* Make sure sk_rcvbuf is big enough to satisfy SO_RCVLOWAT hint */ 1792 int tcp_set_rcvlowat(struct sock *sk, int val) 1793 { 1794 int space, cap; 1795 1796 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK) 1797 cap = sk->sk_rcvbuf >> 1; 1798 else 1799 cap = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2]) >> 1; 1800 val = min(val, cap); 1801 WRITE_ONCE(sk->sk_rcvlowat, val ? : 1); 1802 1803 /* Check if we need to signal EPOLLIN right now */ 1804 tcp_data_ready(sk); 1805 1806 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK) 1807 return 0; 1808 1809 space = tcp_space_from_win(sk, val); 1810 if (space > sk->sk_rcvbuf) { 1811 WRITE_ONCE(sk->sk_rcvbuf, space); 1812 WRITE_ONCE(tcp_sk(sk)->window_clamp, val); 1813 } 1814 return 0; 1815 } 1816 EXPORT_IPV6_MOD(tcp_set_rcvlowat); 1817 1818 void tcp_update_recv_tstamps(struct sk_buff *skb, 1819 struct scm_timestamping_internal *tss) 1820 { 1821 if (skb->tstamp) 1822 tss->ts[0] = ktime_to_timespec64(skb->tstamp); 1823 else 1824 tss->ts[0] = (struct timespec64) {0}; 1825 1826 if (skb_hwtstamps(skb)->hwtstamp) 1827 tss->ts[2] = ktime_to_timespec64(skb_hwtstamps(skb)->hwtstamp); 1828 else 1829 tss->ts[2] = (struct timespec64) {0}; 1830 } 1831 1832 #ifdef CONFIG_MMU 1833 static const struct vm_operations_struct tcp_vm_ops = { 1834 }; 1835 1836 int tcp_mmap(struct file *file, struct socket *sock, 1837 struct vm_area_struct *vma) 1838 { 1839 if (vma->vm_flags & (VM_WRITE | VM_EXEC)) 1840 return -EPERM; 1841 vm_flags_clear(vma, VM_MAYWRITE | VM_MAYEXEC); 1842 1843 /* Instruct vm_insert_page() to not mmap_read_lock(mm) */ 1844 vm_flags_set(vma, VM_MIXEDMAP); 1845 1846 vma->vm_ops = &tcp_vm_ops; 1847 return 0; 1848 } 1849 EXPORT_IPV6_MOD(tcp_mmap); 1850 1851 static skb_frag_t *skb_advance_to_frag(struct sk_buff *skb, u32 offset_skb, 1852 u32 *offset_frag) 1853 { 1854 skb_frag_t *frag; 1855 1856 if (unlikely(offset_skb >= skb->len)) 1857 return NULL; 1858 1859 offset_skb -= skb_headlen(skb); 1860 if ((int)offset_skb < 0 || skb_has_frag_list(skb)) 1861 return NULL; 1862 1863 frag = skb_shinfo(skb)->frags; 1864 while (offset_skb) { 1865 if (skb_frag_size(frag) > offset_skb) { 1866 *offset_frag = offset_skb; 1867 return frag; 1868 } 1869 offset_skb -= skb_frag_size(frag); 1870 ++frag; 1871 } 1872 *offset_frag = 0; 1873 return frag; 1874 } 1875 1876 static bool can_map_frag(const skb_frag_t *frag) 1877 { 1878 struct page *page; 1879 1880 if (skb_frag_size(frag) != PAGE_SIZE || skb_frag_off(frag)) 1881 return false; 1882 1883 page = skb_frag_page(frag); 1884 1885 if (PageCompound(page) || page->mapping) 1886 return false; 1887 1888 return true; 1889 } 1890 1891 static int find_next_mappable_frag(const skb_frag_t *frag, 1892 int remaining_in_skb) 1893 { 1894 int offset = 0; 1895 1896 if (likely(can_map_frag(frag))) 1897 return 0; 1898 1899 while (offset < remaining_in_skb && !can_map_frag(frag)) { 1900 offset += skb_frag_size(frag); 1901 ++frag; 1902 } 1903 return offset; 1904 } 1905 1906 static void tcp_zerocopy_set_hint_for_skb(struct sock *sk, 1907 struct tcp_zerocopy_receive *zc, 1908 struct sk_buff *skb, u32 offset) 1909 { 1910 u32 frag_offset, partial_frag_remainder = 0; 1911 int mappable_offset; 1912 skb_frag_t *frag; 1913 1914 /* worst case: skip to next skb. try to improve on this case below */ 1915 zc->recv_skip_hint = skb->len - offset; 1916 1917 /* Find the frag containing this offset (and how far into that frag) */ 1918 frag = skb_advance_to_frag(skb, offset, &frag_offset); 1919 if (!frag) 1920 return; 1921 1922 if (frag_offset) { 1923 struct skb_shared_info *info = skb_shinfo(skb); 1924 1925 /* We read part of the last frag, must recvmsg() rest of skb. */ 1926 if (frag == &info->frags[info->nr_frags - 1]) 1927 return; 1928 1929 /* Else, we must at least read the remainder in this frag. */ 1930 partial_frag_remainder = skb_frag_size(frag) - frag_offset; 1931 zc->recv_skip_hint -= partial_frag_remainder; 1932 ++frag; 1933 } 1934 1935 /* partial_frag_remainder: If part way through a frag, must read rest. 1936 * mappable_offset: Bytes till next mappable frag, *not* counting bytes 1937 * in partial_frag_remainder. 1938 */ 1939 mappable_offset = find_next_mappable_frag(frag, zc->recv_skip_hint); 1940 zc->recv_skip_hint = mappable_offset + partial_frag_remainder; 1941 } 1942 1943 static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len, 1944 int flags, struct scm_timestamping_internal *tss, 1945 int *cmsg_flags); 1946 static int receive_fallback_to_copy(struct sock *sk, 1947 struct tcp_zerocopy_receive *zc, int inq, 1948 struct scm_timestamping_internal *tss) 1949 { 1950 unsigned long copy_address = (unsigned long)zc->copybuf_address; 1951 struct msghdr msg = {}; 1952 int err; 1953 1954 zc->length = 0; 1955 zc->recv_skip_hint = 0; 1956 1957 if (copy_address != zc->copybuf_address) 1958 return -EINVAL; 1959 1960 err = import_ubuf(ITER_DEST, (void __user *)copy_address, inq, 1961 &msg.msg_iter); 1962 if (err) 1963 return err; 1964 1965 err = tcp_recvmsg_locked(sk, &msg, inq, MSG_DONTWAIT, 1966 tss, &zc->msg_flags); 1967 if (err < 0) 1968 return err; 1969 1970 zc->copybuf_len = err; 1971 if (likely(zc->copybuf_len)) { 1972 struct sk_buff *skb; 1973 u32 offset; 1974 1975 skb = tcp_recv_skb(sk, tcp_sk(sk)->copied_seq, &offset); 1976 if (skb) 1977 tcp_zerocopy_set_hint_for_skb(sk, zc, skb, offset); 1978 } 1979 return 0; 1980 } 1981 1982 static int tcp_copy_straggler_data(struct tcp_zerocopy_receive *zc, 1983 struct sk_buff *skb, u32 copylen, 1984 u32 *offset, u32 *seq) 1985 { 1986 unsigned long copy_address = (unsigned long)zc->copybuf_address; 1987 struct msghdr msg = {}; 1988 int err; 1989 1990 if (copy_address != zc->copybuf_address) 1991 return -EINVAL; 1992 1993 err = import_ubuf(ITER_DEST, (void __user *)copy_address, copylen, 1994 &msg.msg_iter); 1995 if (err) 1996 return err; 1997 err = skb_copy_datagram_msg(skb, *offset, &msg, copylen); 1998 if (err) 1999 return err; 2000 zc->recv_skip_hint -= copylen; 2001 *offset += copylen; 2002 *seq += copylen; 2003 return (__s32)copylen; 2004 } 2005 2006 static int tcp_zc_handle_leftover(struct tcp_zerocopy_receive *zc, 2007 struct sock *sk, 2008 struct sk_buff *skb, 2009 u32 *seq, 2010 s32 copybuf_len, 2011 struct scm_timestamping_internal *tss) 2012 { 2013 u32 offset, copylen = min_t(u32, copybuf_len, zc->recv_skip_hint); 2014 2015 if (!copylen) 2016 return 0; 2017 /* skb is null if inq < PAGE_SIZE. */ 2018 if (skb) { 2019 offset = *seq - TCP_SKB_CB(skb)->seq; 2020 } else { 2021 skb = tcp_recv_skb(sk, *seq, &offset); 2022 if (TCP_SKB_CB(skb)->has_rxtstamp) { 2023 tcp_update_recv_tstamps(skb, tss); 2024 zc->msg_flags |= TCP_CMSG_TS; 2025 } 2026 } 2027 2028 zc->copybuf_len = tcp_copy_straggler_data(zc, skb, copylen, &offset, 2029 seq); 2030 return zc->copybuf_len < 0 ? 0 : copylen; 2031 } 2032 2033 static int tcp_zerocopy_vm_insert_batch_error(struct vm_area_struct *vma, 2034 struct page **pending_pages, 2035 unsigned long pages_remaining, 2036 unsigned long *address, 2037 u32 *length, 2038 u32 *seq, 2039 struct tcp_zerocopy_receive *zc, 2040 u32 total_bytes_to_map, 2041 int err) 2042 { 2043 /* At least one page did not map. Try zapping if we skipped earlier. */ 2044 if (err == -EBUSY && 2045 zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT) { 2046 u32 maybe_zap_len; 2047 2048 maybe_zap_len = total_bytes_to_map - /* All bytes to map */ 2049 *length + /* Mapped or pending */ 2050 (pages_remaining * PAGE_SIZE); /* Failed map. */ 2051 zap_page_range_single(vma, *address, maybe_zap_len, NULL); 2052 err = 0; 2053 } 2054 2055 if (!err) { 2056 unsigned long leftover_pages = pages_remaining; 2057 int bytes_mapped; 2058 2059 /* We called zap_page_range_single, try to reinsert. */ 2060 err = vm_insert_pages(vma, *address, 2061 pending_pages, 2062 &pages_remaining); 2063 bytes_mapped = PAGE_SIZE * (leftover_pages - pages_remaining); 2064 *seq += bytes_mapped; 2065 *address += bytes_mapped; 2066 } 2067 if (err) { 2068 /* Either we were unable to zap, OR we zapped, retried an 2069 * insert, and still had an issue. Either ways, pages_remaining 2070 * is the number of pages we were unable to map, and we unroll 2071 * some state we speculatively touched before. 2072 */ 2073 const int bytes_not_mapped = PAGE_SIZE * pages_remaining; 2074 2075 *length -= bytes_not_mapped; 2076 zc->recv_skip_hint += bytes_not_mapped; 2077 } 2078 return err; 2079 } 2080 2081 static int tcp_zerocopy_vm_insert_batch(struct vm_area_struct *vma, 2082 struct page **pages, 2083 unsigned int pages_to_map, 2084 unsigned long *address, 2085 u32 *length, 2086 u32 *seq, 2087 struct tcp_zerocopy_receive *zc, 2088 u32 total_bytes_to_map) 2089 { 2090 unsigned long pages_remaining = pages_to_map; 2091 unsigned int pages_mapped; 2092 unsigned int bytes_mapped; 2093 int err; 2094 2095 err = vm_insert_pages(vma, *address, pages, &pages_remaining); 2096 pages_mapped = pages_to_map - (unsigned int)pages_remaining; 2097 bytes_mapped = PAGE_SIZE * pages_mapped; 2098 /* Even if vm_insert_pages fails, it may have partially succeeded in 2099 * mapping (some but not all of the pages). 2100 */ 2101 *seq += bytes_mapped; 2102 *address += bytes_mapped; 2103 2104 if (likely(!err)) 2105 return 0; 2106 2107 /* Error: maybe zap and retry + rollback state for failed inserts. */ 2108 return tcp_zerocopy_vm_insert_batch_error(vma, pages + pages_mapped, 2109 pages_remaining, address, length, seq, zc, total_bytes_to_map, 2110 err); 2111 } 2112 2113 #define TCP_VALID_ZC_MSG_FLAGS (TCP_CMSG_TS) 2114 static void tcp_zc_finalize_rx_tstamp(struct sock *sk, 2115 struct tcp_zerocopy_receive *zc, 2116 struct scm_timestamping_internal *tss) 2117 { 2118 unsigned long msg_control_addr; 2119 struct msghdr cmsg_dummy; 2120 2121 msg_control_addr = (unsigned long)zc->msg_control; 2122 cmsg_dummy.msg_control_user = (void __user *)msg_control_addr; 2123 cmsg_dummy.msg_controllen = 2124 (__kernel_size_t)zc->msg_controllen; 2125 cmsg_dummy.msg_flags = in_compat_syscall() 2126 ? MSG_CMSG_COMPAT : 0; 2127 cmsg_dummy.msg_control_is_user = true; 2128 zc->msg_flags = 0; 2129 if (zc->msg_control == msg_control_addr && 2130 zc->msg_controllen == cmsg_dummy.msg_controllen) { 2131 tcp_recv_timestamp(&cmsg_dummy, sk, tss); 2132 zc->msg_control = (__u64) 2133 ((uintptr_t)cmsg_dummy.msg_control_user); 2134 zc->msg_controllen = 2135 (__u64)cmsg_dummy.msg_controllen; 2136 zc->msg_flags = (__u32)cmsg_dummy.msg_flags; 2137 } 2138 } 2139 2140 static struct vm_area_struct *find_tcp_vma(struct mm_struct *mm, 2141 unsigned long address, 2142 bool *mmap_locked) 2143 { 2144 struct vm_area_struct *vma = lock_vma_under_rcu(mm, address); 2145 2146 if (vma) { 2147 if (vma->vm_ops != &tcp_vm_ops) { 2148 vma_end_read(vma); 2149 return NULL; 2150 } 2151 *mmap_locked = false; 2152 return vma; 2153 } 2154 2155 mmap_read_lock(mm); 2156 vma = vma_lookup(mm, address); 2157 if (!vma || vma->vm_ops != &tcp_vm_ops) { 2158 mmap_read_unlock(mm); 2159 return NULL; 2160 } 2161 *mmap_locked = true; 2162 return vma; 2163 } 2164 2165 #define TCP_ZEROCOPY_PAGE_BATCH_SIZE 32 2166 static int tcp_zerocopy_receive(struct sock *sk, 2167 struct tcp_zerocopy_receive *zc, 2168 struct scm_timestamping_internal *tss) 2169 { 2170 u32 length = 0, offset, vma_len, avail_len, copylen = 0; 2171 unsigned long address = (unsigned long)zc->address; 2172 struct page *pages[TCP_ZEROCOPY_PAGE_BATCH_SIZE]; 2173 s32 copybuf_len = zc->copybuf_len; 2174 struct tcp_sock *tp = tcp_sk(sk); 2175 const skb_frag_t *frags = NULL; 2176 unsigned int pages_to_map = 0; 2177 struct vm_area_struct *vma; 2178 struct sk_buff *skb = NULL; 2179 u32 seq = tp->copied_seq; 2180 u32 total_bytes_to_map; 2181 int inq = tcp_inq(sk); 2182 bool mmap_locked; 2183 int ret; 2184 2185 zc->copybuf_len = 0; 2186 zc->msg_flags = 0; 2187 2188 if (address & (PAGE_SIZE - 1) || address != zc->address) 2189 return -EINVAL; 2190 2191 if (sk->sk_state == TCP_LISTEN) 2192 return -ENOTCONN; 2193 2194 sock_rps_record_flow(sk); 2195 2196 if (inq && inq <= copybuf_len) 2197 return receive_fallback_to_copy(sk, zc, inq, tss); 2198 2199 if (inq < PAGE_SIZE) { 2200 zc->length = 0; 2201 zc->recv_skip_hint = inq; 2202 if (!inq && sock_flag(sk, SOCK_DONE)) 2203 return -EIO; 2204 return 0; 2205 } 2206 2207 vma = find_tcp_vma(current->mm, address, &mmap_locked); 2208 if (!vma) 2209 return -EINVAL; 2210 2211 vma_len = min_t(unsigned long, zc->length, vma->vm_end - address); 2212 avail_len = min_t(u32, vma_len, inq); 2213 total_bytes_to_map = avail_len & ~(PAGE_SIZE - 1); 2214 if (total_bytes_to_map) { 2215 if (!(zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT)) 2216 zap_page_range_single(vma, address, total_bytes_to_map, 2217 NULL); 2218 zc->length = total_bytes_to_map; 2219 zc->recv_skip_hint = 0; 2220 } else { 2221 zc->length = avail_len; 2222 zc->recv_skip_hint = avail_len; 2223 } 2224 ret = 0; 2225 while (length + PAGE_SIZE <= zc->length) { 2226 int mappable_offset; 2227 struct page *page; 2228 2229 if (zc->recv_skip_hint < PAGE_SIZE) { 2230 u32 offset_frag; 2231 2232 if (skb) { 2233 if (zc->recv_skip_hint > 0) 2234 break; 2235 skb = skb->next; 2236 offset = seq - TCP_SKB_CB(skb)->seq; 2237 } else { 2238 skb = tcp_recv_skb(sk, seq, &offset); 2239 } 2240 2241 if (!skb_frags_readable(skb)) 2242 break; 2243 2244 if (TCP_SKB_CB(skb)->has_rxtstamp) { 2245 tcp_update_recv_tstamps(skb, tss); 2246 zc->msg_flags |= TCP_CMSG_TS; 2247 } 2248 zc->recv_skip_hint = skb->len - offset; 2249 frags = skb_advance_to_frag(skb, offset, &offset_frag); 2250 if (!frags || offset_frag) 2251 break; 2252 } 2253 2254 mappable_offset = find_next_mappable_frag(frags, 2255 zc->recv_skip_hint); 2256 if (mappable_offset) { 2257 zc->recv_skip_hint = mappable_offset; 2258 break; 2259 } 2260 page = skb_frag_page(frags); 2261 if (WARN_ON_ONCE(!page)) 2262 break; 2263 2264 prefetchw(page); 2265 pages[pages_to_map++] = page; 2266 length += PAGE_SIZE; 2267 zc->recv_skip_hint -= PAGE_SIZE; 2268 frags++; 2269 if (pages_to_map == TCP_ZEROCOPY_PAGE_BATCH_SIZE || 2270 zc->recv_skip_hint < PAGE_SIZE) { 2271 /* Either full batch, or we're about to go to next skb 2272 * (and we cannot unroll failed ops across skbs). 2273 */ 2274 ret = tcp_zerocopy_vm_insert_batch(vma, pages, 2275 pages_to_map, 2276 &address, &length, 2277 &seq, zc, 2278 total_bytes_to_map); 2279 if (ret) 2280 goto out; 2281 pages_to_map = 0; 2282 } 2283 } 2284 if (pages_to_map) { 2285 ret = tcp_zerocopy_vm_insert_batch(vma, pages, pages_to_map, 2286 &address, &length, &seq, 2287 zc, total_bytes_to_map); 2288 } 2289 out: 2290 if (mmap_locked) 2291 mmap_read_unlock(current->mm); 2292 else 2293 vma_end_read(vma); 2294 /* Try to copy straggler data. */ 2295 if (!ret) 2296 copylen = tcp_zc_handle_leftover(zc, sk, skb, &seq, copybuf_len, tss); 2297 2298 if (length + copylen) { 2299 WRITE_ONCE(tp->copied_seq, seq); 2300 tcp_rcv_space_adjust(sk); 2301 2302 /* Clean up data we have read: This will do ACK frames. */ 2303 tcp_recv_skb(sk, seq, &offset); 2304 tcp_cleanup_rbuf(sk, length + copylen); 2305 ret = 0; 2306 if (length == zc->length) 2307 zc->recv_skip_hint = 0; 2308 } else { 2309 if (!zc->recv_skip_hint && sock_flag(sk, SOCK_DONE)) 2310 ret = -EIO; 2311 } 2312 zc->length = length; 2313 return ret; 2314 } 2315 #endif 2316 2317 /* Similar to __sock_recv_timestamp, but does not require an skb */ 2318 void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk, 2319 struct scm_timestamping_internal *tss) 2320 { 2321 int new_tstamp = sock_flag(sk, SOCK_TSTAMP_NEW); 2322 u32 tsflags = READ_ONCE(sk->sk_tsflags); 2323 bool has_timestamping = false; 2324 2325 if (tss->ts[0].tv_sec || tss->ts[0].tv_nsec) { 2326 if (sock_flag(sk, SOCK_RCVTSTAMP)) { 2327 if (sock_flag(sk, SOCK_RCVTSTAMPNS)) { 2328 if (new_tstamp) { 2329 struct __kernel_timespec kts = { 2330 .tv_sec = tss->ts[0].tv_sec, 2331 .tv_nsec = tss->ts[0].tv_nsec, 2332 }; 2333 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_NEW, 2334 sizeof(kts), &kts); 2335 } else { 2336 struct __kernel_old_timespec ts_old = { 2337 .tv_sec = tss->ts[0].tv_sec, 2338 .tv_nsec = tss->ts[0].tv_nsec, 2339 }; 2340 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_OLD, 2341 sizeof(ts_old), &ts_old); 2342 } 2343 } else { 2344 if (new_tstamp) { 2345 struct __kernel_sock_timeval stv = { 2346 .tv_sec = tss->ts[0].tv_sec, 2347 .tv_usec = tss->ts[0].tv_nsec / 1000, 2348 }; 2349 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_NEW, 2350 sizeof(stv), &stv); 2351 } else { 2352 struct __kernel_old_timeval tv = { 2353 .tv_sec = tss->ts[0].tv_sec, 2354 .tv_usec = tss->ts[0].tv_nsec / 1000, 2355 }; 2356 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_OLD, 2357 sizeof(tv), &tv); 2358 } 2359 } 2360 } 2361 2362 if (tsflags & SOF_TIMESTAMPING_SOFTWARE && 2363 (tsflags & SOF_TIMESTAMPING_RX_SOFTWARE || 2364 !(tsflags & SOF_TIMESTAMPING_OPT_RX_FILTER))) 2365 has_timestamping = true; 2366 else 2367 tss->ts[0] = (struct timespec64) {0}; 2368 } 2369 2370 if (tss->ts[2].tv_sec || tss->ts[2].tv_nsec) { 2371 if (tsflags & SOF_TIMESTAMPING_RAW_HARDWARE && 2372 (tsflags & SOF_TIMESTAMPING_RX_HARDWARE || 2373 !(tsflags & SOF_TIMESTAMPING_OPT_RX_FILTER))) 2374 has_timestamping = true; 2375 else 2376 tss->ts[2] = (struct timespec64) {0}; 2377 } 2378 2379 if (has_timestamping) { 2380 tss->ts[1] = (struct timespec64) {0}; 2381 if (sock_flag(sk, SOCK_TSTAMP_NEW)) 2382 put_cmsg_scm_timestamping64(msg, tss); 2383 else 2384 put_cmsg_scm_timestamping(msg, tss); 2385 } 2386 } 2387 2388 static int tcp_inq_hint(struct sock *sk) 2389 { 2390 const struct tcp_sock *tp = tcp_sk(sk); 2391 u32 copied_seq = READ_ONCE(tp->copied_seq); 2392 u32 rcv_nxt = READ_ONCE(tp->rcv_nxt); 2393 int inq; 2394 2395 inq = rcv_nxt - copied_seq; 2396 if (unlikely(inq < 0 || copied_seq != READ_ONCE(tp->copied_seq))) { 2397 lock_sock(sk); 2398 inq = tp->rcv_nxt - tp->copied_seq; 2399 release_sock(sk); 2400 } 2401 /* After receiving a FIN, tell the user-space to continue reading 2402 * by returning a non-zero inq. 2403 */ 2404 if (inq == 0 && sock_flag(sk, SOCK_DONE)) 2405 inq = 1; 2406 return inq; 2407 } 2408 2409 /* batch __xa_alloc() calls and reduce xa_lock()/xa_unlock() overhead. */ 2410 struct tcp_xa_pool { 2411 u8 max; /* max <= MAX_SKB_FRAGS */ 2412 u8 idx; /* idx <= max */ 2413 __u32 tokens[MAX_SKB_FRAGS]; 2414 netmem_ref netmems[MAX_SKB_FRAGS]; 2415 }; 2416 2417 static void tcp_xa_pool_commit_locked(struct sock *sk, struct tcp_xa_pool *p) 2418 { 2419 int i; 2420 2421 /* Commit part that has been copied to user space. */ 2422 for (i = 0; i < p->idx; i++) 2423 __xa_cmpxchg(&sk->sk_user_frags, p->tokens[i], XA_ZERO_ENTRY, 2424 (__force void *)p->netmems[i], GFP_KERNEL); 2425 /* Rollback what has been pre-allocated and is no longer needed. */ 2426 for (; i < p->max; i++) 2427 __xa_erase(&sk->sk_user_frags, p->tokens[i]); 2428 2429 p->max = 0; 2430 p->idx = 0; 2431 } 2432 2433 static void tcp_xa_pool_commit(struct sock *sk, struct tcp_xa_pool *p) 2434 { 2435 if (!p->max) 2436 return; 2437 2438 xa_lock_bh(&sk->sk_user_frags); 2439 2440 tcp_xa_pool_commit_locked(sk, p); 2441 2442 xa_unlock_bh(&sk->sk_user_frags); 2443 } 2444 2445 static int tcp_xa_pool_refill(struct sock *sk, struct tcp_xa_pool *p, 2446 unsigned int max_frags) 2447 { 2448 int err, k; 2449 2450 if (p->idx < p->max) 2451 return 0; 2452 2453 xa_lock_bh(&sk->sk_user_frags); 2454 2455 tcp_xa_pool_commit_locked(sk, p); 2456 2457 for (k = 0; k < max_frags; k++) { 2458 err = __xa_alloc(&sk->sk_user_frags, &p->tokens[k], 2459 XA_ZERO_ENTRY, xa_limit_31b, GFP_KERNEL); 2460 if (err) 2461 break; 2462 } 2463 2464 xa_unlock_bh(&sk->sk_user_frags); 2465 2466 p->max = k; 2467 p->idx = 0; 2468 return k ? 0 : err; 2469 } 2470 2471 /* On error, returns the -errno. On success, returns number of bytes sent to the 2472 * user. May not consume all of @remaining_len. 2473 */ 2474 static int tcp_recvmsg_dmabuf(struct sock *sk, const struct sk_buff *skb, 2475 unsigned int offset, struct msghdr *msg, 2476 int remaining_len) 2477 { 2478 struct dmabuf_cmsg dmabuf_cmsg = { 0 }; 2479 struct tcp_xa_pool tcp_xa_pool; 2480 unsigned int start; 2481 int i, copy, n; 2482 int sent = 0; 2483 int err = 0; 2484 2485 tcp_xa_pool.max = 0; 2486 tcp_xa_pool.idx = 0; 2487 do { 2488 start = skb_headlen(skb); 2489 2490 if (skb_frags_readable(skb)) { 2491 err = -ENODEV; 2492 goto out; 2493 } 2494 2495 /* Copy header. */ 2496 copy = start - offset; 2497 if (copy > 0) { 2498 copy = min(copy, remaining_len); 2499 2500 n = copy_to_iter(skb->data + offset, copy, 2501 &msg->msg_iter); 2502 if (n != copy) { 2503 err = -EFAULT; 2504 goto out; 2505 } 2506 2507 offset += copy; 2508 remaining_len -= copy; 2509 2510 /* First a dmabuf_cmsg for # bytes copied to user 2511 * buffer. 2512 */ 2513 memset(&dmabuf_cmsg, 0, sizeof(dmabuf_cmsg)); 2514 dmabuf_cmsg.frag_size = copy; 2515 err = put_cmsg_notrunc(msg, SOL_SOCKET, 2516 SO_DEVMEM_LINEAR, 2517 sizeof(dmabuf_cmsg), 2518 &dmabuf_cmsg); 2519 if (err) 2520 goto out; 2521 2522 sent += copy; 2523 2524 if (remaining_len == 0) 2525 goto out; 2526 } 2527 2528 /* after that, send information of dmabuf pages through a 2529 * sequence of cmsg 2530 */ 2531 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2532 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2533 struct net_iov *niov; 2534 u64 frag_offset; 2535 int end; 2536 2537 /* !skb_frags_readable() should indicate that ALL the 2538 * frags in this skb are dmabuf net_iovs. We're checking 2539 * for that flag above, but also check individual frags 2540 * here. If the tcp stack is not setting 2541 * skb_frags_readable() correctly, we still don't want 2542 * to crash here. 2543 */ 2544 if (!skb_frag_net_iov(frag)) { 2545 net_err_ratelimited("Found non-dmabuf skb with net_iov"); 2546 err = -ENODEV; 2547 goto out; 2548 } 2549 2550 niov = skb_frag_net_iov(frag); 2551 if (!net_is_devmem_iov(niov)) { 2552 err = -ENODEV; 2553 goto out; 2554 } 2555 2556 end = start + skb_frag_size(frag); 2557 copy = end - offset; 2558 2559 if (copy > 0) { 2560 copy = min(copy, remaining_len); 2561 2562 frag_offset = net_iov_virtual_addr(niov) + 2563 skb_frag_off(frag) + offset - 2564 start; 2565 dmabuf_cmsg.frag_offset = frag_offset; 2566 dmabuf_cmsg.frag_size = copy; 2567 err = tcp_xa_pool_refill(sk, &tcp_xa_pool, 2568 skb_shinfo(skb)->nr_frags - i); 2569 if (err) 2570 goto out; 2571 2572 /* Will perform the exchange later */ 2573 dmabuf_cmsg.frag_token = tcp_xa_pool.tokens[tcp_xa_pool.idx]; 2574 dmabuf_cmsg.dmabuf_id = net_devmem_iov_binding_id(niov); 2575 2576 offset += copy; 2577 remaining_len -= copy; 2578 2579 err = put_cmsg_notrunc(msg, SOL_SOCKET, 2580 SO_DEVMEM_DMABUF, 2581 sizeof(dmabuf_cmsg), 2582 &dmabuf_cmsg); 2583 if (err) 2584 goto out; 2585 2586 atomic_long_inc(&niov->pp_ref_count); 2587 tcp_xa_pool.netmems[tcp_xa_pool.idx++] = skb_frag_netmem(frag); 2588 2589 sent += copy; 2590 2591 if (remaining_len == 0) 2592 goto out; 2593 } 2594 start = end; 2595 } 2596 2597 tcp_xa_pool_commit(sk, &tcp_xa_pool); 2598 if (!remaining_len) 2599 goto out; 2600 2601 /* if remaining_len is not satisfied yet, we need to go to the 2602 * next frag in the frag_list to satisfy remaining_len. 2603 */ 2604 skb = skb_shinfo(skb)->frag_list ?: skb->next; 2605 2606 offset = offset - start; 2607 } while (skb); 2608 2609 if (remaining_len) { 2610 err = -EFAULT; 2611 goto out; 2612 } 2613 2614 out: 2615 tcp_xa_pool_commit(sk, &tcp_xa_pool); 2616 if (!sent) 2617 sent = err; 2618 2619 return sent; 2620 } 2621 2622 /* 2623 * This routine copies from a sock struct into the user buffer. 2624 * 2625 * Technical note: in 2.3 we work on _locked_ socket, so that 2626 * tricks with *seq access order and skb->users are not required. 2627 * Probably, code can be easily improved even more. 2628 */ 2629 2630 static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len, 2631 int flags, struct scm_timestamping_internal *tss, 2632 int *cmsg_flags) 2633 { 2634 struct tcp_sock *tp = tcp_sk(sk); 2635 int last_copied_dmabuf = -1; /* uninitialized */ 2636 int copied = 0; 2637 u32 peek_seq; 2638 u32 *seq; 2639 unsigned long used; 2640 int err; 2641 int target; /* Read at least this many bytes */ 2642 long timeo; 2643 struct sk_buff *skb, *last; 2644 u32 peek_offset = 0; 2645 u32 urg_hole = 0; 2646 2647 err = -ENOTCONN; 2648 if (sk->sk_state == TCP_LISTEN) 2649 goto out; 2650 2651 if (tp->recvmsg_inq) { 2652 *cmsg_flags = TCP_CMSG_INQ; 2653 msg->msg_get_inq = 1; 2654 } 2655 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 2656 2657 /* Urgent data needs to be handled specially. */ 2658 if (flags & MSG_OOB) 2659 goto recv_urg; 2660 2661 if (unlikely(tp->repair)) { 2662 err = -EPERM; 2663 if (!(flags & MSG_PEEK)) 2664 goto out; 2665 2666 if (tp->repair_queue == TCP_SEND_QUEUE) 2667 goto recv_sndq; 2668 2669 err = -EINVAL; 2670 if (tp->repair_queue == TCP_NO_QUEUE) 2671 goto out; 2672 2673 /* 'common' recv queue MSG_PEEK-ing */ 2674 } 2675 2676 seq = &tp->copied_seq; 2677 if (flags & MSG_PEEK) { 2678 peek_offset = max(sk_peek_offset(sk, flags), 0); 2679 peek_seq = tp->copied_seq + peek_offset; 2680 seq = &peek_seq; 2681 } 2682 2683 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 2684 2685 do { 2686 u32 offset; 2687 2688 /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */ 2689 if (unlikely(tp->urg_data) && tp->urg_seq == *seq) { 2690 if (copied) 2691 break; 2692 if (signal_pending(current)) { 2693 copied = timeo ? sock_intr_errno(timeo) : -EAGAIN; 2694 break; 2695 } 2696 } 2697 2698 /* Next get a buffer. */ 2699 2700 last = skb_peek_tail(&sk->sk_receive_queue); 2701 skb_queue_walk(&sk->sk_receive_queue, skb) { 2702 last = skb; 2703 /* Now that we have two receive queues this 2704 * shouldn't happen. 2705 */ 2706 if (WARN(before(*seq, TCP_SKB_CB(skb)->seq), 2707 "TCP recvmsg seq # bug: copied %X, seq %X, rcvnxt %X, fl %X\n", 2708 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, 2709 flags)) 2710 break; 2711 2712 offset = *seq - TCP_SKB_CB(skb)->seq; 2713 if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) { 2714 pr_err_once("%s: found a SYN, please report !\n", __func__); 2715 offset--; 2716 } 2717 if (offset < skb->len) 2718 goto found_ok_skb; 2719 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 2720 goto found_fin_ok; 2721 WARN(!(flags & MSG_PEEK), 2722 "TCP recvmsg seq # bug 2: copied %X, seq %X, rcvnxt %X, fl %X\n", 2723 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags); 2724 } 2725 2726 /* Well, if we have backlog, try to process it now yet. */ 2727 2728 if (copied >= target && !READ_ONCE(sk->sk_backlog.tail)) 2729 break; 2730 2731 if (copied) { 2732 if (!timeo || 2733 sk->sk_err || 2734 sk->sk_state == TCP_CLOSE || 2735 (sk->sk_shutdown & RCV_SHUTDOWN) || 2736 signal_pending(current)) 2737 break; 2738 } else { 2739 if (sock_flag(sk, SOCK_DONE)) 2740 break; 2741 2742 if (sk->sk_err) { 2743 copied = sock_error(sk); 2744 break; 2745 } 2746 2747 if (sk->sk_shutdown & RCV_SHUTDOWN) 2748 break; 2749 2750 if (sk->sk_state == TCP_CLOSE) { 2751 /* This occurs when user tries to read 2752 * from never connected socket. 2753 */ 2754 copied = -ENOTCONN; 2755 break; 2756 } 2757 2758 if (!timeo) { 2759 copied = -EAGAIN; 2760 break; 2761 } 2762 2763 if (signal_pending(current)) { 2764 copied = sock_intr_errno(timeo); 2765 break; 2766 } 2767 } 2768 2769 if (copied >= target) { 2770 /* Do not sleep, just process backlog. */ 2771 __sk_flush_backlog(sk); 2772 } else { 2773 tcp_cleanup_rbuf(sk, copied); 2774 err = sk_wait_data(sk, &timeo, last); 2775 if (err < 0) { 2776 err = copied ? : err; 2777 goto out; 2778 } 2779 } 2780 2781 if ((flags & MSG_PEEK) && 2782 (peek_seq - peek_offset - copied - urg_hole != tp->copied_seq)) { 2783 net_dbg_ratelimited("TCP(%s:%d): Application bug, race in MSG_PEEK\n", 2784 current->comm, 2785 task_pid_nr(current)); 2786 peek_seq = tp->copied_seq + peek_offset; 2787 } 2788 continue; 2789 2790 found_ok_skb: 2791 /* Ok so how much can we use? */ 2792 used = skb->len - offset; 2793 if (len < used) 2794 used = len; 2795 2796 /* Do we have urgent data here? */ 2797 if (unlikely(tp->urg_data)) { 2798 u32 urg_offset = tp->urg_seq - *seq; 2799 if (urg_offset < used) { 2800 if (!urg_offset) { 2801 if (!sock_flag(sk, SOCK_URGINLINE)) { 2802 WRITE_ONCE(*seq, *seq + 1); 2803 urg_hole++; 2804 offset++; 2805 used--; 2806 if (!used) 2807 goto skip_copy; 2808 } 2809 } else 2810 used = urg_offset; 2811 } 2812 } 2813 2814 if (!(flags & MSG_TRUNC)) { 2815 if (last_copied_dmabuf != -1 && 2816 last_copied_dmabuf != !skb_frags_readable(skb)) 2817 break; 2818 2819 if (skb_frags_readable(skb)) { 2820 err = skb_copy_datagram_msg(skb, offset, msg, 2821 used); 2822 if (err) { 2823 /* Exception. Bailout! */ 2824 if (!copied) 2825 copied = -EFAULT; 2826 break; 2827 } 2828 } else { 2829 if (!(flags & MSG_SOCK_DEVMEM)) { 2830 /* dmabuf skbs can only be received 2831 * with the MSG_SOCK_DEVMEM flag. 2832 */ 2833 if (!copied) 2834 copied = -EFAULT; 2835 2836 break; 2837 } 2838 2839 err = tcp_recvmsg_dmabuf(sk, skb, offset, msg, 2840 used); 2841 if (err < 0) { 2842 if (!copied) 2843 copied = err; 2844 2845 break; 2846 } 2847 used = err; 2848 } 2849 } 2850 2851 last_copied_dmabuf = !skb_frags_readable(skb); 2852 2853 WRITE_ONCE(*seq, *seq + used); 2854 copied += used; 2855 len -= used; 2856 if (flags & MSG_PEEK) 2857 sk_peek_offset_fwd(sk, used); 2858 else 2859 sk_peek_offset_bwd(sk, used); 2860 tcp_rcv_space_adjust(sk); 2861 2862 skip_copy: 2863 if (unlikely(tp->urg_data) && after(tp->copied_seq, tp->urg_seq)) { 2864 WRITE_ONCE(tp->urg_data, 0); 2865 tcp_fast_path_check(sk); 2866 } 2867 2868 if (TCP_SKB_CB(skb)->has_rxtstamp) { 2869 tcp_update_recv_tstamps(skb, tss); 2870 *cmsg_flags |= TCP_CMSG_TS; 2871 } 2872 2873 if (used + offset < skb->len) 2874 continue; 2875 2876 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 2877 goto found_fin_ok; 2878 if (!(flags & MSG_PEEK)) 2879 tcp_eat_recv_skb(sk, skb); 2880 continue; 2881 2882 found_fin_ok: 2883 /* Process the FIN. */ 2884 WRITE_ONCE(*seq, *seq + 1); 2885 if (!(flags & MSG_PEEK)) 2886 tcp_eat_recv_skb(sk, skb); 2887 break; 2888 } while (len > 0); 2889 2890 /* According to UNIX98, msg_name/msg_namelen are ignored 2891 * on connected socket. I was just happy when found this 8) --ANK 2892 */ 2893 2894 /* Clean up data we have read: This will do ACK frames. */ 2895 tcp_cleanup_rbuf(sk, copied); 2896 return copied; 2897 2898 out: 2899 return err; 2900 2901 recv_urg: 2902 err = tcp_recv_urg(sk, msg, len, flags); 2903 goto out; 2904 2905 recv_sndq: 2906 err = tcp_peek_sndq(sk, msg, len); 2907 goto out; 2908 } 2909 2910 int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags, 2911 int *addr_len) 2912 { 2913 int cmsg_flags = 0, ret; 2914 struct scm_timestamping_internal tss; 2915 2916 if (unlikely(flags & MSG_ERRQUEUE)) 2917 return inet_recv_error(sk, msg, len, addr_len); 2918 2919 if (sk_can_busy_loop(sk) && 2920 skb_queue_empty_lockless(&sk->sk_receive_queue) && 2921 sk->sk_state == TCP_ESTABLISHED) 2922 sk_busy_loop(sk, flags & MSG_DONTWAIT); 2923 2924 lock_sock(sk); 2925 ret = tcp_recvmsg_locked(sk, msg, len, flags, &tss, &cmsg_flags); 2926 release_sock(sk); 2927 2928 if ((cmsg_flags || msg->msg_get_inq) && ret >= 0) { 2929 if (cmsg_flags & TCP_CMSG_TS) 2930 tcp_recv_timestamp(msg, sk, &tss); 2931 if (msg->msg_get_inq) { 2932 msg->msg_inq = tcp_inq_hint(sk); 2933 if (cmsg_flags & TCP_CMSG_INQ) 2934 put_cmsg(msg, SOL_TCP, TCP_CM_INQ, 2935 sizeof(msg->msg_inq), &msg->msg_inq); 2936 } 2937 } 2938 return ret; 2939 } 2940 EXPORT_IPV6_MOD(tcp_recvmsg); 2941 2942 void tcp_set_state(struct sock *sk, int state) 2943 { 2944 int oldstate = sk->sk_state; 2945 2946 /* We defined a new enum for TCP states that are exported in BPF 2947 * so as not force the internal TCP states to be frozen. The 2948 * following checks will detect if an internal state value ever 2949 * differs from the BPF value. If this ever happens, then we will 2950 * need to remap the internal value to the BPF value before calling 2951 * tcp_call_bpf_2arg. 2952 */ 2953 BUILD_BUG_ON((int)BPF_TCP_ESTABLISHED != (int)TCP_ESTABLISHED); 2954 BUILD_BUG_ON((int)BPF_TCP_SYN_SENT != (int)TCP_SYN_SENT); 2955 BUILD_BUG_ON((int)BPF_TCP_SYN_RECV != (int)TCP_SYN_RECV); 2956 BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT1 != (int)TCP_FIN_WAIT1); 2957 BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT2 != (int)TCP_FIN_WAIT2); 2958 BUILD_BUG_ON((int)BPF_TCP_TIME_WAIT != (int)TCP_TIME_WAIT); 2959 BUILD_BUG_ON((int)BPF_TCP_CLOSE != (int)TCP_CLOSE); 2960 BUILD_BUG_ON((int)BPF_TCP_CLOSE_WAIT != (int)TCP_CLOSE_WAIT); 2961 BUILD_BUG_ON((int)BPF_TCP_LAST_ACK != (int)TCP_LAST_ACK); 2962 BUILD_BUG_ON((int)BPF_TCP_LISTEN != (int)TCP_LISTEN); 2963 BUILD_BUG_ON((int)BPF_TCP_CLOSING != (int)TCP_CLOSING); 2964 BUILD_BUG_ON((int)BPF_TCP_NEW_SYN_RECV != (int)TCP_NEW_SYN_RECV); 2965 BUILD_BUG_ON((int)BPF_TCP_BOUND_INACTIVE != (int)TCP_BOUND_INACTIVE); 2966 BUILD_BUG_ON((int)BPF_TCP_MAX_STATES != (int)TCP_MAX_STATES); 2967 2968 /* bpf uapi header bpf.h defines an anonymous enum with values 2969 * BPF_TCP_* used by bpf programs. Currently gcc built vmlinux 2970 * is able to emit this enum in DWARF due to the above BUILD_BUG_ON. 2971 * But clang built vmlinux does not have this enum in DWARF 2972 * since clang removes the above code before generating IR/debuginfo. 2973 * Let us explicitly emit the type debuginfo to ensure the 2974 * above-mentioned anonymous enum in the vmlinux DWARF and hence BTF 2975 * regardless of which compiler is used. 2976 */ 2977 BTF_TYPE_EMIT_ENUM(BPF_TCP_ESTABLISHED); 2978 2979 if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_STATE_CB_FLAG)) 2980 tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state); 2981 2982 switch (state) { 2983 case TCP_ESTABLISHED: 2984 if (oldstate != TCP_ESTABLISHED) 2985 TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB); 2986 break; 2987 case TCP_CLOSE_WAIT: 2988 if (oldstate == TCP_SYN_RECV) 2989 TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB); 2990 break; 2991 2992 case TCP_CLOSE: 2993 if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED) 2994 TCP_INC_STATS(sock_net(sk), TCP_MIB_ESTABRESETS); 2995 2996 sk->sk_prot->unhash(sk); 2997 if (inet_csk(sk)->icsk_bind_hash && 2998 !(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) 2999 inet_put_port(sk); 3000 fallthrough; 3001 default: 3002 if (oldstate == TCP_ESTABLISHED || oldstate == TCP_CLOSE_WAIT) 3003 TCP_DEC_STATS(sock_net(sk), TCP_MIB_CURRESTAB); 3004 } 3005 3006 /* Change state AFTER socket is unhashed to avoid closed 3007 * socket sitting in hash tables. 3008 */ 3009 inet_sk_state_store(sk, state); 3010 } 3011 EXPORT_SYMBOL_GPL(tcp_set_state); 3012 3013 /* 3014 * State processing on a close. This implements the state shift for 3015 * sending our FIN frame. Note that we only send a FIN for some 3016 * states. A shutdown() may have already sent the FIN, or we may be 3017 * closed. 3018 */ 3019 3020 static const unsigned char new_state[16] = { 3021 /* current state: new state: action: */ 3022 [0 /* (Invalid) */] = TCP_CLOSE, 3023 [TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 3024 [TCP_SYN_SENT] = TCP_CLOSE, 3025 [TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 3026 [TCP_FIN_WAIT1] = TCP_FIN_WAIT1, 3027 [TCP_FIN_WAIT2] = TCP_FIN_WAIT2, 3028 [TCP_TIME_WAIT] = TCP_CLOSE, 3029 [TCP_CLOSE] = TCP_CLOSE, 3030 [TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN, 3031 [TCP_LAST_ACK] = TCP_LAST_ACK, 3032 [TCP_LISTEN] = TCP_CLOSE, 3033 [TCP_CLOSING] = TCP_CLOSING, 3034 [TCP_NEW_SYN_RECV] = TCP_CLOSE, /* should not happen ! */ 3035 }; 3036 3037 static int tcp_close_state(struct sock *sk) 3038 { 3039 int next = (int)new_state[sk->sk_state]; 3040 int ns = next & TCP_STATE_MASK; 3041 3042 tcp_set_state(sk, ns); 3043 3044 return next & TCP_ACTION_FIN; 3045 } 3046 3047 /* 3048 * Shutdown the sending side of a connection. Much like close except 3049 * that we don't receive shut down or sock_set_flag(sk, SOCK_DEAD). 3050 */ 3051 3052 void tcp_shutdown(struct sock *sk, int how) 3053 { 3054 /* We need to grab some memory, and put together a FIN, 3055 * and then put it into the queue to be sent. 3056 * Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92. 3057 */ 3058 if (!(how & SEND_SHUTDOWN)) 3059 return; 3060 3061 /* If we've already sent a FIN, or it's a closed state, skip this. */ 3062 if ((1 << sk->sk_state) & 3063 (TCPF_ESTABLISHED | TCPF_SYN_SENT | 3064 TCPF_CLOSE_WAIT)) { 3065 /* Clear out any half completed packets. FIN if needed. */ 3066 if (tcp_close_state(sk)) 3067 tcp_send_fin(sk); 3068 } 3069 } 3070 EXPORT_IPV6_MOD(tcp_shutdown); 3071 3072 int tcp_orphan_count_sum(void) 3073 { 3074 int i, total = 0; 3075 3076 for_each_possible_cpu(i) 3077 total += per_cpu(tcp_orphan_count, i); 3078 3079 return max(total, 0); 3080 } 3081 3082 static int tcp_orphan_cache; 3083 static struct timer_list tcp_orphan_timer; 3084 #define TCP_ORPHAN_TIMER_PERIOD msecs_to_jiffies(100) 3085 3086 static void tcp_orphan_update(struct timer_list *unused) 3087 { 3088 WRITE_ONCE(tcp_orphan_cache, tcp_orphan_count_sum()); 3089 mod_timer(&tcp_orphan_timer, jiffies + TCP_ORPHAN_TIMER_PERIOD); 3090 } 3091 3092 static bool tcp_too_many_orphans(int shift) 3093 { 3094 return READ_ONCE(tcp_orphan_cache) << shift > 3095 READ_ONCE(sysctl_tcp_max_orphans); 3096 } 3097 3098 static bool tcp_out_of_memory(const struct sock *sk) 3099 { 3100 if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF && 3101 sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2)) 3102 return true; 3103 return false; 3104 } 3105 3106 bool tcp_check_oom(const struct sock *sk, int shift) 3107 { 3108 bool too_many_orphans, out_of_socket_memory; 3109 3110 too_many_orphans = tcp_too_many_orphans(shift); 3111 out_of_socket_memory = tcp_out_of_memory(sk); 3112 3113 if (too_many_orphans) 3114 net_info_ratelimited("too many orphaned sockets\n"); 3115 if (out_of_socket_memory) 3116 net_info_ratelimited("out of memory -- consider tuning tcp_mem\n"); 3117 return too_many_orphans || out_of_socket_memory; 3118 } 3119 3120 void __tcp_close(struct sock *sk, long timeout) 3121 { 3122 bool data_was_unread = false; 3123 struct sk_buff *skb; 3124 int state; 3125 3126 WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK); 3127 3128 if (sk->sk_state == TCP_LISTEN) { 3129 tcp_set_state(sk, TCP_CLOSE); 3130 3131 /* Special case. */ 3132 inet_csk_listen_stop(sk); 3133 3134 goto adjudge_to_death; 3135 } 3136 3137 /* We need to flush the recv. buffs. We do this only on the 3138 * descriptor close, not protocol-sourced closes, because the 3139 * reader process may not have drained the data yet! 3140 */ 3141 while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) { 3142 u32 end_seq = TCP_SKB_CB(skb)->end_seq; 3143 3144 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 3145 end_seq--; 3146 if (after(end_seq, tcp_sk(sk)->copied_seq)) 3147 data_was_unread = true; 3148 tcp_eat_recv_skb(sk, skb); 3149 } 3150 3151 /* If socket has been already reset (e.g. in tcp_reset()) - kill it. */ 3152 if (sk->sk_state == TCP_CLOSE) 3153 goto adjudge_to_death; 3154 3155 /* As outlined in RFC 2525, section 2.17, we send a RST here because 3156 * data was lost. To witness the awful effects of the old behavior of 3157 * always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk 3158 * GET in an FTP client, suspend the process, wait for the client to 3159 * advertise a zero window, then kill -9 the FTP client, wheee... 3160 * Note: timeout is always zero in such a case. 3161 */ 3162 if (unlikely(tcp_sk(sk)->repair)) { 3163 sk->sk_prot->disconnect(sk, 0); 3164 } else if (data_was_unread) { 3165 /* Unread data was tossed, zap the connection. */ 3166 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE); 3167 tcp_set_state(sk, TCP_CLOSE); 3168 tcp_send_active_reset(sk, sk->sk_allocation, 3169 SK_RST_REASON_TCP_ABORT_ON_CLOSE); 3170 } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) { 3171 /* Check zero linger _after_ checking for unread data. */ 3172 sk->sk_prot->disconnect(sk, 0); 3173 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA); 3174 } else if (tcp_close_state(sk)) { 3175 /* We FIN if the application ate all the data before 3176 * zapping the connection. 3177 */ 3178 3179 /* RED-PEN. Formally speaking, we have broken TCP state 3180 * machine. State transitions: 3181 * 3182 * TCP_ESTABLISHED -> TCP_FIN_WAIT1 3183 * TCP_SYN_RECV -> TCP_FIN_WAIT1 (it is difficult) 3184 * TCP_CLOSE_WAIT -> TCP_LAST_ACK 3185 * 3186 * are legal only when FIN has been sent (i.e. in window), 3187 * rather than queued out of window. Purists blame. 3188 * 3189 * F.e. "RFC state" is ESTABLISHED, 3190 * if Linux state is FIN-WAIT-1, but FIN is still not sent. 3191 * 3192 * The visible declinations are that sometimes 3193 * we enter time-wait state, when it is not required really 3194 * (harmless), do not send active resets, when they are 3195 * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when 3196 * they look as CLOSING or LAST_ACK for Linux) 3197 * Probably, I missed some more holelets. 3198 * --ANK 3199 * XXX (TFO) - To start off we don't support SYN+ACK+FIN 3200 * in a single packet! (May consider it later but will 3201 * probably need API support or TCP_CORK SYN-ACK until 3202 * data is written and socket is closed.) 3203 */ 3204 tcp_send_fin(sk); 3205 } 3206 3207 sk_stream_wait_close(sk, timeout); 3208 3209 adjudge_to_death: 3210 state = sk->sk_state; 3211 sock_hold(sk); 3212 sock_orphan(sk); 3213 3214 local_bh_disable(); 3215 bh_lock_sock(sk); 3216 /* remove backlog if any, without releasing ownership. */ 3217 __release_sock(sk); 3218 3219 tcp_orphan_count_inc(); 3220 3221 /* Have we already been destroyed by a softirq or backlog? */ 3222 if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE) 3223 goto out; 3224 3225 /* This is a (useful) BSD violating of the RFC. There is a 3226 * problem with TCP as specified in that the other end could 3227 * keep a socket open forever with no application left this end. 3228 * We use a 1 minute timeout (about the same as BSD) then kill 3229 * our end. If they send after that then tough - BUT: long enough 3230 * that we won't make the old 4*rto = almost no time - whoops 3231 * reset mistake. 3232 * 3233 * Nope, it was not mistake. It is really desired behaviour 3234 * f.e. on http servers, when such sockets are useless, but 3235 * consume significant resources. Let's do it with special 3236 * linger2 option. --ANK 3237 */ 3238 3239 if (sk->sk_state == TCP_FIN_WAIT2) { 3240 struct tcp_sock *tp = tcp_sk(sk); 3241 if (READ_ONCE(tp->linger2) < 0) { 3242 tcp_set_state(sk, TCP_CLOSE); 3243 tcp_send_active_reset(sk, GFP_ATOMIC, 3244 SK_RST_REASON_TCP_ABORT_ON_LINGER); 3245 __NET_INC_STATS(sock_net(sk), 3246 LINUX_MIB_TCPABORTONLINGER); 3247 } else { 3248 const int tmo = tcp_fin_time(sk); 3249 3250 if (tmo > TCP_TIMEWAIT_LEN) { 3251 tcp_reset_keepalive_timer(sk, 3252 tmo - TCP_TIMEWAIT_LEN); 3253 } else { 3254 tcp_time_wait(sk, TCP_FIN_WAIT2, tmo); 3255 goto out; 3256 } 3257 } 3258 } 3259 if (sk->sk_state != TCP_CLOSE) { 3260 if (tcp_check_oom(sk, 0)) { 3261 tcp_set_state(sk, TCP_CLOSE); 3262 tcp_send_active_reset(sk, GFP_ATOMIC, 3263 SK_RST_REASON_TCP_ABORT_ON_MEMORY); 3264 __NET_INC_STATS(sock_net(sk), 3265 LINUX_MIB_TCPABORTONMEMORY); 3266 } else if (!check_net(sock_net(sk))) { 3267 /* Not possible to send reset; just close */ 3268 tcp_set_state(sk, TCP_CLOSE); 3269 } 3270 } 3271 3272 if (sk->sk_state == TCP_CLOSE) { 3273 struct request_sock *req; 3274 3275 req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk, 3276 lockdep_sock_is_held(sk)); 3277 /* We could get here with a non-NULL req if the socket is 3278 * aborted (e.g., closed with unread data) before 3WHS 3279 * finishes. 3280 */ 3281 if (req) 3282 reqsk_fastopen_remove(sk, req, false); 3283 inet_csk_destroy_sock(sk); 3284 } 3285 /* Otherwise, socket is reprieved until protocol close. */ 3286 3287 out: 3288 bh_unlock_sock(sk); 3289 local_bh_enable(); 3290 } 3291 3292 void tcp_close(struct sock *sk, long timeout) 3293 { 3294 lock_sock(sk); 3295 __tcp_close(sk, timeout); 3296 release_sock(sk); 3297 if (!sk->sk_net_refcnt) 3298 inet_csk_clear_xmit_timers_sync(sk); 3299 sock_put(sk); 3300 } 3301 EXPORT_SYMBOL(tcp_close); 3302 3303 /* These states need RST on ABORT according to RFC793 */ 3304 3305 static inline bool tcp_need_reset(int state) 3306 { 3307 return (1 << state) & 3308 (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 | 3309 TCPF_FIN_WAIT2 | TCPF_SYN_RECV); 3310 } 3311 3312 static void tcp_rtx_queue_purge(struct sock *sk) 3313 { 3314 struct rb_node *p = rb_first(&sk->tcp_rtx_queue); 3315 3316 tcp_sk(sk)->highest_sack = NULL; 3317 while (p) { 3318 struct sk_buff *skb = rb_to_skb(p); 3319 3320 p = rb_next(p); 3321 /* Since we are deleting whole queue, no need to 3322 * list_del(&skb->tcp_tsorted_anchor) 3323 */ 3324 tcp_rtx_queue_unlink(skb, sk); 3325 tcp_wmem_free_skb(sk, skb); 3326 } 3327 } 3328 3329 void tcp_write_queue_purge(struct sock *sk) 3330 { 3331 struct sk_buff *skb; 3332 3333 tcp_chrono_stop(sk, TCP_CHRONO_BUSY); 3334 while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) { 3335 tcp_skb_tsorted_anchor_cleanup(skb); 3336 tcp_wmem_free_skb(sk, skb); 3337 } 3338 tcp_rtx_queue_purge(sk); 3339 INIT_LIST_HEAD(&tcp_sk(sk)->tsorted_sent_queue); 3340 tcp_clear_all_retrans_hints(tcp_sk(sk)); 3341 tcp_sk(sk)->packets_out = 0; 3342 inet_csk(sk)->icsk_backoff = 0; 3343 } 3344 3345 int tcp_disconnect(struct sock *sk, int flags) 3346 { 3347 struct inet_sock *inet = inet_sk(sk); 3348 struct inet_connection_sock *icsk = inet_csk(sk); 3349 struct tcp_sock *tp = tcp_sk(sk); 3350 int old_state = sk->sk_state; 3351 struct request_sock *req; 3352 u32 seq; 3353 3354 if (old_state != TCP_CLOSE) 3355 tcp_set_state(sk, TCP_CLOSE); 3356 3357 /* ABORT function of RFC793 */ 3358 if (old_state == TCP_LISTEN) { 3359 inet_csk_listen_stop(sk); 3360 } else if (unlikely(tp->repair)) { 3361 WRITE_ONCE(sk->sk_err, ECONNABORTED); 3362 } else if (tcp_need_reset(old_state)) { 3363 tcp_send_active_reset(sk, gfp_any(), SK_RST_REASON_TCP_STATE); 3364 WRITE_ONCE(sk->sk_err, ECONNRESET); 3365 } else if (tp->snd_nxt != tp->write_seq && 3366 (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK)) { 3367 /* The last check adjusts for discrepancy of Linux wrt. RFC 3368 * states 3369 */ 3370 tcp_send_active_reset(sk, gfp_any(), 3371 SK_RST_REASON_TCP_DISCONNECT_WITH_DATA); 3372 WRITE_ONCE(sk->sk_err, ECONNRESET); 3373 } else if (old_state == TCP_SYN_SENT) 3374 WRITE_ONCE(sk->sk_err, ECONNRESET); 3375 3376 tcp_clear_xmit_timers(sk); 3377 __skb_queue_purge(&sk->sk_receive_queue); 3378 WRITE_ONCE(tp->copied_seq, tp->rcv_nxt); 3379 WRITE_ONCE(tp->urg_data, 0); 3380 sk_set_peek_off(sk, -1); 3381 tcp_write_queue_purge(sk); 3382 tcp_fastopen_active_disable_ofo_check(sk); 3383 skb_rbtree_purge(&tp->out_of_order_queue); 3384 3385 inet->inet_dport = 0; 3386 3387 inet_bhash2_reset_saddr(sk); 3388 3389 WRITE_ONCE(sk->sk_shutdown, 0); 3390 sock_reset_flag(sk, SOCK_DONE); 3391 tp->srtt_us = 0; 3392 tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT); 3393 tp->rcv_rtt_last_tsecr = 0; 3394 3395 seq = tp->write_seq + tp->max_window + 2; 3396 if (!seq) 3397 seq = 1; 3398 WRITE_ONCE(tp->write_seq, seq); 3399 3400 icsk->icsk_backoff = 0; 3401 WRITE_ONCE(icsk->icsk_probes_out, 0); 3402 icsk->icsk_probes_tstamp = 0; 3403 icsk->icsk_rto = TCP_TIMEOUT_INIT; 3404 WRITE_ONCE(icsk->icsk_rto_min, TCP_RTO_MIN); 3405 WRITE_ONCE(icsk->icsk_delack_max, TCP_DELACK_MAX); 3406 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; 3407 tcp_snd_cwnd_set(tp, TCP_INIT_CWND); 3408 tp->snd_cwnd_cnt = 0; 3409 tp->is_cwnd_limited = 0; 3410 tp->max_packets_out = 0; 3411 tp->window_clamp = 0; 3412 tp->delivered = 0; 3413 tp->delivered_ce = 0; 3414 tp->accecn_fail_mode = 0; 3415 tp->saw_accecn_opt = TCP_ACCECN_OPT_NOT_SEEN; 3416 tcp_accecn_init_counters(tp); 3417 tp->prev_ecnfield = 0; 3418 tp->accecn_opt_tstamp = 0; 3419 if (icsk->icsk_ca_initialized && icsk->icsk_ca_ops->release) 3420 icsk->icsk_ca_ops->release(sk); 3421 memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv)); 3422 icsk->icsk_ca_initialized = 0; 3423 tcp_set_ca_state(sk, TCP_CA_Open); 3424 tp->is_sack_reneg = 0; 3425 tcp_clear_retrans(tp); 3426 tp->total_retrans = 0; 3427 inet_csk_delack_init(sk); 3428 /* Initialize rcv_mss to TCP_MIN_MSS to avoid division by 0 3429 * issue in __tcp_select_window() 3430 */ 3431 icsk->icsk_ack.rcv_mss = TCP_MIN_MSS; 3432 memset(&tp->rx_opt, 0, sizeof(tp->rx_opt)); 3433 __sk_dst_reset(sk); 3434 dst_release(unrcu_pointer(xchg(&sk->sk_rx_dst, NULL))); 3435 tcp_saved_syn_free(tp); 3436 tp->compressed_ack = 0; 3437 tp->segs_in = 0; 3438 tp->segs_out = 0; 3439 tp->bytes_sent = 0; 3440 tp->bytes_acked = 0; 3441 tp->bytes_received = 0; 3442 tp->bytes_retrans = 0; 3443 tp->data_segs_in = 0; 3444 tp->data_segs_out = 0; 3445 tp->duplicate_sack[0].start_seq = 0; 3446 tp->duplicate_sack[0].end_seq = 0; 3447 tp->dsack_dups = 0; 3448 tp->reord_seen = 0; 3449 tp->retrans_out = 0; 3450 tp->sacked_out = 0; 3451 tp->tlp_high_seq = 0; 3452 tp->last_oow_ack_time = 0; 3453 tp->plb_rehash = 0; 3454 /* There's a bubble in the pipe until at least the first ACK. */ 3455 tp->app_limited = ~0U; 3456 tp->rate_app_limited = 1; 3457 tp->rack.mstamp = 0; 3458 tp->rack.advanced = 0; 3459 tp->rack.reo_wnd_steps = 1; 3460 tp->rack.last_delivered = 0; 3461 tp->rack.reo_wnd_persist = 0; 3462 tp->rack.dsack_seen = 0; 3463 tp->syn_data_acked = 0; 3464 tp->syn_fastopen_child = 0; 3465 tp->rx_opt.saw_tstamp = 0; 3466 tp->rx_opt.dsack = 0; 3467 tp->rx_opt.num_sacks = 0; 3468 tp->rcv_ooopack = 0; 3469 3470 3471 /* Clean up fastopen related fields */ 3472 req = rcu_dereference_protected(tp->fastopen_rsk, 3473 lockdep_sock_is_held(sk)); 3474 if (req) 3475 reqsk_fastopen_remove(sk, req, false); 3476 tcp_free_fastopen_req(tp); 3477 inet_clear_bit(DEFER_CONNECT, sk); 3478 tp->fastopen_client_fail = 0; 3479 3480 WARN_ON(inet->inet_num && !icsk->icsk_bind_hash); 3481 3482 if (sk->sk_frag.page) { 3483 put_page(sk->sk_frag.page); 3484 sk->sk_frag.page = NULL; 3485 sk->sk_frag.offset = 0; 3486 } 3487 sk_error_report(sk); 3488 return 0; 3489 } 3490 EXPORT_SYMBOL(tcp_disconnect); 3491 3492 static inline bool tcp_can_repair_sock(const struct sock *sk) 3493 { 3494 return sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) && 3495 (sk->sk_state != TCP_LISTEN); 3496 } 3497 3498 static int tcp_repair_set_window(struct tcp_sock *tp, sockptr_t optbuf, int len) 3499 { 3500 struct tcp_repair_window opt; 3501 3502 if (!tp->repair) 3503 return -EPERM; 3504 3505 if (len != sizeof(opt)) 3506 return -EINVAL; 3507 3508 if (copy_from_sockptr(&opt, optbuf, sizeof(opt))) 3509 return -EFAULT; 3510 3511 if (opt.max_window < opt.snd_wnd) 3512 return -EINVAL; 3513 3514 if (after(opt.snd_wl1, tp->rcv_nxt + opt.rcv_wnd)) 3515 return -EINVAL; 3516 3517 if (after(opt.rcv_wup, tp->rcv_nxt)) 3518 return -EINVAL; 3519 3520 tp->snd_wl1 = opt.snd_wl1; 3521 tp->snd_wnd = opt.snd_wnd; 3522 tp->max_window = opt.max_window; 3523 3524 tp->rcv_wnd = opt.rcv_wnd; 3525 tp->rcv_wup = opt.rcv_wup; 3526 3527 return 0; 3528 } 3529 3530 static int tcp_repair_options_est(struct sock *sk, sockptr_t optbuf, 3531 unsigned int len) 3532 { 3533 struct tcp_sock *tp = tcp_sk(sk); 3534 struct tcp_repair_opt opt; 3535 size_t offset = 0; 3536 3537 while (len >= sizeof(opt)) { 3538 if (copy_from_sockptr_offset(&opt, optbuf, offset, sizeof(opt))) 3539 return -EFAULT; 3540 3541 offset += sizeof(opt); 3542 len -= sizeof(opt); 3543 3544 switch (opt.opt_code) { 3545 case TCPOPT_MSS: 3546 tp->rx_opt.mss_clamp = opt.opt_val; 3547 tcp_mtup_init(sk); 3548 break; 3549 case TCPOPT_WINDOW: 3550 { 3551 u16 snd_wscale = opt.opt_val & 0xFFFF; 3552 u16 rcv_wscale = opt.opt_val >> 16; 3553 3554 if (snd_wscale > TCP_MAX_WSCALE || rcv_wscale > TCP_MAX_WSCALE) 3555 return -EFBIG; 3556 3557 tp->rx_opt.snd_wscale = snd_wscale; 3558 tp->rx_opt.rcv_wscale = rcv_wscale; 3559 tp->rx_opt.wscale_ok = 1; 3560 } 3561 break; 3562 case TCPOPT_SACK_PERM: 3563 if (opt.opt_val != 0) 3564 return -EINVAL; 3565 3566 tp->rx_opt.sack_ok |= TCP_SACK_SEEN; 3567 break; 3568 case TCPOPT_TIMESTAMP: 3569 if (opt.opt_val != 0) 3570 return -EINVAL; 3571 3572 tp->rx_opt.tstamp_ok = 1; 3573 break; 3574 } 3575 } 3576 3577 return 0; 3578 } 3579 3580 DEFINE_STATIC_KEY_FALSE(tcp_tx_delay_enabled); 3581 EXPORT_IPV6_MOD(tcp_tx_delay_enabled); 3582 3583 static void tcp_enable_tx_delay(void) 3584 { 3585 if (!static_branch_unlikely(&tcp_tx_delay_enabled)) { 3586 static int __tcp_tx_delay_enabled = 0; 3587 3588 if (cmpxchg(&__tcp_tx_delay_enabled, 0, 1) == 0) { 3589 static_branch_enable(&tcp_tx_delay_enabled); 3590 pr_info("TCP_TX_DELAY enabled\n"); 3591 } 3592 } 3593 } 3594 3595 /* When set indicates to always queue non-full frames. Later the user clears 3596 * this option and we transmit any pending partial frames in the queue. This is 3597 * meant to be used alongside sendfile() to get properly filled frames when the 3598 * user (for example) must write out headers with a write() call first and then 3599 * use sendfile to send out the data parts. 3600 * 3601 * TCP_CORK can be set together with TCP_NODELAY and it is stronger than 3602 * TCP_NODELAY. 3603 */ 3604 void __tcp_sock_set_cork(struct sock *sk, bool on) 3605 { 3606 struct tcp_sock *tp = tcp_sk(sk); 3607 3608 if (on) { 3609 tp->nonagle |= TCP_NAGLE_CORK; 3610 } else { 3611 tp->nonagle &= ~TCP_NAGLE_CORK; 3612 if (tp->nonagle & TCP_NAGLE_OFF) 3613 tp->nonagle |= TCP_NAGLE_PUSH; 3614 tcp_push_pending_frames(sk); 3615 } 3616 } 3617 3618 void tcp_sock_set_cork(struct sock *sk, bool on) 3619 { 3620 lock_sock(sk); 3621 __tcp_sock_set_cork(sk, on); 3622 release_sock(sk); 3623 } 3624 EXPORT_SYMBOL(tcp_sock_set_cork); 3625 3626 /* TCP_NODELAY is weaker than TCP_CORK, so that this option on corked socket is 3627 * remembered, but it is not activated until cork is cleared. 3628 * 3629 * However, when TCP_NODELAY is set we make an explicit push, which overrides 3630 * even TCP_CORK for currently queued segments. 3631 */ 3632 void __tcp_sock_set_nodelay(struct sock *sk, bool on) 3633 { 3634 if (on) { 3635 tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH; 3636 tcp_push_pending_frames(sk); 3637 } else { 3638 tcp_sk(sk)->nonagle &= ~TCP_NAGLE_OFF; 3639 } 3640 } 3641 3642 void tcp_sock_set_nodelay(struct sock *sk) 3643 { 3644 lock_sock(sk); 3645 __tcp_sock_set_nodelay(sk, true); 3646 release_sock(sk); 3647 } 3648 EXPORT_SYMBOL(tcp_sock_set_nodelay); 3649 3650 static void __tcp_sock_set_quickack(struct sock *sk, int val) 3651 { 3652 if (!val) { 3653 inet_csk_enter_pingpong_mode(sk); 3654 return; 3655 } 3656 3657 inet_csk_exit_pingpong_mode(sk); 3658 if ((1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) && 3659 inet_csk_ack_scheduled(sk)) { 3660 inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_PUSHED; 3661 tcp_cleanup_rbuf(sk, 1); 3662 if (!(val & 1)) 3663 inet_csk_enter_pingpong_mode(sk); 3664 } 3665 } 3666 3667 void tcp_sock_set_quickack(struct sock *sk, int val) 3668 { 3669 lock_sock(sk); 3670 __tcp_sock_set_quickack(sk, val); 3671 release_sock(sk); 3672 } 3673 EXPORT_SYMBOL(tcp_sock_set_quickack); 3674 3675 int tcp_sock_set_syncnt(struct sock *sk, int val) 3676 { 3677 if (val < 1 || val > MAX_TCP_SYNCNT) 3678 return -EINVAL; 3679 3680 WRITE_ONCE(inet_csk(sk)->icsk_syn_retries, val); 3681 return 0; 3682 } 3683 EXPORT_SYMBOL(tcp_sock_set_syncnt); 3684 3685 int tcp_sock_set_user_timeout(struct sock *sk, int val) 3686 { 3687 /* Cap the max time in ms TCP will retry or probe the window 3688 * before giving up and aborting (ETIMEDOUT) a connection. 3689 */ 3690 if (val < 0) 3691 return -EINVAL; 3692 3693 WRITE_ONCE(inet_csk(sk)->icsk_user_timeout, val); 3694 return 0; 3695 } 3696 EXPORT_SYMBOL(tcp_sock_set_user_timeout); 3697 3698 int tcp_sock_set_keepidle_locked(struct sock *sk, int val) 3699 { 3700 struct tcp_sock *tp = tcp_sk(sk); 3701 3702 if (val < 1 || val > MAX_TCP_KEEPIDLE) 3703 return -EINVAL; 3704 3705 /* Paired with WRITE_ONCE() in keepalive_time_when() */ 3706 WRITE_ONCE(tp->keepalive_time, val * HZ); 3707 if (sock_flag(sk, SOCK_KEEPOPEN) && 3708 !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) { 3709 u32 elapsed = keepalive_time_elapsed(tp); 3710 3711 if (tp->keepalive_time > elapsed) 3712 elapsed = tp->keepalive_time - elapsed; 3713 else 3714 elapsed = 0; 3715 tcp_reset_keepalive_timer(sk, elapsed); 3716 } 3717 3718 return 0; 3719 } 3720 3721 int tcp_sock_set_keepidle(struct sock *sk, int val) 3722 { 3723 int err; 3724 3725 lock_sock(sk); 3726 err = tcp_sock_set_keepidle_locked(sk, val); 3727 release_sock(sk); 3728 return err; 3729 } 3730 EXPORT_SYMBOL(tcp_sock_set_keepidle); 3731 3732 int tcp_sock_set_keepintvl(struct sock *sk, int val) 3733 { 3734 if (val < 1 || val > MAX_TCP_KEEPINTVL) 3735 return -EINVAL; 3736 3737 WRITE_ONCE(tcp_sk(sk)->keepalive_intvl, val * HZ); 3738 return 0; 3739 } 3740 EXPORT_SYMBOL(tcp_sock_set_keepintvl); 3741 3742 int tcp_sock_set_keepcnt(struct sock *sk, int val) 3743 { 3744 if (val < 1 || val > MAX_TCP_KEEPCNT) 3745 return -EINVAL; 3746 3747 /* Paired with READ_ONCE() in keepalive_probes() */ 3748 WRITE_ONCE(tcp_sk(sk)->keepalive_probes, val); 3749 return 0; 3750 } 3751 EXPORT_SYMBOL(tcp_sock_set_keepcnt); 3752 3753 int tcp_set_window_clamp(struct sock *sk, int val) 3754 { 3755 u32 old_window_clamp, new_window_clamp, new_rcv_ssthresh; 3756 struct tcp_sock *tp = tcp_sk(sk); 3757 3758 if (!val) { 3759 if (sk->sk_state != TCP_CLOSE) 3760 return -EINVAL; 3761 WRITE_ONCE(tp->window_clamp, 0); 3762 return 0; 3763 } 3764 3765 old_window_clamp = tp->window_clamp; 3766 new_window_clamp = max_t(int, SOCK_MIN_RCVBUF / 2, val); 3767 3768 if (new_window_clamp == old_window_clamp) 3769 return 0; 3770 3771 WRITE_ONCE(tp->window_clamp, new_window_clamp); 3772 3773 /* Need to apply the reserved mem provisioning only 3774 * when shrinking the window clamp. 3775 */ 3776 if (new_window_clamp < old_window_clamp) { 3777 __tcp_adjust_rcv_ssthresh(sk, new_window_clamp); 3778 } else { 3779 new_rcv_ssthresh = min(tp->rcv_wnd, new_window_clamp); 3780 tp->rcv_ssthresh = max(new_rcv_ssthresh, tp->rcv_ssthresh); 3781 } 3782 return 0; 3783 } 3784 3785 int tcp_sock_set_maxseg(struct sock *sk, int val) 3786 { 3787 /* Values greater than interface MTU won't take effect. However 3788 * at the point when this call is done we typically don't yet 3789 * know which interface is going to be used 3790 */ 3791 if (val && (val < TCP_MIN_MSS || val > MAX_TCP_WINDOW)) 3792 return -EINVAL; 3793 3794 WRITE_ONCE(tcp_sk(sk)->rx_opt.user_mss, val); 3795 return 0; 3796 } 3797 3798 /* 3799 * Socket option code for TCP. 3800 */ 3801 int do_tcp_setsockopt(struct sock *sk, int level, int optname, 3802 sockptr_t optval, unsigned int optlen) 3803 { 3804 struct tcp_sock *tp = tcp_sk(sk); 3805 struct inet_connection_sock *icsk = inet_csk(sk); 3806 struct net *net = sock_net(sk); 3807 int val; 3808 int err = 0; 3809 3810 /* These are data/string values, all the others are ints */ 3811 switch (optname) { 3812 case TCP_CONGESTION: { 3813 char name[TCP_CA_NAME_MAX]; 3814 3815 if (optlen < 1) 3816 return -EINVAL; 3817 3818 val = strncpy_from_sockptr(name, optval, 3819 min_t(long, TCP_CA_NAME_MAX-1, optlen)); 3820 if (val < 0) 3821 return -EFAULT; 3822 name[val] = 0; 3823 3824 sockopt_lock_sock(sk); 3825 err = tcp_set_congestion_control(sk, name, !has_current_bpf_ctx(), 3826 sockopt_ns_capable(sock_net(sk)->user_ns, 3827 CAP_NET_ADMIN)); 3828 sockopt_release_sock(sk); 3829 return err; 3830 } 3831 case TCP_ULP: { 3832 char name[TCP_ULP_NAME_MAX]; 3833 3834 if (optlen < 1) 3835 return -EINVAL; 3836 3837 val = strncpy_from_sockptr(name, optval, 3838 min_t(long, TCP_ULP_NAME_MAX - 1, 3839 optlen)); 3840 if (val < 0) 3841 return -EFAULT; 3842 name[val] = 0; 3843 3844 sockopt_lock_sock(sk); 3845 err = tcp_set_ulp(sk, name); 3846 sockopt_release_sock(sk); 3847 return err; 3848 } 3849 case TCP_FASTOPEN_KEY: { 3850 __u8 key[TCP_FASTOPEN_KEY_BUF_LENGTH]; 3851 __u8 *backup_key = NULL; 3852 3853 /* Allow a backup key as well to facilitate key rotation 3854 * First key is the active one. 3855 */ 3856 if (optlen != TCP_FASTOPEN_KEY_LENGTH && 3857 optlen != TCP_FASTOPEN_KEY_BUF_LENGTH) 3858 return -EINVAL; 3859 3860 if (copy_from_sockptr(key, optval, optlen)) 3861 return -EFAULT; 3862 3863 if (optlen == TCP_FASTOPEN_KEY_BUF_LENGTH) 3864 backup_key = key + TCP_FASTOPEN_KEY_LENGTH; 3865 3866 return tcp_fastopen_reset_cipher(net, sk, key, backup_key); 3867 } 3868 default: 3869 /* fallthru */ 3870 break; 3871 } 3872 3873 if (optlen < sizeof(int)) 3874 return -EINVAL; 3875 3876 if (copy_from_sockptr(&val, optval, sizeof(val))) 3877 return -EFAULT; 3878 3879 /* Handle options that can be set without locking the socket. */ 3880 switch (optname) { 3881 case TCP_SYNCNT: 3882 return tcp_sock_set_syncnt(sk, val); 3883 case TCP_USER_TIMEOUT: 3884 return tcp_sock_set_user_timeout(sk, val); 3885 case TCP_KEEPINTVL: 3886 return tcp_sock_set_keepintvl(sk, val); 3887 case TCP_KEEPCNT: 3888 return tcp_sock_set_keepcnt(sk, val); 3889 case TCP_LINGER2: 3890 if (val < 0) 3891 WRITE_ONCE(tp->linger2, -1); 3892 else if (val > TCP_FIN_TIMEOUT_MAX / HZ) 3893 WRITE_ONCE(tp->linger2, TCP_FIN_TIMEOUT_MAX); 3894 else 3895 WRITE_ONCE(tp->linger2, val * HZ); 3896 return 0; 3897 case TCP_DEFER_ACCEPT: 3898 /* Translate value in seconds to number of retransmits */ 3899 WRITE_ONCE(icsk->icsk_accept_queue.rskq_defer_accept, 3900 secs_to_retrans(val, TCP_TIMEOUT_INIT / HZ, 3901 TCP_RTO_MAX / HZ)); 3902 return 0; 3903 case TCP_RTO_MAX_MS: 3904 if (val < MSEC_PER_SEC || val > TCP_RTO_MAX_SEC * MSEC_PER_SEC) 3905 return -EINVAL; 3906 WRITE_ONCE(inet_csk(sk)->icsk_rto_max, msecs_to_jiffies(val)); 3907 return 0; 3908 case TCP_RTO_MIN_US: { 3909 int rto_min = usecs_to_jiffies(val); 3910 3911 if (rto_min > TCP_RTO_MIN || rto_min < TCP_TIMEOUT_MIN) 3912 return -EINVAL; 3913 WRITE_ONCE(inet_csk(sk)->icsk_rto_min, rto_min); 3914 return 0; 3915 } 3916 case TCP_DELACK_MAX_US: { 3917 int delack_max = usecs_to_jiffies(val); 3918 3919 if (delack_max > TCP_DELACK_MAX || delack_max < TCP_TIMEOUT_MIN) 3920 return -EINVAL; 3921 WRITE_ONCE(inet_csk(sk)->icsk_delack_max, delack_max); 3922 return 0; 3923 } 3924 case TCP_MAXSEG: 3925 return tcp_sock_set_maxseg(sk, val); 3926 } 3927 3928 sockopt_lock_sock(sk); 3929 3930 switch (optname) { 3931 case TCP_NODELAY: 3932 __tcp_sock_set_nodelay(sk, val); 3933 break; 3934 3935 case TCP_THIN_LINEAR_TIMEOUTS: 3936 if (val < 0 || val > 1) 3937 err = -EINVAL; 3938 else 3939 tp->thin_lto = val; 3940 break; 3941 3942 case TCP_THIN_DUPACK: 3943 if (val < 0 || val > 1) 3944 err = -EINVAL; 3945 break; 3946 3947 case TCP_REPAIR: 3948 if (!tcp_can_repair_sock(sk)) 3949 err = -EPERM; 3950 else if (val == TCP_REPAIR_ON) { 3951 tp->repair = 1; 3952 sk->sk_reuse = SK_FORCE_REUSE; 3953 tp->repair_queue = TCP_NO_QUEUE; 3954 } else if (val == TCP_REPAIR_OFF) { 3955 tp->repair = 0; 3956 sk->sk_reuse = SK_NO_REUSE; 3957 tcp_send_window_probe(sk); 3958 } else if (val == TCP_REPAIR_OFF_NO_WP) { 3959 tp->repair = 0; 3960 sk->sk_reuse = SK_NO_REUSE; 3961 } else 3962 err = -EINVAL; 3963 3964 break; 3965 3966 case TCP_REPAIR_QUEUE: 3967 if (!tp->repair) 3968 err = -EPERM; 3969 else if ((unsigned int)val < TCP_QUEUES_NR) 3970 tp->repair_queue = val; 3971 else 3972 err = -EINVAL; 3973 break; 3974 3975 case TCP_QUEUE_SEQ: 3976 if (sk->sk_state != TCP_CLOSE) { 3977 err = -EPERM; 3978 } else if (tp->repair_queue == TCP_SEND_QUEUE) { 3979 if (!tcp_rtx_queue_empty(sk)) 3980 err = -EPERM; 3981 else 3982 WRITE_ONCE(tp->write_seq, val); 3983 } else if (tp->repair_queue == TCP_RECV_QUEUE) { 3984 if (tp->rcv_nxt != tp->copied_seq) { 3985 err = -EPERM; 3986 } else { 3987 WRITE_ONCE(tp->rcv_nxt, val); 3988 WRITE_ONCE(tp->copied_seq, val); 3989 } 3990 } else { 3991 err = -EINVAL; 3992 } 3993 break; 3994 3995 case TCP_REPAIR_OPTIONS: 3996 if (!tp->repair) 3997 err = -EINVAL; 3998 else if (sk->sk_state == TCP_ESTABLISHED && !tp->bytes_sent) 3999 err = tcp_repair_options_est(sk, optval, optlen); 4000 else 4001 err = -EPERM; 4002 break; 4003 4004 case TCP_CORK: 4005 __tcp_sock_set_cork(sk, val); 4006 break; 4007 4008 case TCP_KEEPIDLE: 4009 err = tcp_sock_set_keepidle_locked(sk, val); 4010 break; 4011 case TCP_SAVE_SYN: 4012 /* 0: disable, 1: enable, 2: start from ether_header */ 4013 if (val < 0 || val > 2) 4014 err = -EINVAL; 4015 else 4016 tp->save_syn = val; 4017 break; 4018 4019 case TCP_WINDOW_CLAMP: 4020 err = tcp_set_window_clamp(sk, val); 4021 break; 4022 4023 case TCP_QUICKACK: 4024 __tcp_sock_set_quickack(sk, val); 4025 break; 4026 4027 case TCP_AO_REPAIR: 4028 if (!tcp_can_repair_sock(sk)) { 4029 err = -EPERM; 4030 break; 4031 } 4032 err = tcp_ao_set_repair(sk, optval, optlen); 4033 break; 4034 #ifdef CONFIG_TCP_AO 4035 case TCP_AO_ADD_KEY: 4036 case TCP_AO_DEL_KEY: 4037 case TCP_AO_INFO: { 4038 /* If this is the first TCP-AO setsockopt() on the socket, 4039 * sk_state has to be LISTEN or CLOSE. Allow TCP_REPAIR 4040 * in any state. 4041 */ 4042 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) 4043 goto ao_parse; 4044 if (rcu_dereference_protected(tcp_sk(sk)->ao_info, 4045 lockdep_sock_is_held(sk))) 4046 goto ao_parse; 4047 if (tp->repair) 4048 goto ao_parse; 4049 err = -EISCONN; 4050 break; 4051 ao_parse: 4052 err = tp->af_specific->ao_parse(sk, optname, optval, optlen); 4053 break; 4054 } 4055 #endif 4056 #ifdef CONFIG_TCP_MD5SIG 4057 case TCP_MD5SIG: 4058 case TCP_MD5SIG_EXT: 4059 err = tp->af_specific->md5_parse(sk, optname, optval, optlen); 4060 break; 4061 #endif 4062 case TCP_FASTOPEN: 4063 if (val >= 0 && ((1 << sk->sk_state) & (TCPF_CLOSE | 4064 TCPF_LISTEN))) { 4065 tcp_fastopen_init_key_once(net); 4066 4067 fastopen_queue_tune(sk, val); 4068 } else { 4069 err = -EINVAL; 4070 } 4071 break; 4072 case TCP_FASTOPEN_CONNECT: 4073 if (val > 1 || val < 0) { 4074 err = -EINVAL; 4075 } else if (READ_ONCE(net->ipv4.sysctl_tcp_fastopen) & 4076 TFO_CLIENT_ENABLE) { 4077 if (sk->sk_state == TCP_CLOSE) 4078 tp->fastopen_connect = val; 4079 else 4080 err = -EINVAL; 4081 } else { 4082 err = -EOPNOTSUPP; 4083 } 4084 break; 4085 case TCP_FASTOPEN_NO_COOKIE: 4086 if (val > 1 || val < 0) 4087 err = -EINVAL; 4088 else if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) 4089 err = -EINVAL; 4090 else 4091 tp->fastopen_no_cookie = val; 4092 break; 4093 case TCP_TIMESTAMP: 4094 if (!tp->repair) { 4095 err = -EPERM; 4096 break; 4097 } 4098 /* val is an opaque field, 4099 * and low order bit contains usec_ts enable bit. 4100 * Its a best effort, and we do not care if user makes an error. 4101 */ 4102 tp->tcp_usec_ts = val & 1; 4103 WRITE_ONCE(tp->tsoffset, val - tcp_clock_ts(tp->tcp_usec_ts)); 4104 break; 4105 case TCP_REPAIR_WINDOW: 4106 err = tcp_repair_set_window(tp, optval, optlen); 4107 break; 4108 case TCP_NOTSENT_LOWAT: 4109 WRITE_ONCE(tp->notsent_lowat, val); 4110 sk->sk_write_space(sk); 4111 break; 4112 case TCP_INQ: 4113 if (val > 1 || val < 0) 4114 err = -EINVAL; 4115 else 4116 tp->recvmsg_inq = val; 4117 break; 4118 case TCP_TX_DELAY: 4119 if (val) 4120 tcp_enable_tx_delay(); 4121 WRITE_ONCE(tp->tcp_tx_delay, val); 4122 break; 4123 default: 4124 err = -ENOPROTOOPT; 4125 break; 4126 } 4127 4128 sockopt_release_sock(sk); 4129 return err; 4130 } 4131 4132 int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, 4133 unsigned int optlen) 4134 { 4135 const struct inet_connection_sock *icsk = inet_csk(sk); 4136 4137 if (level != SOL_TCP) 4138 /* Paired with WRITE_ONCE() in do_ipv6_setsockopt() and tcp_v6_connect() */ 4139 return READ_ONCE(icsk->icsk_af_ops)->setsockopt(sk, level, optname, 4140 optval, optlen); 4141 return do_tcp_setsockopt(sk, level, optname, optval, optlen); 4142 } 4143 EXPORT_IPV6_MOD(tcp_setsockopt); 4144 4145 static void tcp_get_info_chrono_stats(const struct tcp_sock *tp, 4146 struct tcp_info *info) 4147 { 4148 u64 stats[__TCP_CHRONO_MAX], total = 0; 4149 enum tcp_chrono i; 4150 4151 for (i = TCP_CHRONO_BUSY; i < __TCP_CHRONO_MAX; ++i) { 4152 stats[i] = tp->chrono_stat[i - 1]; 4153 if (i == tp->chrono_type) 4154 stats[i] += tcp_jiffies32 - tp->chrono_start; 4155 stats[i] *= USEC_PER_SEC / HZ; 4156 total += stats[i]; 4157 } 4158 4159 info->tcpi_busy_time = total; 4160 info->tcpi_rwnd_limited = stats[TCP_CHRONO_RWND_LIMITED]; 4161 info->tcpi_sndbuf_limited = stats[TCP_CHRONO_SNDBUF_LIMITED]; 4162 } 4163 4164 /* Return information about state of tcp endpoint in API format. */ 4165 void tcp_get_info(struct sock *sk, struct tcp_info *info) 4166 { 4167 const struct tcp_sock *tp = tcp_sk(sk); /* iff sk_type == SOCK_STREAM */ 4168 const struct inet_connection_sock *icsk = inet_csk(sk); 4169 const u8 ect1_idx = INET_ECN_ECT_1 - 1; 4170 const u8 ect0_idx = INET_ECN_ECT_0 - 1; 4171 const u8 ce_idx = INET_ECN_CE - 1; 4172 unsigned long rate; 4173 u32 now; 4174 u64 rate64; 4175 bool slow; 4176 4177 memset(info, 0, sizeof(*info)); 4178 if (sk->sk_type != SOCK_STREAM) 4179 return; 4180 4181 info->tcpi_state = inet_sk_state_load(sk); 4182 4183 /* Report meaningful fields for all TCP states, including listeners */ 4184 rate = READ_ONCE(sk->sk_pacing_rate); 4185 rate64 = (rate != ~0UL) ? rate : ~0ULL; 4186 info->tcpi_pacing_rate = rate64; 4187 4188 rate = READ_ONCE(sk->sk_max_pacing_rate); 4189 rate64 = (rate != ~0UL) ? rate : ~0ULL; 4190 info->tcpi_max_pacing_rate = rate64; 4191 4192 info->tcpi_reordering = tp->reordering; 4193 info->tcpi_snd_cwnd = tcp_snd_cwnd(tp); 4194 4195 if (info->tcpi_state == TCP_LISTEN) { 4196 /* listeners aliased fields : 4197 * tcpi_unacked -> Number of children ready for accept() 4198 * tcpi_sacked -> max backlog 4199 */ 4200 info->tcpi_unacked = READ_ONCE(sk->sk_ack_backlog); 4201 info->tcpi_sacked = READ_ONCE(sk->sk_max_ack_backlog); 4202 return; 4203 } 4204 4205 slow = lock_sock_fast(sk); 4206 4207 info->tcpi_ca_state = icsk->icsk_ca_state; 4208 info->tcpi_retransmits = icsk->icsk_retransmits; 4209 info->tcpi_probes = icsk->icsk_probes_out; 4210 info->tcpi_backoff = icsk->icsk_backoff; 4211 4212 if (tp->rx_opt.tstamp_ok) 4213 info->tcpi_options |= TCPI_OPT_TIMESTAMPS; 4214 if (tcp_is_sack(tp)) 4215 info->tcpi_options |= TCPI_OPT_SACK; 4216 if (tp->rx_opt.wscale_ok) { 4217 info->tcpi_options |= TCPI_OPT_WSCALE; 4218 info->tcpi_snd_wscale = tp->rx_opt.snd_wscale; 4219 info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale; 4220 } 4221 4222 if (tcp_ecn_mode_any(tp)) 4223 info->tcpi_options |= TCPI_OPT_ECN; 4224 if (tp->ecn_flags & TCP_ECN_SEEN) 4225 info->tcpi_options |= TCPI_OPT_ECN_SEEN; 4226 if (tp->syn_data_acked) 4227 info->tcpi_options |= TCPI_OPT_SYN_DATA; 4228 if (tp->tcp_usec_ts) 4229 info->tcpi_options |= TCPI_OPT_USEC_TS; 4230 if (tp->syn_fastopen_child) 4231 info->tcpi_options |= TCPI_OPT_TFO_CHILD; 4232 4233 info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto); 4234 info->tcpi_ato = jiffies_to_usecs(min_t(u32, icsk->icsk_ack.ato, 4235 tcp_delack_max(sk))); 4236 info->tcpi_snd_mss = tp->mss_cache; 4237 info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss; 4238 4239 info->tcpi_unacked = tp->packets_out; 4240 info->tcpi_sacked = tp->sacked_out; 4241 4242 info->tcpi_lost = tp->lost_out; 4243 info->tcpi_retrans = tp->retrans_out; 4244 4245 now = tcp_jiffies32; 4246 info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime); 4247 info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime); 4248 info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp); 4249 4250 info->tcpi_pmtu = icsk->icsk_pmtu_cookie; 4251 info->tcpi_rcv_ssthresh = tp->rcv_ssthresh; 4252 info->tcpi_rtt = tp->srtt_us >> 3; 4253 info->tcpi_rttvar = tp->mdev_us >> 2; 4254 info->tcpi_snd_ssthresh = tp->snd_ssthresh; 4255 info->tcpi_advmss = tp->advmss; 4256 4257 info->tcpi_rcv_rtt = tp->rcv_rtt_est.rtt_us >> 3; 4258 info->tcpi_rcv_space = tp->rcvq_space.space; 4259 4260 info->tcpi_total_retrans = tp->total_retrans; 4261 4262 info->tcpi_bytes_acked = tp->bytes_acked; 4263 info->tcpi_bytes_received = tp->bytes_received; 4264 info->tcpi_notsent_bytes = max_t(int, 0, tp->write_seq - tp->snd_nxt); 4265 tcp_get_info_chrono_stats(tp, info); 4266 4267 info->tcpi_segs_out = tp->segs_out; 4268 4269 /* segs_in and data_segs_in can be updated from tcp_segs_in() from BH */ 4270 info->tcpi_segs_in = READ_ONCE(tp->segs_in); 4271 info->tcpi_data_segs_in = READ_ONCE(tp->data_segs_in); 4272 4273 info->tcpi_min_rtt = tcp_min_rtt(tp); 4274 info->tcpi_data_segs_out = tp->data_segs_out; 4275 4276 info->tcpi_delivery_rate_app_limited = tp->rate_app_limited ? 1 : 0; 4277 rate64 = tcp_compute_delivery_rate(tp); 4278 if (rate64) 4279 info->tcpi_delivery_rate = rate64; 4280 info->tcpi_delivered = tp->delivered; 4281 info->tcpi_delivered_ce = tp->delivered_ce; 4282 info->tcpi_bytes_sent = tp->bytes_sent; 4283 info->tcpi_bytes_retrans = tp->bytes_retrans; 4284 info->tcpi_dsack_dups = tp->dsack_dups; 4285 info->tcpi_reord_seen = tp->reord_seen; 4286 info->tcpi_rcv_ooopack = tp->rcv_ooopack; 4287 info->tcpi_snd_wnd = tp->snd_wnd; 4288 info->tcpi_rcv_wnd = tp->rcv_wnd; 4289 info->tcpi_rehash = tp->plb_rehash + tp->timeout_rehash; 4290 info->tcpi_fastopen_client_fail = tp->fastopen_client_fail; 4291 4292 info->tcpi_total_rto = tp->total_rto; 4293 info->tcpi_total_rto_recoveries = tp->total_rto_recoveries; 4294 info->tcpi_total_rto_time = tp->total_rto_time; 4295 if (tp->rto_stamp) 4296 info->tcpi_total_rto_time += tcp_clock_ms() - tp->rto_stamp; 4297 4298 info->tcpi_accecn_fail_mode = tp->accecn_fail_mode; 4299 info->tcpi_accecn_opt_seen = tp->saw_accecn_opt; 4300 info->tcpi_received_ce = tp->received_ce; 4301 info->tcpi_delivered_e1_bytes = tp->delivered_ecn_bytes[ect1_idx]; 4302 info->tcpi_delivered_e0_bytes = tp->delivered_ecn_bytes[ect0_idx]; 4303 info->tcpi_delivered_ce_bytes = tp->delivered_ecn_bytes[ce_idx]; 4304 info->tcpi_received_e1_bytes = tp->received_ecn_bytes[ect1_idx]; 4305 info->tcpi_received_e0_bytes = tp->received_ecn_bytes[ect0_idx]; 4306 info->tcpi_received_ce_bytes = tp->received_ecn_bytes[ce_idx]; 4307 4308 unlock_sock_fast(sk, slow); 4309 } 4310 EXPORT_SYMBOL_GPL(tcp_get_info); 4311 4312 static size_t tcp_opt_stats_get_size(void) 4313 { 4314 return 4315 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BUSY */ 4316 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_RWND_LIMITED */ 4317 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_SNDBUF_LIMITED */ 4318 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_DATA_SEGS_OUT */ 4319 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_TOTAL_RETRANS */ 4320 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_PACING_RATE */ 4321 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_DELIVERY_RATE */ 4322 nla_total_size(sizeof(u32)) + /* TCP_NLA_SND_CWND */ 4323 nla_total_size(sizeof(u32)) + /* TCP_NLA_REORDERING */ 4324 nla_total_size(sizeof(u32)) + /* TCP_NLA_MIN_RTT */ 4325 nla_total_size(sizeof(u8)) + /* TCP_NLA_RECUR_RETRANS */ 4326 nla_total_size(sizeof(u8)) + /* TCP_NLA_DELIVERY_RATE_APP_LMT */ 4327 nla_total_size(sizeof(u32)) + /* TCP_NLA_SNDQ_SIZE */ 4328 nla_total_size(sizeof(u8)) + /* TCP_NLA_CA_STATE */ 4329 nla_total_size(sizeof(u32)) + /* TCP_NLA_SND_SSTHRESH */ 4330 nla_total_size(sizeof(u32)) + /* TCP_NLA_DELIVERED */ 4331 nla_total_size(sizeof(u32)) + /* TCP_NLA_DELIVERED_CE */ 4332 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BYTES_SENT */ 4333 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BYTES_RETRANS */ 4334 nla_total_size(sizeof(u32)) + /* TCP_NLA_DSACK_DUPS */ 4335 nla_total_size(sizeof(u32)) + /* TCP_NLA_REORD_SEEN */ 4336 nla_total_size(sizeof(u32)) + /* TCP_NLA_SRTT */ 4337 nla_total_size(sizeof(u16)) + /* TCP_NLA_TIMEOUT_REHASH */ 4338 nla_total_size(sizeof(u32)) + /* TCP_NLA_BYTES_NOTSENT */ 4339 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_EDT */ 4340 nla_total_size(sizeof(u8)) + /* TCP_NLA_TTL */ 4341 nla_total_size(sizeof(u32)) + /* TCP_NLA_REHASH */ 4342 0; 4343 } 4344 4345 /* Returns TTL or hop limit of an incoming packet from skb. */ 4346 static u8 tcp_skb_ttl_or_hop_limit(const struct sk_buff *skb) 4347 { 4348 if (skb->protocol == htons(ETH_P_IP)) 4349 return ip_hdr(skb)->ttl; 4350 else if (skb->protocol == htons(ETH_P_IPV6)) 4351 return ipv6_hdr(skb)->hop_limit; 4352 else 4353 return 0; 4354 } 4355 4356 struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk, 4357 const struct sk_buff *orig_skb, 4358 const struct sk_buff *ack_skb) 4359 { 4360 const struct tcp_sock *tp = tcp_sk(sk); 4361 struct sk_buff *stats; 4362 struct tcp_info info; 4363 unsigned long rate; 4364 u64 rate64; 4365 4366 stats = alloc_skb(tcp_opt_stats_get_size(), GFP_ATOMIC); 4367 if (!stats) 4368 return NULL; 4369 4370 tcp_get_info_chrono_stats(tp, &info); 4371 nla_put_u64_64bit(stats, TCP_NLA_BUSY, 4372 info.tcpi_busy_time, TCP_NLA_PAD); 4373 nla_put_u64_64bit(stats, TCP_NLA_RWND_LIMITED, 4374 info.tcpi_rwnd_limited, TCP_NLA_PAD); 4375 nla_put_u64_64bit(stats, TCP_NLA_SNDBUF_LIMITED, 4376 info.tcpi_sndbuf_limited, TCP_NLA_PAD); 4377 nla_put_u64_64bit(stats, TCP_NLA_DATA_SEGS_OUT, 4378 tp->data_segs_out, TCP_NLA_PAD); 4379 nla_put_u64_64bit(stats, TCP_NLA_TOTAL_RETRANS, 4380 tp->total_retrans, TCP_NLA_PAD); 4381 4382 rate = READ_ONCE(sk->sk_pacing_rate); 4383 rate64 = (rate != ~0UL) ? rate : ~0ULL; 4384 nla_put_u64_64bit(stats, TCP_NLA_PACING_RATE, rate64, TCP_NLA_PAD); 4385 4386 rate64 = tcp_compute_delivery_rate(tp); 4387 nla_put_u64_64bit(stats, TCP_NLA_DELIVERY_RATE, rate64, TCP_NLA_PAD); 4388 4389 nla_put_u32(stats, TCP_NLA_SND_CWND, tcp_snd_cwnd(tp)); 4390 nla_put_u32(stats, TCP_NLA_REORDERING, tp->reordering); 4391 nla_put_u32(stats, TCP_NLA_MIN_RTT, tcp_min_rtt(tp)); 4392 4393 nla_put_u8(stats, TCP_NLA_RECUR_RETRANS, 4394 READ_ONCE(inet_csk(sk)->icsk_retransmits)); 4395 nla_put_u8(stats, TCP_NLA_DELIVERY_RATE_APP_LMT, !!tp->rate_app_limited); 4396 nla_put_u32(stats, TCP_NLA_SND_SSTHRESH, tp->snd_ssthresh); 4397 nla_put_u32(stats, TCP_NLA_DELIVERED, tp->delivered); 4398 nla_put_u32(stats, TCP_NLA_DELIVERED_CE, tp->delivered_ce); 4399 4400 nla_put_u32(stats, TCP_NLA_SNDQ_SIZE, tp->write_seq - tp->snd_una); 4401 nla_put_u8(stats, TCP_NLA_CA_STATE, inet_csk(sk)->icsk_ca_state); 4402 4403 nla_put_u64_64bit(stats, TCP_NLA_BYTES_SENT, tp->bytes_sent, 4404 TCP_NLA_PAD); 4405 nla_put_u64_64bit(stats, TCP_NLA_BYTES_RETRANS, tp->bytes_retrans, 4406 TCP_NLA_PAD); 4407 nla_put_u32(stats, TCP_NLA_DSACK_DUPS, tp->dsack_dups); 4408 nla_put_u32(stats, TCP_NLA_REORD_SEEN, tp->reord_seen); 4409 nla_put_u32(stats, TCP_NLA_SRTT, tp->srtt_us >> 3); 4410 nla_put_u16(stats, TCP_NLA_TIMEOUT_REHASH, tp->timeout_rehash); 4411 nla_put_u32(stats, TCP_NLA_BYTES_NOTSENT, 4412 max_t(int, 0, tp->write_seq - tp->snd_nxt)); 4413 nla_put_u64_64bit(stats, TCP_NLA_EDT, orig_skb->skb_mstamp_ns, 4414 TCP_NLA_PAD); 4415 if (ack_skb) 4416 nla_put_u8(stats, TCP_NLA_TTL, 4417 tcp_skb_ttl_or_hop_limit(ack_skb)); 4418 4419 nla_put_u32(stats, TCP_NLA_REHASH, tp->plb_rehash + tp->timeout_rehash); 4420 return stats; 4421 } 4422 4423 int do_tcp_getsockopt(struct sock *sk, int level, 4424 int optname, sockptr_t optval, sockptr_t optlen) 4425 { 4426 struct inet_connection_sock *icsk = inet_csk(sk); 4427 struct tcp_sock *tp = tcp_sk(sk); 4428 struct net *net = sock_net(sk); 4429 int user_mss; 4430 int val, len; 4431 4432 if (copy_from_sockptr(&len, optlen, sizeof(int))) 4433 return -EFAULT; 4434 4435 if (len < 0) 4436 return -EINVAL; 4437 4438 len = min_t(unsigned int, len, sizeof(int)); 4439 4440 switch (optname) { 4441 case TCP_MAXSEG: 4442 val = tp->mss_cache; 4443 user_mss = READ_ONCE(tp->rx_opt.user_mss); 4444 if (user_mss && 4445 ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) 4446 val = user_mss; 4447 if (tp->repair) 4448 val = tp->rx_opt.mss_clamp; 4449 break; 4450 case TCP_NODELAY: 4451 val = !!(tp->nonagle&TCP_NAGLE_OFF); 4452 break; 4453 case TCP_CORK: 4454 val = !!(tp->nonagle&TCP_NAGLE_CORK); 4455 break; 4456 case TCP_KEEPIDLE: 4457 val = keepalive_time_when(tp) / HZ; 4458 break; 4459 case TCP_KEEPINTVL: 4460 val = keepalive_intvl_when(tp) / HZ; 4461 break; 4462 case TCP_KEEPCNT: 4463 val = keepalive_probes(tp); 4464 break; 4465 case TCP_SYNCNT: 4466 val = READ_ONCE(icsk->icsk_syn_retries) ? : 4467 READ_ONCE(net->ipv4.sysctl_tcp_syn_retries); 4468 break; 4469 case TCP_LINGER2: 4470 val = READ_ONCE(tp->linger2); 4471 if (val >= 0) 4472 val = (val ? : READ_ONCE(net->ipv4.sysctl_tcp_fin_timeout)) / HZ; 4473 break; 4474 case TCP_DEFER_ACCEPT: 4475 val = READ_ONCE(icsk->icsk_accept_queue.rskq_defer_accept); 4476 val = retrans_to_secs(val, TCP_TIMEOUT_INIT / HZ, 4477 TCP_RTO_MAX / HZ); 4478 break; 4479 case TCP_WINDOW_CLAMP: 4480 val = READ_ONCE(tp->window_clamp); 4481 break; 4482 case TCP_INFO: { 4483 struct tcp_info info; 4484 4485 if (copy_from_sockptr(&len, optlen, sizeof(int))) 4486 return -EFAULT; 4487 4488 tcp_get_info(sk, &info); 4489 4490 len = min_t(unsigned int, len, sizeof(info)); 4491 if (copy_to_sockptr(optlen, &len, sizeof(int))) 4492 return -EFAULT; 4493 if (copy_to_sockptr(optval, &info, len)) 4494 return -EFAULT; 4495 return 0; 4496 } 4497 case TCP_CC_INFO: { 4498 const struct tcp_congestion_ops *ca_ops; 4499 union tcp_cc_info info; 4500 size_t sz = 0; 4501 int attr; 4502 4503 if (copy_from_sockptr(&len, optlen, sizeof(int))) 4504 return -EFAULT; 4505 4506 ca_ops = icsk->icsk_ca_ops; 4507 if (ca_ops && ca_ops->get_info) 4508 sz = ca_ops->get_info(sk, ~0U, &attr, &info); 4509 4510 len = min_t(unsigned int, len, sz); 4511 if (copy_to_sockptr(optlen, &len, sizeof(int))) 4512 return -EFAULT; 4513 if (copy_to_sockptr(optval, &info, len)) 4514 return -EFAULT; 4515 return 0; 4516 } 4517 case TCP_QUICKACK: 4518 val = !inet_csk_in_pingpong_mode(sk); 4519 break; 4520 4521 case TCP_CONGESTION: 4522 if (copy_from_sockptr(&len, optlen, sizeof(int))) 4523 return -EFAULT; 4524 len = min_t(unsigned int, len, TCP_CA_NAME_MAX); 4525 if (copy_to_sockptr(optlen, &len, sizeof(int))) 4526 return -EFAULT; 4527 if (copy_to_sockptr(optval, icsk->icsk_ca_ops->name, len)) 4528 return -EFAULT; 4529 return 0; 4530 4531 case TCP_ULP: 4532 if (copy_from_sockptr(&len, optlen, sizeof(int))) 4533 return -EFAULT; 4534 len = min_t(unsigned int, len, TCP_ULP_NAME_MAX); 4535 if (!icsk->icsk_ulp_ops) { 4536 len = 0; 4537 if (copy_to_sockptr(optlen, &len, sizeof(int))) 4538 return -EFAULT; 4539 return 0; 4540 } 4541 if (copy_to_sockptr(optlen, &len, sizeof(int))) 4542 return -EFAULT; 4543 if (copy_to_sockptr(optval, icsk->icsk_ulp_ops->name, len)) 4544 return -EFAULT; 4545 return 0; 4546 4547 case TCP_FASTOPEN_KEY: { 4548 u64 key[TCP_FASTOPEN_KEY_BUF_LENGTH / sizeof(u64)]; 4549 unsigned int key_len; 4550 4551 if (copy_from_sockptr(&len, optlen, sizeof(int))) 4552 return -EFAULT; 4553 4554 key_len = tcp_fastopen_get_cipher(net, icsk, key) * 4555 TCP_FASTOPEN_KEY_LENGTH; 4556 len = min_t(unsigned int, len, key_len); 4557 if (copy_to_sockptr(optlen, &len, sizeof(int))) 4558 return -EFAULT; 4559 if (copy_to_sockptr(optval, key, len)) 4560 return -EFAULT; 4561 return 0; 4562 } 4563 case TCP_THIN_LINEAR_TIMEOUTS: 4564 val = tp->thin_lto; 4565 break; 4566 4567 case TCP_THIN_DUPACK: 4568 val = 0; 4569 break; 4570 4571 case TCP_REPAIR: 4572 val = tp->repair; 4573 break; 4574 4575 case TCP_REPAIR_QUEUE: 4576 if (tp->repair) 4577 val = tp->repair_queue; 4578 else 4579 return -EINVAL; 4580 break; 4581 4582 case TCP_REPAIR_WINDOW: { 4583 struct tcp_repair_window opt; 4584 4585 if (copy_from_sockptr(&len, optlen, sizeof(int))) 4586 return -EFAULT; 4587 4588 if (len != sizeof(opt)) 4589 return -EINVAL; 4590 4591 if (!tp->repair) 4592 return -EPERM; 4593 4594 opt.snd_wl1 = tp->snd_wl1; 4595 opt.snd_wnd = tp->snd_wnd; 4596 opt.max_window = tp->max_window; 4597 opt.rcv_wnd = tp->rcv_wnd; 4598 opt.rcv_wup = tp->rcv_wup; 4599 4600 if (copy_to_sockptr(optval, &opt, len)) 4601 return -EFAULT; 4602 return 0; 4603 } 4604 case TCP_QUEUE_SEQ: 4605 if (tp->repair_queue == TCP_SEND_QUEUE) 4606 val = tp->write_seq; 4607 else if (tp->repair_queue == TCP_RECV_QUEUE) 4608 val = tp->rcv_nxt; 4609 else 4610 return -EINVAL; 4611 break; 4612 4613 case TCP_USER_TIMEOUT: 4614 val = READ_ONCE(icsk->icsk_user_timeout); 4615 break; 4616 4617 case TCP_FASTOPEN: 4618 val = READ_ONCE(icsk->icsk_accept_queue.fastopenq.max_qlen); 4619 break; 4620 4621 case TCP_FASTOPEN_CONNECT: 4622 val = tp->fastopen_connect; 4623 break; 4624 4625 case TCP_FASTOPEN_NO_COOKIE: 4626 val = tp->fastopen_no_cookie; 4627 break; 4628 4629 case TCP_TX_DELAY: 4630 val = READ_ONCE(tp->tcp_tx_delay); 4631 break; 4632 4633 case TCP_TIMESTAMP: 4634 val = tcp_clock_ts(tp->tcp_usec_ts) + READ_ONCE(tp->tsoffset); 4635 if (tp->tcp_usec_ts) 4636 val |= 1; 4637 else 4638 val &= ~1; 4639 break; 4640 case TCP_NOTSENT_LOWAT: 4641 val = READ_ONCE(tp->notsent_lowat); 4642 break; 4643 case TCP_INQ: 4644 val = tp->recvmsg_inq; 4645 break; 4646 case TCP_SAVE_SYN: 4647 val = tp->save_syn; 4648 break; 4649 case TCP_SAVED_SYN: { 4650 if (copy_from_sockptr(&len, optlen, sizeof(int))) 4651 return -EFAULT; 4652 4653 sockopt_lock_sock(sk); 4654 if (tp->saved_syn) { 4655 if (len < tcp_saved_syn_len(tp->saved_syn)) { 4656 len = tcp_saved_syn_len(tp->saved_syn); 4657 if (copy_to_sockptr(optlen, &len, sizeof(int))) { 4658 sockopt_release_sock(sk); 4659 return -EFAULT; 4660 } 4661 sockopt_release_sock(sk); 4662 return -EINVAL; 4663 } 4664 len = tcp_saved_syn_len(tp->saved_syn); 4665 if (copy_to_sockptr(optlen, &len, sizeof(int))) { 4666 sockopt_release_sock(sk); 4667 return -EFAULT; 4668 } 4669 if (copy_to_sockptr(optval, tp->saved_syn->data, len)) { 4670 sockopt_release_sock(sk); 4671 return -EFAULT; 4672 } 4673 tcp_saved_syn_free(tp); 4674 sockopt_release_sock(sk); 4675 } else { 4676 sockopt_release_sock(sk); 4677 len = 0; 4678 if (copy_to_sockptr(optlen, &len, sizeof(int))) 4679 return -EFAULT; 4680 } 4681 return 0; 4682 } 4683 #ifdef CONFIG_MMU 4684 case TCP_ZEROCOPY_RECEIVE: { 4685 struct scm_timestamping_internal tss; 4686 struct tcp_zerocopy_receive zc = {}; 4687 int err; 4688 4689 if (copy_from_sockptr(&len, optlen, sizeof(int))) 4690 return -EFAULT; 4691 if (len < 0 || 4692 len < offsetofend(struct tcp_zerocopy_receive, length)) 4693 return -EINVAL; 4694 if (unlikely(len > sizeof(zc))) { 4695 err = check_zeroed_sockptr(optval, sizeof(zc), 4696 len - sizeof(zc)); 4697 if (err < 1) 4698 return err == 0 ? -EINVAL : err; 4699 len = sizeof(zc); 4700 if (copy_to_sockptr(optlen, &len, sizeof(int))) 4701 return -EFAULT; 4702 } 4703 if (copy_from_sockptr(&zc, optval, len)) 4704 return -EFAULT; 4705 if (zc.reserved) 4706 return -EINVAL; 4707 if (zc.msg_flags & ~(TCP_VALID_ZC_MSG_FLAGS)) 4708 return -EINVAL; 4709 sockopt_lock_sock(sk); 4710 err = tcp_zerocopy_receive(sk, &zc, &tss); 4711 err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname, 4712 &zc, &len, err); 4713 sockopt_release_sock(sk); 4714 if (len >= offsetofend(struct tcp_zerocopy_receive, msg_flags)) 4715 goto zerocopy_rcv_cmsg; 4716 switch (len) { 4717 case offsetofend(struct tcp_zerocopy_receive, msg_flags): 4718 goto zerocopy_rcv_cmsg; 4719 case offsetofend(struct tcp_zerocopy_receive, msg_controllen): 4720 case offsetofend(struct tcp_zerocopy_receive, msg_control): 4721 case offsetofend(struct tcp_zerocopy_receive, flags): 4722 case offsetofend(struct tcp_zerocopy_receive, copybuf_len): 4723 case offsetofend(struct tcp_zerocopy_receive, copybuf_address): 4724 case offsetofend(struct tcp_zerocopy_receive, err): 4725 goto zerocopy_rcv_sk_err; 4726 case offsetofend(struct tcp_zerocopy_receive, inq): 4727 goto zerocopy_rcv_inq; 4728 case offsetofend(struct tcp_zerocopy_receive, length): 4729 default: 4730 goto zerocopy_rcv_out; 4731 } 4732 zerocopy_rcv_cmsg: 4733 if (zc.msg_flags & TCP_CMSG_TS) 4734 tcp_zc_finalize_rx_tstamp(sk, &zc, &tss); 4735 else 4736 zc.msg_flags = 0; 4737 zerocopy_rcv_sk_err: 4738 if (!err) 4739 zc.err = sock_error(sk); 4740 zerocopy_rcv_inq: 4741 zc.inq = tcp_inq_hint(sk); 4742 zerocopy_rcv_out: 4743 if (!err && copy_to_sockptr(optval, &zc, len)) 4744 err = -EFAULT; 4745 return err; 4746 } 4747 #endif 4748 case TCP_AO_REPAIR: 4749 if (!tcp_can_repair_sock(sk)) 4750 return -EPERM; 4751 return tcp_ao_get_repair(sk, optval, optlen); 4752 case TCP_AO_GET_KEYS: 4753 case TCP_AO_INFO: { 4754 int err; 4755 4756 sockopt_lock_sock(sk); 4757 if (optname == TCP_AO_GET_KEYS) 4758 err = tcp_ao_get_mkts(sk, optval, optlen); 4759 else 4760 err = tcp_ao_get_sock_info(sk, optval, optlen); 4761 sockopt_release_sock(sk); 4762 4763 return err; 4764 } 4765 case TCP_IS_MPTCP: 4766 val = 0; 4767 break; 4768 case TCP_RTO_MAX_MS: 4769 val = jiffies_to_msecs(tcp_rto_max(sk)); 4770 break; 4771 case TCP_RTO_MIN_US: 4772 val = jiffies_to_usecs(READ_ONCE(inet_csk(sk)->icsk_rto_min)); 4773 break; 4774 case TCP_DELACK_MAX_US: 4775 val = jiffies_to_usecs(READ_ONCE(inet_csk(sk)->icsk_delack_max)); 4776 break; 4777 default: 4778 return -ENOPROTOOPT; 4779 } 4780 4781 if (copy_to_sockptr(optlen, &len, sizeof(int))) 4782 return -EFAULT; 4783 if (copy_to_sockptr(optval, &val, len)) 4784 return -EFAULT; 4785 return 0; 4786 } 4787 4788 bool tcp_bpf_bypass_getsockopt(int level, int optname) 4789 { 4790 /* TCP do_tcp_getsockopt has optimized getsockopt implementation 4791 * to avoid extra socket lock for TCP_ZEROCOPY_RECEIVE. 4792 */ 4793 if (level == SOL_TCP && optname == TCP_ZEROCOPY_RECEIVE) 4794 return true; 4795 4796 return false; 4797 } 4798 EXPORT_IPV6_MOD(tcp_bpf_bypass_getsockopt); 4799 4800 int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval, 4801 int __user *optlen) 4802 { 4803 struct inet_connection_sock *icsk = inet_csk(sk); 4804 4805 if (level != SOL_TCP) 4806 /* Paired with WRITE_ONCE() in do_ipv6_setsockopt() and tcp_v6_connect() */ 4807 return READ_ONCE(icsk->icsk_af_ops)->getsockopt(sk, level, optname, 4808 optval, optlen); 4809 return do_tcp_getsockopt(sk, level, optname, USER_SOCKPTR(optval), 4810 USER_SOCKPTR(optlen)); 4811 } 4812 EXPORT_IPV6_MOD(tcp_getsockopt); 4813 4814 #ifdef CONFIG_TCP_MD5SIG 4815 int tcp_md5_sigpool_id = -1; 4816 EXPORT_IPV6_MOD_GPL(tcp_md5_sigpool_id); 4817 4818 int tcp_md5_alloc_sigpool(void) 4819 { 4820 size_t scratch_size; 4821 int ret; 4822 4823 scratch_size = sizeof(union tcp_md5sum_block) + sizeof(struct tcphdr); 4824 ret = tcp_sigpool_alloc_ahash("md5", scratch_size); 4825 if (ret >= 0) { 4826 /* As long as any md5 sigpool was allocated, the return 4827 * id would stay the same. Re-write the id only for the case 4828 * when previously all MD5 keys were deleted and this call 4829 * allocates the first MD5 key, which may return a different 4830 * sigpool id than was used previously. 4831 */ 4832 WRITE_ONCE(tcp_md5_sigpool_id, ret); /* Avoids the compiler potentially being smart here */ 4833 return 0; 4834 } 4835 return ret; 4836 } 4837 4838 void tcp_md5_release_sigpool(void) 4839 { 4840 tcp_sigpool_release(READ_ONCE(tcp_md5_sigpool_id)); 4841 } 4842 4843 void tcp_md5_add_sigpool(void) 4844 { 4845 tcp_sigpool_get(READ_ONCE(tcp_md5_sigpool_id)); 4846 } 4847 4848 int tcp_md5_hash_key(struct tcp_sigpool *hp, 4849 const struct tcp_md5sig_key *key) 4850 { 4851 u8 keylen = READ_ONCE(key->keylen); /* paired with WRITE_ONCE() in tcp_md5_do_add */ 4852 struct scatterlist sg; 4853 4854 sg_init_one(&sg, key->key, keylen); 4855 ahash_request_set_crypt(hp->req, &sg, NULL, keylen); 4856 4857 /* We use data_race() because tcp_md5_do_add() might change 4858 * key->key under us 4859 */ 4860 return data_race(crypto_ahash_update(hp->req)); 4861 } 4862 EXPORT_IPV6_MOD(tcp_md5_hash_key); 4863 4864 /* Called with rcu_read_lock() */ 4865 static enum skb_drop_reason 4866 tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb, 4867 const void *saddr, const void *daddr, 4868 int family, int l3index, const __u8 *hash_location) 4869 { 4870 /* This gets called for each TCP segment that has TCP-MD5 option. 4871 * We have 3 drop cases: 4872 * o No MD5 hash and one expected. 4873 * o MD5 hash and we're not expecting one. 4874 * o MD5 hash and its wrong. 4875 */ 4876 const struct tcp_sock *tp = tcp_sk(sk); 4877 struct tcp_md5sig_key *key; 4878 u8 newhash[16]; 4879 int genhash; 4880 4881 key = tcp_md5_do_lookup(sk, l3index, saddr, family); 4882 4883 if (!key && hash_location) { 4884 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5UNEXPECTED); 4885 trace_tcp_hash_md5_unexpected(sk, skb); 4886 return SKB_DROP_REASON_TCP_MD5UNEXPECTED; 4887 } 4888 4889 /* Check the signature. 4890 * To support dual stack listeners, we need to handle 4891 * IPv4-mapped case. 4892 */ 4893 if (family == AF_INET) 4894 genhash = tcp_v4_md5_hash_skb(newhash, key, NULL, skb); 4895 else 4896 genhash = tp->af_specific->calc_md5_hash(newhash, key, 4897 NULL, skb); 4898 if (genhash || memcmp(hash_location, newhash, 16) != 0) { 4899 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5FAILURE); 4900 trace_tcp_hash_md5_mismatch(sk, skb); 4901 return SKB_DROP_REASON_TCP_MD5FAILURE; 4902 } 4903 return SKB_NOT_DROPPED_YET; 4904 } 4905 #else 4906 static inline enum skb_drop_reason 4907 tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb, 4908 const void *saddr, const void *daddr, 4909 int family, int l3index, const __u8 *hash_location) 4910 { 4911 return SKB_NOT_DROPPED_YET; 4912 } 4913 4914 #endif 4915 4916 /* Called with rcu_read_lock() */ 4917 enum skb_drop_reason 4918 tcp_inbound_hash(struct sock *sk, const struct request_sock *req, 4919 const struct sk_buff *skb, 4920 const void *saddr, const void *daddr, 4921 int family, int dif, int sdif) 4922 { 4923 const struct tcphdr *th = tcp_hdr(skb); 4924 const struct tcp_ao_hdr *aoh; 4925 const __u8 *md5_location; 4926 int l3index; 4927 4928 /* Invalid option or two times meet any of auth options */ 4929 if (tcp_parse_auth_options(th, &md5_location, &aoh)) { 4930 trace_tcp_hash_bad_header(sk, skb); 4931 return SKB_DROP_REASON_TCP_AUTH_HDR; 4932 } 4933 4934 if (req) { 4935 if (tcp_rsk_used_ao(req) != !!aoh) { 4936 u8 keyid, rnext, maclen; 4937 4938 if (aoh) { 4939 keyid = aoh->keyid; 4940 rnext = aoh->rnext_keyid; 4941 maclen = tcp_ao_hdr_maclen(aoh); 4942 } else { 4943 keyid = rnext = maclen = 0; 4944 } 4945 4946 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOBAD); 4947 trace_tcp_ao_handshake_failure(sk, skb, keyid, rnext, maclen); 4948 return SKB_DROP_REASON_TCP_AOFAILURE; 4949 } 4950 } 4951 4952 /* sdif set, means packet ingressed via a device 4953 * in an L3 domain and dif is set to the l3mdev 4954 */ 4955 l3index = sdif ? dif : 0; 4956 4957 /* Fast path: unsigned segments */ 4958 if (likely(!md5_location && !aoh)) { 4959 /* Drop if there's TCP-MD5 or TCP-AO key with any rcvid/sndid 4960 * for the remote peer. On TCP-AO established connection 4961 * the last key is impossible to remove, so there's 4962 * always at least one current_key. 4963 */ 4964 if (tcp_ao_required(sk, saddr, family, l3index, true)) { 4965 trace_tcp_hash_ao_required(sk, skb); 4966 return SKB_DROP_REASON_TCP_AONOTFOUND; 4967 } 4968 if (unlikely(tcp_md5_do_lookup(sk, l3index, saddr, family))) { 4969 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5NOTFOUND); 4970 trace_tcp_hash_md5_required(sk, skb); 4971 return SKB_DROP_REASON_TCP_MD5NOTFOUND; 4972 } 4973 return SKB_NOT_DROPPED_YET; 4974 } 4975 4976 if (aoh) 4977 return tcp_inbound_ao_hash(sk, skb, family, req, l3index, aoh); 4978 4979 return tcp_inbound_md5_hash(sk, skb, saddr, daddr, family, 4980 l3index, md5_location); 4981 } 4982 EXPORT_IPV6_MOD_GPL(tcp_inbound_hash); 4983 4984 void tcp_done(struct sock *sk) 4985 { 4986 struct request_sock *req; 4987 4988 /* We might be called with a new socket, after 4989 * inet_csk_prepare_forced_close() has been called 4990 * so we can not use lockdep_sock_is_held(sk) 4991 */ 4992 req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk, 1); 4993 4994 if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV) 4995 TCP_INC_STATS(sock_net(sk), TCP_MIB_ATTEMPTFAILS); 4996 4997 tcp_set_state(sk, TCP_CLOSE); 4998 tcp_clear_xmit_timers(sk); 4999 if (req) 5000 reqsk_fastopen_remove(sk, req, false); 5001 5002 WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK); 5003 5004 if (!sock_flag(sk, SOCK_DEAD)) 5005 sk->sk_state_change(sk); 5006 else 5007 inet_csk_destroy_sock(sk); 5008 } 5009 EXPORT_SYMBOL_GPL(tcp_done); 5010 5011 int tcp_abort(struct sock *sk, int err) 5012 { 5013 int state = inet_sk_state_load(sk); 5014 5015 if (state == TCP_NEW_SYN_RECV) { 5016 struct request_sock *req = inet_reqsk(sk); 5017 5018 local_bh_disable(); 5019 inet_csk_reqsk_queue_drop(req->rsk_listener, req); 5020 local_bh_enable(); 5021 return 0; 5022 } 5023 if (state == TCP_TIME_WAIT) { 5024 struct inet_timewait_sock *tw = inet_twsk(sk); 5025 5026 refcount_inc(&tw->tw_refcnt); 5027 local_bh_disable(); 5028 inet_twsk_deschedule_put(tw); 5029 local_bh_enable(); 5030 return 0; 5031 } 5032 5033 /* BPF context ensures sock locking. */ 5034 if (!has_current_bpf_ctx()) 5035 /* Don't race with userspace socket closes such as tcp_close. */ 5036 lock_sock(sk); 5037 5038 /* Avoid closing the same socket twice. */ 5039 if (sk->sk_state == TCP_CLOSE) { 5040 if (!has_current_bpf_ctx()) 5041 release_sock(sk); 5042 return -ENOENT; 5043 } 5044 5045 if (sk->sk_state == TCP_LISTEN) { 5046 tcp_set_state(sk, TCP_CLOSE); 5047 inet_csk_listen_stop(sk); 5048 } 5049 5050 /* Don't race with BH socket closes such as inet_csk_listen_stop. */ 5051 local_bh_disable(); 5052 bh_lock_sock(sk); 5053 5054 if (tcp_need_reset(sk->sk_state)) 5055 tcp_send_active_reset(sk, GFP_ATOMIC, 5056 SK_RST_REASON_TCP_STATE); 5057 tcp_done_with_error(sk, err); 5058 5059 bh_unlock_sock(sk); 5060 local_bh_enable(); 5061 if (!has_current_bpf_ctx()) 5062 release_sock(sk); 5063 return 0; 5064 } 5065 EXPORT_SYMBOL_GPL(tcp_abort); 5066 5067 extern struct tcp_congestion_ops tcp_reno; 5068 5069 static __initdata unsigned long thash_entries; 5070 static int __init set_thash_entries(char *str) 5071 { 5072 ssize_t ret; 5073 5074 if (!str) 5075 return 0; 5076 5077 ret = kstrtoul(str, 0, &thash_entries); 5078 if (ret) 5079 return 0; 5080 5081 return 1; 5082 } 5083 __setup("thash_entries=", set_thash_entries); 5084 5085 static void __init tcp_init_mem(void) 5086 { 5087 unsigned long limit = nr_free_buffer_pages() / 16; 5088 5089 limit = max(limit, 128UL); 5090 sysctl_tcp_mem[0] = limit / 4 * 3; /* 4.68 % */ 5091 sysctl_tcp_mem[1] = limit; /* 6.25 % */ 5092 sysctl_tcp_mem[2] = sysctl_tcp_mem[0] * 2; /* 9.37 % */ 5093 } 5094 5095 static void __init tcp_struct_check(void) 5096 { 5097 /* TX read-mostly hotpath cache lines */ 5098 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, max_window); 5099 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, rcv_ssthresh); 5100 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, reordering); 5101 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, notsent_lowat); 5102 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, gso_segs); 5103 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, retransmit_skb_hint); 5104 #if IS_ENABLED(CONFIG_TLS_DEVICE) 5105 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, tcp_clean_acked); 5106 #endif 5107 5108 /* TXRX read-mostly hotpath cache lines */ 5109 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, tsoffset); 5110 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, snd_wnd); 5111 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, mss_cache); 5112 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, snd_cwnd); 5113 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, prr_out); 5114 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, lost_out); 5115 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, sacked_out); 5116 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, scaling_ratio); 5117 5118 /* RX read-mostly hotpath cache lines */ 5119 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, copied_seq); 5120 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, snd_wl1); 5121 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, tlp_high_seq); 5122 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, rttvar_us); 5123 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, retrans_out); 5124 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, advmss); 5125 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, urg_data); 5126 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, lost); 5127 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, rtt_min); 5128 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, out_of_order_queue); 5129 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, snd_ssthresh); 5130 5131 /* TX read-write hotpath cache lines */ 5132 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, segs_out); 5133 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, data_segs_out); 5134 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, bytes_sent); 5135 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, snd_sml); 5136 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_start); 5137 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_stat); 5138 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, write_seq); 5139 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, pushed_seq); 5140 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, lsndtime); 5141 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, mdev_us); 5142 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tcp_wstamp_ns); 5143 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, accecn_opt_tstamp); 5144 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, rtt_seq); 5145 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tsorted_sent_queue); 5146 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, highest_sack); 5147 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, ecn_flags); 5148 5149 /* TXRX read-write hotpath cache lines */ 5150 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, pred_flags); 5151 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, tcp_clock_cache); 5152 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, tcp_mstamp); 5153 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_nxt); 5154 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_nxt); 5155 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_una); 5156 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, window_clamp); 5157 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, srtt_us); 5158 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, packets_out); 5159 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_up); 5160 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, delivered); 5161 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, delivered_ce); 5162 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, received_ce); 5163 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, received_ecn_bytes); 5164 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, app_limited); 5165 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_wnd); 5166 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_tstamp); 5167 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rx_opt); 5168 5169 /* RX read-write hotpath cache lines */ 5170 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, bytes_received); 5171 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, segs_in); 5172 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, data_segs_in); 5173 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_wup); 5174 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, max_packets_out); 5175 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, cwnd_usage_seq); 5176 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rate_delivered); 5177 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rate_interval_us); 5178 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_rtt_last_tsecr); 5179 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, delivered_ecn_bytes); 5180 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, first_tx_mstamp); 5181 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, delivered_mstamp); 5182 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, bytes_acked); 5183 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_rtt_est); 5184 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcvq_space); 5185 } 5186 5187 void __init tcp_init(void) 5188 { 5189 int max_rshare, max_wshare, cnt; 5190 unsigned long limit; 5191 unsigned int i; 5192 5193 BUILD_BUG_ON(TCP_MIN_SND_MSS <= MAX_TCP_OPTION_SPACE); 5194 BUILD_BUG_ON(sizeof(struct tcp_skb_cb) > 5195 sizeof_field(struct sk_buff, cb)); 5196 5197 tcp_struct_check(); 5198 5199 percpu_counter_init(&tcp_sockets_allocated, 0, GFP_KERNEL); 5200 5201 timer_setup(&tcp_orphan_timer, tcp_orphan_update, TIMER_DEFERRABLE); 5202 mod_timer(&tcp_orphan_timer, jiffies + TCP_ORPHAN_TIMER_PERIOD); 5203 5204 inet_hashinfo2_init(&tcp_hashinfo, "tcp_listen_portaddr_hash", 5205 thash_entries, 21, /* one slot per 2 MB*/ 5206 0, 64 * 1024); 5207 tcp_hashinfo.bind_bucket_cachep = 5208 kmem_cache_create("tcp_bind_bucket", 5209 sizeof(struct inet_bind_bucket), 0, 5210 SLAB_HWCACHE_ALIGN | SLAB_PANIC | 5211 SLAB_ACCOUNT, 5212 NULL); 5213 tcp_hashinfo.bind2_bucket_cachep = 5214 kmem_cache_create("tcp_bind2_bucket", 5215 sizeof(struct inet_bind2_bucket), 0, 5216 SLAB_HWCACHE_ALIGN | SLAB_PANIC | 5217 SLAB_ACCOUNT, 5218 NULL); 5219 5220 /* Size and allocate the main established and bind bucket 5221 * hash tables. 5222 * 5223 * The methodology is similar to that of the buffer cache. 5224 */ 5225 tcp_hashinfo.ehash = 5226 alloc_large_system_hash("TCP established", 5227 sizeof(struct inet_ehash_bucket), 5228 thash_entries, 5229 17, /* one slot per 128 KB of memory */ 5230 0, 5231 NULL, 5232 &tcp_hashinfo.ehash_mask, 5233 0, 5234 thash_entries ? 0 : 512 * 1024); 5235 for (i = 0; i <= tcp_hashinfo.ehash_mask; i++) 5236 INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].chain, i); 5237 5238 if (inet_ehash_locks_alloc(&tcp_hashinfo)) 5239 panic("TCP: failed to alloc ehash_locks"); 5240 tcp_hashinfo.bhash = 5241 alloc_large_system_hash("TCP bind", 5242 2 * sizeof(struct inet_bind_hashbucket), 5243 tcp_hashinfo.ehash_mask + 1, 5244 17, /* one slot per 128 KB of memory */ 5245 0, 5246 &tcp_hashinfo.bhash_size, 5247 NULL, 5248 0, 5249 64 * 1024); 5250 tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size; 5251 tcp_hashinfo.bhash2 = tcp_hashinfo.bhash + tcp_hashinfo.bhash_size; 5252 for (i = 0; i < tcp_hashinfo.bhash_size; i++) { 5253 spin_lock_init(&tcp_hashinfo.bhash[i].lock); 5254 INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain); 5255 spin_lock_init(&tcp_hashinfo.bhash2[i].lock); 5256 INIT_HLIST_HEAD(&tcp_hashinfo.bhash2[i].chain); 5257 } 5258 5259 tcp_hashinfo.pernet = false; 5260 5261 cnt = tcp_hashinfo.ehash_mask + 1; 5262 sysctl_tcp_max_orphans = cnt / 2; 5263 5264 tcp_init_mem(); 5265 /* Set per-socket limits to no more than 1/128 the pressure threshold */ 5266 limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7); 5267 max_wshare = min(4UL*1024*1024, limit); 5268 max_rshare = min(32UL*1024*1024, limit); 5269 5270 init_net.ipv4.sysctl_tcp_wmem[0] = PAGE_SIZE; 5271 init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024; 5272 init_net.ipv4.sysctl_tcp_wmem[2] = max(64*1024, max_wshare); 5273 5274 init_net.ipv4.sysctl_tcp_rmem[0] = PAGE_SIZE; 5275 init_net.ipv4.sysctl_tcp_rmem[1] = 131072; 5276 init_net.ipv4.sysctl_tcp_rmem[2] = max(131072, max_rshare); 5277 5278 pr_info("Hash tables configured (established %u bind %u)\n", 5279 tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size); 5280 5281 tcp_v4_init(); 5282 tcp_metrics_init(); 5283 BUG_ON(tcp_register_congestion_control(&tcp_reno) != 0); 5284 tcp_tsq_work_init(); 5285 mptcp_init(); 5286 } 5287