1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * Implementation of the Transmission Control Protocol(TCP). 8 * 9 * Authors: Ross Biro 10 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 11 * Mark Evans, <evansmp@uhura.aston.ac.uk> 12 * Corey Minyard <wf-rch!minyard@relay.EU.net> 13 * Florian La Roche, <flla@stud.uni-sb.de> 14 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu> 15 * Linus Torvalds, <torvalds@cs.helsinki.fi> 16 * Alan Cox, <gw4pts@gw4pts.ampr.org> 17 * Matthew Dillon, <dillon@apollo.west.oic.com> 18 * Arnt Gulbrandsen, <agulbra@nvg.unit.no> 19 * Jorge Cwik, <jorge@laser.satlink.net> 20 * 21 * Fixes: 22 * Alan Cox : Numerous verify_area() calls 23 * Alan Cox : Set the ACK bit on a reset 24 * Alan Cox : Stopped it crashing if it closed while 25 * sk->inuse=1 and was trying to connect 26 * (tcp_err()). 27 * Alan Cox : All icmp error handling was broken 28 * pointers passed where wrong and the 29 * socket was looked up backwards. Nobody 30 * tested any icmp error code obviously. 31 * Alan Cox : tcp_err() now handled properly. It 32 * wakes people on errors. poll 33 * behaves and the icmp error race 34 * has gone by moving it into sock.c 35 * Alan Cox : tcp_send_reset() fixed to work for 36 * everything not just packets for 37 * unknown sockets. 38 * Alan Cox : tcp option processing. 39 * Alan Cox : Reset tweaked (still not 100%) [Had 40 * syn rule wrong] 41 * Herp Rosmanith : More reset fixes 42 * Alan Cox : No longer acks invalid rst frames. 43 * Acking any kind of RST is right out. 44 * Alan Cox : Sets an ignore me flag on an rst 45 * receive otherwise odd bits of prattle 46 * escape still 47 * Alan Cox : Fixed another acking RST frame bug. 48 * Should stop LAN workplace lockups. 49 * Alan Cox : Some tidyups using the new skb list 50 * facilities 51 * Alan Cox : sk->keepopen now seems to work 52 * Alan Cox : Pulls options out correctly on accepts 53 * Alan Cox : Fixed assorted sk->rqueue->next errors 54 * Alan Cox : PSH doesn't end a TCP read. Switched a 55 * bit to skb ops. 56 * Alan Cox : Tidied tcp_data to avoid a potential 57 * nasty. 58 * Alan Cox : Added some better commenting, as the 59 * tcp is hard to follow 60 * Alan Cox : Removed incorrect check for 20 * psh 61 * Michael O'Reilly : ack < copied bug fix. 62 * Johannes Stille : Misc tcp fixes (not all in yet). 63 * Alan Cox : FIN with no memory -> CRASH 64 * Alan Cox : Added socket option proto entries. 65 * Also added awareness of them to accept. 66 * Alan Cox : Added TCP options (SOL_TCP) 67 * Alan Cox : Switched wakeup calls to callbacks, 68 * so the kernel can layer network 69 * sockets. 70 * Alan Cox : Use ip_tos/ip_ttl settings. 71 * Alan Cox : Handle FIN (more) properly (we hope). 72 * Alan Cox : RST frames sent on unsynchronised 73 * state ack error. 74 * Alan Cox : Put in missing check for SYN bit. 75 * Alan Cox : Added tcp_select_window() aka NET2E 76 * window non shrink trick. 77 * Alan Cox : Added a couple of small NET2E timer 78 * fixes 79 * Charles Hedrick : TCP fixes 80 * Toomas Tamm : TCP window fixes 81 * Alan Cox : Small URG fix to rlogin ^C ack fight 82 * Charles Hedrick : Rewrote most of it to actually work 83 * Linus : Rewrote tcp_read() and URG handling 84 * completely 85 * Gerhard Koerting: Fixed some missing timer handling 86 * Matthew Dillon : Reworked TCP machine states as per RFC 87 * Gerhard Koerting: PC/TCP workarounds 88 * Adam Caldwell : Assorted timer/timing errors 89 * Matthew Dillon : Fixed another RST bug 90 * Alan Cox : Move to kernel side addressing changes. 91 * Alan Cox : Beginning work on TCP fastpathing 92 * (not yet usable) 93 * Arnt Gulbrandsen: Turbocharged tcp_check() routine. 94 * Alan Cox : TCP fast path debugging 95 * Alan Cox : Window clamping 96 * Michael Riepe : Bug in tcp_check() 97 * Matt Dillon : More TCP improvements and RST bug fixes 98 * Matt Dillon : Yet more small nasties remove from the 99 * TCP code (Be very nice to this man if 100 * tcp finally works 100%) 8) 101 * Alan Cox : BSD accept semantics. 102 * Alan Cox : Reset on closedown bug. 103 * Peter De Schrijver : ENOTCONN check missing in tcp_sendto(). 104 * Michael Pall : Handle poll() after URG properly in 105 * all cases. 106 * Michael Pall : Undo the last fix in tcp_read_urg() 107 * (multi URG PUSH broke rlogin). 108 * Michael Pall : Fix the multi URG PUSH problem in 109 * tcp_readable(), poll() after URG 110 * works now. 111 * Michael Pall : recv(...,MSG_OOB) never blocks in the 112 * BSD api. 113 * Alan Cox : Changed the semantics of sk->socket to 114 * fix a race and a signal problem with 115 * accept() and async I/O. 116 * Alan Cox : Relaxed the rules on tcp_sendto(). 117 * Yury Shevchuk : Really fixed accept() blocking problem. 118 * Craig I. Hagan : Allow for BSD compatible TIME_WAIT for 119 * clients/servers which listen in on 120 * fixed ports. 121 * Alan Cox : Cleaned the above up and shrank it to 122 * a sensible code size. 123 * Alan Cox : Self connect lockup fix. 124 * Alan Cox : No connect to multicast. 125 * Ross Biro : Close unaccepted children on master 126 * socket close. 127 * Alan Cox : Reset tracing code. 128 * Alan Cox : Spurious resets on shutdown. 129 * Alan Cox : Giant 15 minute/60 second timer error 130 * Alan Cox : Small whoops in polling before an 131 * accept. 132 * Alan Cox : Kept the state trace facility since 133 * it's handy for debugging. 134 * Alan Cox : More reset handler fixes. 135 * Alan Cox : Started rewriting the code based on 136 * the RFC's for other useful protocol 137 * references see: Comer, KA9Q NOS, and 138 * for a reference on the difference 139 * between specifications and how BSD 140 * works see the 4.4lite source. 141 * A.N.Kuznetsov : Don't time wait on completion of tidy 142 * close. 143 * Linus Torvalds : Fin/Shutdown & copied_seq changes. 144 * Linus Torvalds : Fixed BSD port reuse to work first syn 145 * Alan Cox : Reimplemented timers as per the RFC 146 * and using multiple timers for sanity. 147 * Alan Cox : Small bug fixes, and a lot of new 148 * comments. 149 * Alan Cox : Fixed dual reader crash by locking 150 * the buffers (much like datagram.c) 151 * Alan Cox : Fixed stuck sockets in probe. A probe 152 * now gets fed up of retrying without 153 * (even a no space) answer. 154 * Alan Cox : Extracted closing code better 155 * Alan Cox : Fixed the closing state machine to 156 * resemble the RFC. 157 * Alan Cox : More 'per spec' fixes. 158 * Jorge Cwik : Even faster checksumming. 159 * Alan Cox : tcp_data() doesn't ack illegal PSH 160 * only frames. At least one pc tcp stack 161 * generates them. 162 * Alan Cox : Cache last socket. 163 * Alan Cox : Per route irtt. 164 * Matt Day : poll()->select() match BSD precisely on error 165 * Alan Cox : New buffers 166 * Marc Tamsky : Various sk->prot->retransmits and 167 * sk->retransmits misupdating fixed. 168 * Fixed tcp_write_timeout: stuck close, 169 * and TCP syn retries gets used now. 170 * Mark Yarvis : In tcp_read_wakeup(), don't send an 171 * ack if state is TCP_CLOSED. 172 * Alan Cox : Look up device on a retransmit - routes may 173 * change. Doesn't yet cope with MSS shrink right 174 * but it's a start! 175 * Marc Tamsky : Closing in closing fixes. 176 * Mike Shaver : RFC1122 verifications. 177 * Alan Cox : rcv_saddr errors. 178 * Alan Cox : Block double connect(). 179 * Alan Cox : Small hooks for enSKIP. 180 * Alexey Kuznetsov: Path MTU discovery. 181 * Alan Cox : Support soft errors. 182 * Alan Cox : Fix MTU discovery pathological case 183 * when the remote claims no mtu! 184 * Marc Tamsky : TCP_CLOSE fix. 185 * Colin (G3TNE) : Send a reset on syn ack replies in 186 * window but wrong (fixes NT lpd problems) 187 * Pedro Roque : Better TCP window handling, delayed ack. 188 * Joerg Reuter : No modification of locked buffers in 189 * tcp_do_retransmit() 190 * Eric Schenk : Changed receiver side silly window 191 * avoidance algorithm to BSD style 192 * algorithm. This doubles throughput 193 * against machines running Solaris, 194 * and seems to result in general 195 * improvement. 196 * Stefan Magdalinski : adjusted tcp_readable() to fix FIONREAD 197 * Willy Konynenberg : Transparent proxying support. 198 * Mike McLagan : Routing by source 199 * Keith Owens : Do proper merging with partial SKB's in 200 * tcp_do_sendmsg to avoid burstiness. 201 * Eric Schenk : Fix fast close down bug with 202 * shutdown() followed by close(). 203 * Andi Kleen : Make poll agree with SIGIO 204 * Salvatore Sanfilippo : Support SO_LINGER with linger == 1 and 205 * lingertime == 0 (RFC 793 ABORT Call) 206 * Hirokazu Takahashi : Use copy_from_user() instead of 207 * csum_and_copy_from_user() if possible. 208 * 209 * Description of States: 210 * 211 * TCP_SYN_SENT sent a connection request, waiting for ack 212 * 213 * TCP_SYN_RECV received a connection request, sent ack, 214 * waiting for final ack in three-way handshake. 215 * 216 * TCP_ESTABLISHED connection established 217 * 218 * TCP_FIN_WAIT1 our side has shutdown, waiting to complete 219 * transmission of remaining buffered data 220 * 221 * TCP_FIN_WAIT2 all buffered data sent, waiting for remote 222 * to shutdown 223 * 224 * TCP_CLOSING both sides have shutdown but we still have 225 * data we have to finish sending 226 * 227 * TCP_TIME_WAIT timeout to catch resent junk before entering 228 * closed, can only be entered from FIN_WAIT2 229 * or CLOSING. Required because the other end 230 * may not have gotten our last ACK causing it 231 * to retransmit the data packet (which we ignore) 232 * 233 * TCP_CLOSE_WAIT remote side has shutdown and is waiting for 234 * us to finish writing our data and to shutdown 235 * (we have to close() to move on to LAST_ACK) 236 * 237 * TCP_LAST_ACK out side has shutdown after remote has 238 * shutdown. There may still be data in our 239 * buffer that we have to finish sending 240 * 241 * TCP_CLOSE socket is finished 242 */ 243 244 #define pr_fmt(fmt) "TCP: " fmt 245 246 #include <crypto/hash.h> 247 #include <linux/kernel.h> 248 #include <linux/module.h> 249 #include <linux/types.h> 250 #include <linux/fcntl.h> 251 #include <linux/poll.h> 252 #include <linux/inet_diag.h> 253 #include <linux/init.h> 254 #include <linux/fs.h> 255 #include <linux/skbuff.h> 256 #include <linux/scatterlist.h> 257 #include <linux/splice.h> 258 #include <linux/net.h> 259 #include <linux/socket.h> 260 #include <linux/random.h> 261 #include <linux/memblock.h> 262 #include <linux/highmem.h> 263 #include <linux/cache.h> 264 #include <linux/err.h> 265 #include <linux/time.h> 266 #include <linux/slab.h> 267 #include <linux/errqueue.h> 268 #include <linux/static_key.h> 269 #include <linux/btf.h> 270 271 #include <net/icmp.h> 272 #include <net/inet_common.h> 273 #include <net/tcp.h> 274 #include <net/mptcp.h> 275 #include <net/xfrm.h> 276 #include <net/ip.h> 277 #include <net/sock.h> 278 279 #include <linux/uaccess.h> 280 #include <asm/ioctls.h> 281 #include <net/busy_poll.h> 282 283 /* Track pending CMSGs. */ 284 enum { 285 TCP_CMSG_INQ = 1, 286 TCP_CMSG_TS = 2 287 }; 288 289 DEFINE_PER_CPU(unsigned int, tcp_orphan_count); 290 EXPORT_PER_CPU_SYMBOL_GPL(tcp_orphan_count); 291 292 long sysctl_tcp_mem[3] __read_mostly; 293 EXPORT_SYMBOL(sysctl_tcp_mem); 294 295 atomic_long_t tcp_memory_allocated ____cacheline_aligned_in_smp; /* Current allocated memory. */ 296 EXPORT_SYMBOL(tcp_memory_allocated); 297 298 #if IS_ENABLED(CONFIG_SMC) 299 DEFINE_STATIC_KEY_FALSE(tcp_have_smc); 300 EXPORT_SYMBOL(tcp_have_smc); 301 #endif 302 303 /* 304 * Current number of TCP sockets. 305 */ 306 struct percpu_counter tcp_sockets_allocated ____cacheline_aligned_in_smp; 307 EXPORT_SYMBOL(tcp_sockets_allocated); 308 309 /* 310 * TCP splice context 311 */ 312 struct tcp_splice_state { 313 struct pipe_inode_info *pipe; 314 size_t len; 315 unsigned int flags; 316 }; 317 318 /* 319 * Pressure flag: try to collapse. 320 * Technical note: it is used by multiple contexts non atomically. 321 * All the __sk_mem_schedule() is of this nature: accounting 322 * is strict, actions are advisory and have some latency. 323 */ 324 unsigned long tcp_memory_pressure __read_mostly; 325 EXPORT_SYMBOL_GPL(tcp_memory_pressure); 326 327 void tcp_enter_memory_pressure(struct sock *sk) 328 { 329 unsigned long val; 330 331 if (READ_ONCE(tcp_memory_pressure)) 332 return; 333 val = jiffies; 334 335 if (!val) 336 val--; 337 if (!cmpxchg(&tcp_memory_pressure, 0, val)) 338 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURES); 339 } 340 EXPORT_SYMBOL_GPL(tcp_enter_memory_pressure); 341 342 void tcp_leave_memory_pressure(struct sock *sk) 343 { 344 unsigned long val; 345 346 if (!READ_ONCE(tcp_memory_pressure)) 347 return; 348 val = xchg(&tcp_memory_pressure, 0); 349 if (val) 350 NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURESCHRONO, 351 jiffies_to_msecs(jiffies - val)); 352 } 353 EXPORT_SYMBOL_GPL(tcp_leave_memory_pressure); 354 355 /* Convert seconds to retransmits based on initial and max timeout */ 356 static u8 secs_to_retrans(int seconds, int timeout, int rto_max) 357 { 358 u8 res = 0; 359 360 if (seconds > 0) { 361 int period = timeout; 362 363 res = 1; 364 while (seconds > period && res < 255) { 365 res++; 366 timeout <<= 1; 367 if (timeout > rto_max) 368 timeout = rto_max; 369 period += timeout; 370 } 371 } 372 return res; 373 } 374 375 /* Convert retransmits to seconds based on initial and max timeout */ 376 static int retrans_to_secs(u8 retrans, int timeout, int rto_max) 377 { 378 int period = 0; 379 380 if (retrans > 0) { 381 period = timeout; 382 while (--retrans) { 383 timeout <<= 1; 384 if (timeout > rto_max) 385 timeout = rto_max; 386 period += timeout; 387 } 388 } 389 return period; 390 } 391 392 static u64 tcp_compute_delivery_rate(const struct tcp_sock *tp) 393 { 394 u32 rate = READ_ONCE(tp->rate_delivered); 395 u32 intv = READ_ONCE(tp->rate_interval_us); 396 u64 rate64 = 0; 397 398 if (rate && intv) { 399 rate64 = (u64)rate * tp->mss_cache * USEC_PER_SEC; 400 do_div(rate64, intv); 401 } 402 return rate64; 403 } 404 405 /* Address-family independent initialization for a tcp_sock. 406 * 407 * NOTE: A lot of things set to zero explicitly by call to 408 * sk_alloc() so need not be done here. 409 */ 410 void tcp_init_sock(struct sock *sk) 411 { 412 struct inet_connection_sock *icsk = inet_csk(sk); 413 struct tcp_sock *tp = tcp_sk(sk); 414 415 tp->out_of_order_queue = RB_ROOT; 416 sk->tcp_rtx_queue = RB_ROOT; 417 tcp_init_xmit_timers(sk); 418 INIT_LIST_HEAD(&tp->tsq_node); 419 INIT_LIST_HEAD(&tp->tsorted_sent_queue); 420 421 icsk->icsk_rto = TCP_TIMEOUT_INIT; 422 icsk->icsk_rto_min = TCP_RTO_MIN; 423 icsk->icsk_delack_max = TCP_DELACK_MAX; 424 tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT); 425 minmax_reset(&tp->rtt_min, tcp_jiffies32, ~0U); 426 427 /* So many TCP implementations out there (incorrectly) count the 428 * initial SYN frame in their delayed-ACK and congestion control 429 * algorithms that we must have the following bandaid to talk 430 * efficiently to them. -DaveM 431 */ 432 tcp_snd_cwnd_set(tp, TCP_INIT_CWND); 433 434 /* There's a bubble in the pipe until at least the first ACK. */ 435 tp->app_limited = ~0U; 436 437 /* See draft-stevens-tcpca-spec-01 for discussion of the 438 * initialization of these values. 439 */ 440 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; 441 tp->snd_cwnd_clamp = ~0; 442 tp->mss_cache = TCP_MSS_DEFAULT; 443 444 tp->reordering = sock_net(sk)->ipv4.sysctl_tcp_reordering; 445 tcp_assign_congestion_control(sk); 446 447 tp->tsoffset = 0; 448 tp->rack.reo_wnd_steps = 1; 449 450 sk->sk_write_space = sk_stream_write_space; 451 sock_set_flag(sk, SOCK_USE_WRITE_QUEUE); 452 453 icsk->icsk_sync_mss = tcp_sync_mss; 454 455 WRITE_ONCE(sk->sk_sndbuf, sock_net(sk)->ipv4.sysctl_tcp_wmem[1]); 456 WRITE_ONCE(sk->sk_rcvbuf, sock_net(sk)->ipv4.sysctl_tcp_rmem[1]); 457 458 sk_sockets_allocated_inc(sk); 459 } 460 EXPORT_SYMBOL(tcp_init_sock); 461 462 static void tcp_tx_timestamp(struct sock *sk, u16 tsflags) 463 { 464 struct sk_buff *skb = tcp_write_queue_tail(sk); 465 466 if (tsflags && skb) { 467 struct skb_shared_info *shinfo = skb_shinfo(skb); 468 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb); 469 470 sock_tx_timestamp(sk, tsflags, &shinfo->tx_flags); 471 if (tsflags & SOF_TIMESTAMPING_TX_ACK) 472 tcb->txstamp_ack = 1; 473 if (tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK) 474 shinfo->tskey = TCP_SKB_CB(skb)->seq + skb->len - 1; 475 } 476 } 477 478 static bool tcp_stream_is_readable(struct sock *sk, int target) 479 { 480 if (tcp_epollin_ready(sk, target)) 481 return true; 482 return sk_is_readable(sk); 483 } 484 485 /* 486 * Wait for a TCP event. 487 * 488 * Note that we don't need to lock the socket, as the upper poll layers 489 * take care of normal races (between the test and the event) and we don't 490 * go look at any of the socket buffers directly. 491 */ 492 __poll_t tcp_poll(struct file *file, struct socket *sock, poll_table *wait) 493 { 494 __poll_t mask; 495 struct sock *sk = sock->sk; 496 const struct tcp_sock *tp = tcp_sk(sk); 497 int state; 498 499 sock_poll_wait(file, sock, wait); 500 501 state = inet_sk_state_load(sk); 502 if (state == TCP_LISTEN) 503 return inet_csk_listen_poll(sk); 504 505 /* Socket is not locked. We are protected from async events 506 * by poll logic and correct handling of state changes 507 * made by other threads is impossible in any case. 508 */ 509 510 mask = 0; 511 512 /* 513 * EPOLLHUP is certainly not done right. But poll() doesn't 514 * have a notion of HUP in just one direction, and for a 515 * socket the read side is more interesting. 516 * 517 * Some poll() documentation says that EPOLLHUP is incompatible 518 * with the EPOLLOUT/POLLWR flags, so somebody should check this 519 * all. But careful, it tends to be safer to return too many 520 * bits than too few, and you can easily break real applications 521 * if you don't tell them that something has hung up! 522 * 523 * Check-me. 524 * 525 * Check number 1. EPOLLHUP is _UNMASKABLE_ event (see UNIX98 and 526 * our fs/select.c). It means that after we received EOF, 527 * poll always returns immediately, making impossible poll() on write() 528 * in state CLOSE_WAIT. One solution is evident --- to set EPOLLHUP 529 * if and only if shutdown has been made in both directions. 530 * Actually, it is interesting to look how Solaris and DUX 531 * solve this dilemma. I would prefer, if EPOLLHUP were maskable, 532 * then we could set it on SND_SHUTDOWN. BTW examples given 533 * in Stevens' books assume exactly this behaviour, it explains 534 * why EPOLLHUP is incompatible with EPOLLOUT. --ANK 535 * 536 * NOTE. Check for TCP_CLOSE is added. The goal is to prevent 537 * blocking on fresh not-connected or disconnected socket. --ANK 538 */ 539 if (sk->sk_shutdown == SHUTDOWN_MASK || state == TCP_CLOSE) 540 mask |= EPOLLHUP; 541 if (sk->sk_shutdown & RCV_SHUTDOWN) 542 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 543 544 /* Connected or passive Fast Open socket? */ 545 if (state != TCP_SYN_SENT && 546 (state != TCP_SYN_RECV || rcu_access_pointer(tp->fastopen_rsk))) { 547 int target = sock_rcvlowat(sk, 0, INT_MAX); 548 u16 urg_data = READ_ONCE(tp->urg_data); 549 550 if (unlikely(urg_data) && 551 READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq) && 552 !sock_flag(sk, SOCK_URGINLINE)) 553 target++; 554 555 if (tcp_stream_is_readable(sk, target)) 556 mask |= EPOLLIN | EPOLLRDNORM; 557 558 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) { 559 if (__sk_stream_is_writeable(sk, 1)) { 560 mask |= EPOLLOUT | EPOLLWRNORM; 561 } else { /* send SIGIO later */ 562 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 563 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 564 565 /* Race breaker. If space is freed after 566 * wspace test but before the flags are set, 567 * IO signal will be lost. Memory barrier 568 * pairs with the input side. 569 */ 570 smp_mb__after_atomic(); 571 if (__sk_stream_is_writeable(sk, 1)) 572 mask |= EPOLLOUT | EPOLLWRNORM; 573 } 574 } else 575 mask |= EPOLLOUT | EPOLLWRNORM; 576 577 if (urg_data & TCP_URG_VALID) 578 mask |= EPOLLPRI; 579 } else if (state == TCP_SYN_SENT && inet_sk(sk)->defer_connect) { 580 /* Active TCP fastopen socket with defer_connect 581 * Return EPOLLOUT so application can call write() 582 * in order for kernel to generate SYN+data 583 */ 584 mask |= EPOLLOUT | EPOLLWRNORM; 585 } 586 /* This barrier is coupled with smp_wmb() in tcp_reset() */ 587 smp_rmb(); 588 if (sk->sk_err || !skb_queue_empty_lockless(&sk->sk_error_queue)) 589 mask |= EPOLLERR; 590 591 return mask; 592 } 593 EXPORT_SYMBOL(tcp_poll); 594 595 int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg) 596 { 597 struct tcp_sock *tp = tcp_sk(sk); 598 int answ; 599 bool slow; 600 601 switch (cmd) { 602 case SIOCINQ: 603 if (sk->sk_state == TCP_LISTEN) 604 return -EINVAL; 605 606 slow = lock_sock_fast(sk); 607 answ = tcp_inq(sk); 608 unlock_sock_fast(sk, slow); 609 break; 610 case SIOCATMARK: 611 answ = READ_ONCE(tp->urg_data) && 612 READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq); 613 break; 614 case SIOCOUTQ: 615 if (sk->sk_state == TCP_LISTEN) 616 return -EINVAL; 617 618 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) 619 answ = 0; 620 else 621 answ = READ_ONCE(tp->write_seq) - tp->snd_una; 622 break; 623 case SIOCOUTQNSD: 624 if (sk->sk_state == TCP_LISTEN) 625 return -EINVAL; 626 627 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) 628 answ = 0; 629 else 630 answ = READ_ONCE(tp->write_seq) - 631 READ_ONCE(tp->snd_nxt); 632 break; 633 default: 634 return -ENOIOCTLCMD; 635 } 636 637 return put_user(answ, (int __user *)arg); 638 } 639 EXPORT_SYMBOL(tcp_ioctl); 640 641 void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb) 642 { 643 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH; 644 tp->pushed_seq = tp->write_seq; 645 } 646 647 static inline bool forced_push(const struct tcp_sock *tp) 648 { 649 return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1)); 650 } 651 652 void tcp_skb_entail(struct sock *sk, struct sk_buff *skb) 653 { 654 struct tcp_sock *tp = tcp_sk(sk); 655 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb); 656 657 tcb->seq = tcb->end_seq = tp->write_seq; 658 tcb->tcp_flags = TCPHDR_ACK; 659 __skb_header_release(skb); 660 tcp_add_write_queue_tail(sk, skb); 661 sk_wmem_queued_add(sk, skb->truesize); 662 sk_mem_charge(sk, skb->truesize); 663 if (tp->nonagle & TCP_NAGLE_PUSH) 664 tp->nonagle &= ~TCP_NAGLE_PUSH; 665 666 tcp_slow_start_after_idle_check(sk); 667 } 668 669 static inline void tcp_mark_urg(struct tcp_sock *tp, int flags) 670 { 671 if (flags & MSG_OOB) 672 tp->snd_up = tp->write_seq; 673 } 674 675 /* If a not yet filled skb is pushed, do not send it if 676 * we have data packets in Qdisc or NIC queues : 677 * Because TX completion will happen shortly, it gives a chance 678 * to coalesce future sendmsg() payload into this skb, without 679 * need for a timer, and with no latency trade off. 680 * As packets containing data payload have a bigger truesize 681 * than pure acks (dataless) packets, the last checks prevent 682 * autocorking if we only have an ACK in Qdisc/NIC queues, 683 * or if TX completion was delayed after we processed ACK packet. 684 */ 685 static bool tcp_should_autocork(struct sock *sk, struct sk_buff *skb, 686 int size_goal) 687 { 688 return skb->len < size_goal && 689 sock_net(sk)->ipv4.sysctl_tcp_autocorking && 690 !tcp_rtx_queue_empty(sk) && 691 refcount_read(&sk->sk_wmem_alloc) > skb->truesize && 692 tcp_skb_can_collapse_to(skb); 693 } 694 695 void tcp_push(struct sock *sk, int flags, int mss_now, 696 int nonagle, int size_goal) 697 { 698 struct tcp_sock *tp = tcp_sk(sk); 699 struct sk_buff *skb; 700 701 skb = tcp_write_queue_tail(sk); 702 if (!skb) 703 return; 704 if (!(flags & MSG_MORE) || forced_push(tp)) 705 tcp_mark_push(tp, skb); 706 707 tcp_mark_urg(tp, flags); 708 709 if (tcp_should_autocork(sk, skb, size_goal)) { 710 711 /* avoid atomic op if TSQ_THROTTLED bit is already set */ 712 if (!test_bit(TSQ_THROTTLED, &sk->sk_tsq_flags)) { 713 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING); 714 set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags); 715 } 716 /* It is possible TX completion already happened 717 * before we set TSQ_THROTTLED. 718 */ 719 if (refcount_read(&sk->sk_wmem_alloc) > skb->truesize) 720 return; 721 } 722 723 if (flags & MSG_MORE) 724 nonagle = TCP_NAGLE_CORK; 725 726 __tcp_push_pending_frames(sk, mss_now, nonagle); 727 } 728 729 static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb, 730 unsigned int offset, size_t len) 731 { 732 struct tcp_splice_state *tss = rd_desc->arg.data; 733 int ret; 734 735 ret = skb_splice_bits(skb, skb->sk, offset, tss->pipe, 736 min(rd_desc->count, len), tss->flags); 737 if (ret > 0) 738 rd_desc->count -= ret; 739 return ret; 740 } 741 742 static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss) 743 { 744 /* Store TCP splice context information in read_descriptor_t. */ 745 read_descriptor_t rd_desc = { 746 .arg.data = tss, 747 .count = tss->len, 748 }; 749 750 return tcp_read_sock(sk, &rd_desc, tcp_splice_data_recv); 751 } 752 753 /** 754 * tcp_splice_read - splice data from TCP socket to a pipe 755 * @sock: socket to splice from 756 * @ppos: position (not valid) 757 * @pipe: pipe to splice to 758 * @len: number of bytes to splice 759 * @flags: splice modifier flags 760 * 761 * Description: 762 * Will read pages from given socket and fill them into a pipe. 763 * 764 **/ 765 ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos, 766 struct pipe_inode_info *pipe, size_t len, 767 unsigned int flags) 768 { 769 struct sock *sk = sock->sk; 770 struct tcp_splice_state tss = { 771 .pipe = pipe, 772 .len = len, 773 .flags = flags, 774 }; 775 long timeo; 776 ssize_t spliced; 777 int ret; 778 779 sock_rps_record_flow(sk); 780 /* 781 * We can't seek on a socket input 782 */ 783 if (unlikely(*ppos)) 784 return -ESPIPE; 785 786 ret = spliced = 0; 787 788 lock_sock(sk); 789 790 timeo = sock_rcvtimeo(sk, sock->file->f_flags & O_NONBLOCK); 791 while (tss.len) { 792 ret = __tcp_splice_read(sk, &tss); 793 if (ret < 0) 794 break; 795 else if (!ret) { 796 if (spliced) 797 break; 798 if (sock_flag(sk, SOCK_DONE)) 799 break; 800 if (sk->sk_err) { 801 ret = sock_error(sk); 802 break; 803 } 804 if (sk->sk_shutdown & RCV_SHUTDOWN) 805 break; 806 if (sk->sk_state == TCP_CLOSE) { 807 /* 808 * This occurs when user tries to read 809 * from never connected socket. 810 */ 811 ret = -ENOTCONN; 812 break; 813 } 814 if (!timeo) { 815 ret = -EAGAIN; 816 break; 817 } 818 /* if __tcp_splice_read() got nothing while we have 819 * an skb in receive queue, we do not want to loop. 820 * This might happen with URG data. 821 */ 822 if (!skb_queue_empty(&sk->sk_receive_queue)) 823 break; 824 sk_wait_data(sk, &timeo, NULL); 825 if (signal_pending(current)) { 826 ret = sock_intr_errno(timeo); 827 break; 828 } 829 continue; 830 } 831 tss.len -= ret; 832 spliced += ret; 833 834 if (!timeo) 835 break; 836 release_sock(sk); 837 lock_sock(sk); 838 839 if (sk->sk_err || sk->sk_state == TCP_CLOSE || 840 (sk->sk_shutdown & RCV_SHUTDOWN) || 841 signal_pending(current)) 842 break; 843 } 844 845 release_sock(sk); 846 sk_defer_free_flush(sk); 847 848 if (spliced) 849 return spliced; 850 851 return ret; 852 } 853 EXPORT_SYMBOL(tcp_splice_read); 854 855 struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp, 856 bool force_schedule) 857 { 858 struct sk_buff *skb; 859 860 if (unlikely(tcp_under_memory_pressure(sk))) 861 sk_mem_reclaim_partial(sk); 862 863 skb = alloc_skb_fclone(size + MAX_TCP_HEADER, gfp); 864 if (likely(skb)) { 865 bool mem_scheduled; 866 867 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 868 if (force_schedule) { 869 mem_scheduled = true; 870 sk_forced_mem_schedule(sk, skb->truesize); 871 } else { 872 mem_scheduled = sk_wmem_schedule(sk, skb->truesize); 873 } 874 if (likely(mem_scheduled)) { 875 skb_reserve(skb, MAX_TCP_HEADER); 876 skb->ip_summed = CHECKSUM_PARTIAL; 877 INIT_LIST_HEAD(&skb->tcp_tsorted_anchor); 878 return skb; 879 } 880 __kfree_skb(skb); 881 } else { 882 sk->sk_prot->enter_memory_pressure(sk); 883 sk_stream_moderate_sndbuf(sk); 884 } 885 return NULL; 886 } 887 888 static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now, 889 int large_allowed) 890 { 891 struct tcp_sock *tp = tcp_sk(sk); 892 u32 new_size_goal, size_goal; 893 894 if (!large_allowed) 895 return mss_now; 896 897 /* Note : tcp_tso_autosize() will eventually split this later */ 898 new_size_goal = tcp_bound_to_half_wnd(tp, sk->sk_gso_max_size); 899 900 /* We try hard to avoid divides here */ 901 size_goal = tp->gso_segs * mss_now; 902 if (unlikely(new_size_goal < size_goal || 903 new_size_goal >= size_goal + mss_now)) { 904 tp->gso_segs = min_t(u16, new_size_goal / mss_now, 905 sk->sk_gso_max_segs); 906 size_goal = tp->gso_segs * mss_now; 907 } 908 909 return max(size_goal, mss_now); 910 } 911 912 int tcp_send_mss(struct sock *sk, int *size_goal, int flags) 913 { 914 int mss_now; 915 916 mss_now = tcp_current_mss(sk); 917 *size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB)); 918 919 return mss_now; 920 } 921 922 /* In some cases, both sendpage() and sendmsg() could have added 923 * an skb to the write queue, but failed adding payload on it. 924 * We need to remove it to consume less memory, but more 925 * importantly be able to generate EPOLLOUT for Edge Trigger epoll() 926 * users. 927 */ 928 void tcp_remove_empty_skb(struct sock *sk) 929 { 930 struct sk_buff *skb = tcp_write_queue_tail(sk); 931 932 if (skb && TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) { 933 tcp_unlink_write_queue(skb, sk); 934 if (tcp_write_queue_empty(sk)) 935 tcp_chrono_stop(sk, TCP_CHRONO_BUSY); 936 tcp_wmem_free_skb(sk, skb); 937 } 938 } 939 940 /* skb changing from pure zc to mixed, must charge zc */ 941 static int tcp_downgrade_zcopy_pure(struct sock *sk, struct sk_buff *skb) 942 { 943 if (unlikely(skb_zcopy_pure(skb))) { 944 u32 extra = skb->truesize - 945 SKB_TRUESIZE(skb_end_offset(skb)); 946 947 if (!sk_wmem_schedule(sk, extra)) 948 return -ENOMEM; 949 950 sk_mem_charge(sk, extra); 951 skb_shinfo(skb)->flags &= ~SKBFL_PURE_ZEROCOPY; 952 } 953 return 0; 954 } 955 956 static struct sk_buff *tcp_build_frag(struct sock *sk, int size_goal, int flags, 957 struct page *page, int offset, size_t *size) 958 { 959 struct sk_buff *skb = tcp_write_queue_tail(sk); 960 struct tcp_sock *tp = tcp_sk(sk); 961 bool can_coalesce; 962 int copy, i; 963 964 if (!skb || (copy = size_goal - skb->len) <= 0 || 965 !tcp_skb_can_collapse_to(skb)) { 966 new_segment: 967 if (!sk_stream_memory_free(sk)) 968 return NULL; 969 970 skb = tcp_stream_alloc_skb(sk, 0, sk->sk_allocation, 971 tcp_rtx_and_write_queues_empty(sk)); 972 if (!skb) 973 return NULL; 974 975 #ifdef CONFIG_TLS_DEVICE 976 skb->decrypted = !!(flags & MSG_SENDPAGE_DECRYPTED); 977 #endif 978 tcp_skb_entail(sk, skb); 979 copy = size_goal; 980 } 981 982 if (copy > *size) 983 copy = *size; 984 985 i = skb_shinfo(skb)->nr_frags; 986 can_coalesce = skb_can_coalesce(skb, i, page, offset); 987 if (!can_coalesce && i >= sysctl_max_skb_frags) { 988 tcp_mark_push(tp, skb); 989 goto new_segment; 990 } 991 if (tcp_downgrade_zcopy_pure(sk, skb) || !sk_wmem_schedule(sk, copy)) 992 return NULL; 993 994 if (can_coalesce) { 995 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 996 } else { 997 get_page(page); 998 skb_fill_page_desc(skb, i, page, offset, copy); 999 } 1000 1001 if (!(flags & MSG_NO_SHARED_FRAGS)) 1002 skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG; 1003 1004 skb->len += copy; 1005 skb->data_len += copy; 1006 skb->truesize += copy; 1007 sk_wmem_queued_add(sk, copy); 1008 sk_mem_charge(sk, copy); 1009 WRITE_ONCE(tp->write_seq, tp->write_seq + copy); 1010 TCP_SKB_CB(skb)->end_seq += copy; 1011 tcp_skb_pcount_set(skb, 0); 1012 1013 *size = copy; 1014 return skb; 1015 } 1016 1017 ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset, 1018 size_t size, int flags) 1019 { 1020 struct tcp_sock *tp = tcp_sk(sk); 1021 int mss_now, size_goal; 1022 int err; 1023 ssize_t copied; 1024 long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); 1025 1026 if (IS_ENABLED(CONFIG_DEBUG_VM) && 1027 WARN_ONCE(!sendpage_ok(page), 1028 "page must not be a Slab one and have page_count > 0")) 1029 return -EINVAL; 1030 1031 /* Wait for a connection to finish. One exception is TCP Fast Open 1032 * (passive side) where data is allowed to be sent before a connection 1033 * is fully established. 1034 */ 1035 if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) && 1036 !tcp_passive_fastopen(sk)) { 1037 err = sk_stream_wait_connect(sk, &timeo); 1038 if (err != 0) 1039 goto out_err; 1040 } 1041 1042 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 1043 1044 mss_now = tcp_send_mss(sk, &size_goal, flags); 1045 copied = 0; 1046 1047 err = -EPIPE; 1048 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) 1049 goto out_err; 1050 1051 while (size > 0) { 1052 struct sk_buff *skb; 1053 size_t copy = size; 1054 1055 skb = tcp_build_frag(sk, size_goal, flags, page, offset, ©); 1056 if (!skb) 1057 goto wait_for_space; 1058 1059 if (!copied) 1060 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH; 1061 1062 copied += copy; 1063 offset += copy; 1064 size -= copy; 1065 if (!size) 1066 goto out; 1067 1068 if (skb->len < size_goal || (flags & MSG_OOB)) 1069 continue; 1070 1071 if (forced_push(tp)) { 1072 tcp_mark_push(tp, skb); 1073 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH); 1074 } else if (skb == tcp_send_head(sk)) 1075 tcp_push_one(sk, mss_now); 1076 continue; 1077 1078 wait_for_space: 1079 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 1080 tcp_push(sk, flags & ~MSG_MORE, mss_now, 1081 TCP_NAGLE_PUSH, size_goal); 1082 1083 err = sk_stream_wait_memory(sk, &timeo); 1084 if (err != 0) 1085 goto do_error; 1086 1087 mss_now = tcp_send_mss(sk, &size_goal, flags); 1088 } 1089 1090 out: 1091 if (copied) { 1092 tcp_tx_timestamp(sk, sk->sk_tsflags); 1093 if (!(flags & MSG_SENDPAGE_NOTLAST)) 1094 tcp_push(sk, flags, mss_now, tp->nonagle, size_goal); 1095 } 1096 return copied; 1097 1098 do_error: 1099 tcp_remove_empty_skb(sk); 1100 if (copied) 1101 goto out; 1102 out_err: 1103 /* make sure we wake any epoll edge trigger waiter */ 1104 if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) { 1105 sk->sk_write_space(sk); 1106 tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED); 1107 } 1108 return sk_stream_error(sk, flags, err); 1109 } 1110 EXPORT_SYMBOL_GPL(do_tcp_sendpages); 1111 1112 int tcp_sendpage_locked(struct sock *sk, struct page *page, int offset, 1113 size_t size, int flags) 1114 { 1115 if (!(sk->sk_route_caps & NETIF_F_SG)) 1116 return sock_no_sendpage_locked(sk, page, offset, size, flags); 1117 1118 tcp_rate_check_app_limited(sk); /* is sending application-limited? */ 1119 1120 return do_tcp_sendpages(sk, page, offset, size, flags); 1121 } 1122 EXPORT_SYMBOL_GPL(tcp_sendpage_locked); 1123 1124 int tcp_sendpage(struct sock *sk, struct page *page, int offset, 1125 size_t size, int flags) 1126 { 1127 int ret; 1128 1129 lock_sock(sk); 1130 ret = tcp_sendpage_locked(sk, page, offset, size, flags); 1131 release_sock(sk); 1132 1133 return ret; 1134 } 1135 EXPORT_SYMBOL(tcp_sendpage); 1136 1137 void tcp_free_fastopen_req(struct tcp_sock *tp) 1138 { 1139 if (tp->fastopen_req) { 1140 kfree(tp->fastopen_req); 1141 tp->fastopen_req = NULL; 1142 } 1143 } 1144 1145 static int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, 1146 int *copied, size_t size, 1147 struct ubuf_info *uarg) 1148 { 1149 struct tcp_sock *tp = tcp_sk(sk); 1150 struct inet_sock *inet = inet_sk(sk); 1151 struct sockaddr *uaddr = msg->msg_name; 1152 int err, flags; 1153 1154 if (!(sock_net(sk)->ipv4.sysctl_tcp_fastopen & TFO_CLIENT_ENABLE) || 1155 (uaddr && msg->msg_namelen >= sizeof(uaddr->sa_family) && 1156 uaddr->sa_family == AF_UNSPEC)) 1157 return -EOPNOTSUPP; 1158 if (tp->fastopen_req) 1159 return -EALREADY; /* Another Fast Open is in progress */ 1160 1161 tp->fastopen_req = kzalloc(sizeof(struct tcp_fastopen_request), 1162 sk->sk_allocation); 1163 if (unlikely(!tp->fastopen_req)) 1164 return -ENOBUFS; 1165 tp->fastopen_req->data = msg; 1166 tp->fastopen_req->size = size; 1167 tp->fastopen_req->uarg = uarg; 1168 1169 if (inet->defer_connect) { 1170 err = tcp_connect(sk); 1171 /* Same failure procedure as in tcp_v4/6_connect */ 1172 if (err) { 1173 tcp_set_state(sk, TCP_CLOSE); 1174 inet->inet_dport = 0; 1175 sk->sk_route_caps = 0; 1176 } 1177 } 1178 flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0; 1179 err = __inet_stream_connect(sk->sk_socket, uaddr, 1180 msg->msg_namelen, flags, 1); 1181 /* fastopen_req could already be freed in __inet_stream_connect 1182 * if the connection times out or gets rst 1183 */ 1184 if (tp->fastopen_req) { 1185 *copied = tp->fastopen_req->copied; 1186 tcp_free_fastopen_req(tp); 1187 inet->defer_connect = 0; 1188 } 1189 return err; 1190 } 1191 1192 int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) 1193 { 1194 struct tcp_sock *tp = tcp_sk(sk); 1195 struct ubuf_info *uarg = NULL; 1196 struct sk_buff *skb; 1197 struct sockcm_cookie sockc; 1198 int flags, err, copied = 0; 1199 int mss_now = 0, size_goal, copied_syn = 0; 1200 int process_backlog = 0; 1201 bool zc = false; 1202 long timeo; 1203 1204 flags = msg->msg_flags; 1205 1206 if (flags & MSG_ZEROCOPY && size && sock_flag(sk, SOCK_ZEROCOPY)) { 1207 skb = tcp_write_queue_tail(sk); 1208 uarg = msg_zerocopy_realloc(sk, size, skb_zcopy(skb)); 1209 if (!uarg) { 1210 err = -ENOBUFS; 1211 goto out_err; 1212 } 1213 1214 zc = sk->sk_route_caps & NETIF_F_SG; 1215 if (!zc) 1216 uarg->zerocopy = 0; 1217 } 1218 1219 if (unlikely(flags & MSG_FASTOPEN || inet_sk(sk)->defer_connect) && 1220 !tp->repair) { 1221 err = tcp_sendmsg_fastopen(sk, msg, &copied_syn, size, uarg); 1222 if (err == -EINPROGRESS && copied_syn > 0) 1223 goto out; 1224 else if (err) 1225 goto out_err; 1226 } 1227 1228 timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); 1229 1230 tcp_rate_check_app_limited(sk); /* is sending application-limited? */ 1231 1232 /* Wait for a connection to finish. One exception is TCP Fast Open 1233 * (passive side) where data is allowed to be sent before a connection 1234 * is fully established. 1235 */ 1236 if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) && 1237 !tcp_passive_fastopen(sk)) { 1238 err = sk_stream_wait_connect(sk, &timeo); 1239 if (err != 0) 1240 goto do_error; 1241 } 1242 1243 if (unlikely(tp->repair)) { 1244 if (tp->repair_queue == TCP_RECV_QUEUE) { 1245 copied = tcp_send_rcvq(sk, msg, size); 1246 goto out_nopush; 1247 } 1248 1249 err = -EINVAL; 1250 if (tp->repair_queue == TCP_NO_QUEUE) 1251 goto out_err; 1252 1253 /* 'common' sending to sendq */ 1254 } 1255 1256 sockcm_init(&sockc, sk); 1257 if (msg->msg_controllen) { 1258 err = sock_cmsg_send(sk, msg, &sockc); 1259 if (unlikely(err)) { 1260 err = -EINVAL; 1261 goto out_err; 1262 } 1263 } 1264 1265 /* This should be in poll */ 1266 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 1267 1268 /* Ok commence sending. */ 1269 copied = 0; 1270 1271 restart: 1272 mss_now = tcp_send_mss(sk, &size_goal, flags); 1273 1274 err = -EPIPE; 1275 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) 1276 goto do_error; 1277 1278 while (msg_data_left(msg)) { 1279 int copy = 0; 1280 1281 skb = tcp_write_queue_tail(sk); 1282 if (skb) 1283 copy = size_goal - skb->len; 1284 1285 if (copy <= 0 || !tcp_skb_can_collapse_to(skb)) { 1286 bool first_skb; 1287 1288 new_segment: 1289 if (!sk_stream_memory_free(sk)) 1290 goto wait_for_space; 1291 1292 if (unlikely(process_backlog >= 16)) { 1293 process_backlog = 0; 1294 if (sk_flush_backlog(sk)) 1295 goto restart; 1296 } 1297 first_skb = tcp_rtx_and_write_queues_empty(sk); 1298 skb = tcp_stream_alloc_skb(sk, 0, sk->sk_allocation, 1299 first_skb); 1300 if (!skb) 1301 goto wait_for_space; 1302 1303 process_backlog++; 1304 1305 tcp_skb_entail(sk, skb); 1306 copy = size_goal; 1307 1308 /* All packets are restored as if they have 1309 * already been sent. skb_mstamp_ns isn't set to 1310 * avoid wrong rtt estimation. 1311 */ 1312 if (tp->repair) 1313 TCP_SKB_CB(skb)->sacked |= TCPCB_REPAIRED; 1314 } 1315 1316 /* Try to append data to the end of skb. */ 1317 if (copy > msg_data_left(msg)) 1318 copy = msg_data_left(msg); 1319 1320 if (!zc) { 1321 bool merge = true; 1322 int i = skb_shinfo(skb)->nr_frags; 1323 struct page_frag *pfrag = sk_page_frag(sk); 1324 1325 if (!sk_page_frag_refill(sk, pfrag)) 1326 goto wait_for_space; 1327 1328 if (!skb_can_coalesce(skb, i, pfrag->page, 1329 pfrag->offset)) { 1330 if (i >= sysctl_max_skb_frags) { 1331 tcp_mark_push(tp, skb); 1332 goto new_segment; 1333 } 1334 merge = false; 1335 } 1336 1337 copy = min_t(int, copy, pfrag->size - pfrag->offset); 1338 1339 if (tcp_downgrade_zcopy_pure(sk, skb) || 1340 !sk_wmem_schedule(sk, copy)) 1341 goto wait_for_space; 1342 1343 err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb, 1344 pfrag->page, 1345 pfrag->offset, 1346 copy); 1347 if (err) 1348 goto do_error; 1349 1350 /* Update the skb. */ 1351 if (merge) { 1352 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 1353 } else { 1354 skb_fill_page_desc(skb, i, pfrag->page, 1355 pfrag->offset, copy); 1356 page_ref_inc(pfrag->page); 1357 } 1358 pfrag->offset += copy; 1359 } else { 1360 /* First append to a fragless skb builds initial 1361 * pure zerocopy skb 1362 */ 1363 if (!skb->len) 1364 skb_shinfo(skb)->flags |= SKBFL_PURE_ZEROCOPY; 1365 1366 if (!skb_zcopy_pure(skb)) { 1367 if (!sk_wmem_schedule(sk, copy)) 1368 goto wait_for_space; 1369 } 1370 1371 err = skb_zerocopy_iter_stream(sk, skb, msg, copy, uarg); 1372 if (err == -EMSGSIZE || err == -EEXIST) { 1373 tcp_mark_push(tp, skb); 1374 goto new_segment; 1375 } 1376 if (err < 0) 1377 goto do_error; 1378 copy = err; 1379 } 1380 1381 if (!copied) 1382 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH; 1383 1384 WRITE_ONCE(tp->write_seq, tp->write_seq + copy); 1385 TCP_SKB_CB(skb)->end_seq += copy; 1386 tcp_skb_pcount_set(skb, 0); 1387 1388 copied += copy; 1389 if (!msg_data_left(msg)) { 1390 if (unlikely(flags & MSG_EOR)) 1391 TCP_SKB_CB(skb)->eor = 1; 1392 goto out; 1393 } 1394 1395 if (skb->len < size_goal || (flags & MSG_OOB) || unlikely(tp->repair)) 1396 continue; 1397 1398 if (forced_push(tp)) { 1399 tcp_mark_push(tp, skb); 1400 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH); 1401 } else if (skb == tcp_send_head(sk)) 1402 tcp_push_one(sk, mss_now); 1403 continue; 1404 1405 wait_for_space: 1406 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 1407 if (copied) 1408 tcp_push(sk, flags & ~MSG_MORE, mss_now, 1409 TCP_NAGLE_PUSH, size_goal); 1410 1411 err = sk_stream_wait_memory(sk, &timeo); 1412 if (err != 0) 1413 goto do_error; 1414 1415 mss_now = tcp_send_mss(sk, &size_goal, flags); 1416 } 1417 1418 out: 1419 if (copied) { 1420 tcp_tx_timestamp(sk, sockc.tsflags); 1421 tcp_push(sk, flags, mss_now, tp->nonagle, size_goal); 1422 } 1423 out_nopush: 1424 net_zcopy_put(uarg); 1425 return copied + copied_syn; 1426 1427 do_error: 1428 tcp_remove_empty_skb(sk); 1429 1430 if (copied + copied_syn) 1431 goto out; 1432 out_err: 1433 net_zcopy_put_abort(uarg, true); 1434 err = sk_stream_error(sk, flags, err); 1435 /* make sure we wake any epoll edge trigger waiter */ 1436 if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) { 1437 sk->sk_write_space(sk); 1438 tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED); 1439 } 1440 return err; 1441 } 1442 EXPORT_SYMBOL_GPL(tcp_sendmsg_locked); 1443 1444 int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) 1445 { 1446 int ret; 1447 1448 lock_sock(sk); 1449 ret = tcp_sendmsg_locked(sk, msg, size); 1450 release_sock(sk); 1451 1452 return ret; 1453 } 1454 EXPORT_SYMBOL(tcp_sendmsg); 1455 1456 /* 1457 * Handle reading urgent data. BSD has very simple semantics for 1458 * this, no blocking and very strange errors 8) 1459 */ 1460 1461 static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags) 1462 { 1463 struct tcp_sock *tp = tcp_sk(sk); 1464 1465 /* No URG data to read. */ 1466 if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data || 1467 tp->urg_data == TCP_URG_READ) 1468 return -EINVAL; /* Yes this is right ! */ 1469 1470 if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE)) 1471 return -ENOTCONN; 1472 1473 if (tp->urg_data & TCP_URG_VALID) { 1474 int err = 0; 1475 char c = tp->urg_data; 1476 1477 if (!(flags & MSG_PEEK)) 1478 WRITE_ONCE(tp->urg_data, TCP_URG_READ); 1479 1480 /* Read urgent data. */ 1481 msg->msg_flags |= MSG_OOB; 1482 1483 if (len > 0) { 1484 if (!(flags & MSG_TRUNC)) 1485 err = memcpy_to_msg(msg, &c, 1); 1486 len = 1; 1487 } else 1488 msg->msg_flags |= MSG_TRUNC; 1489 1490 return err ? -EFAULT : len; 1491 } 1492 1493 if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN)) 1494 return 0; 1495 1496 /* Fixed the recv(..., MSG_OOB) behaviour. BSD docs and 1497 * the available implementations agree in this case: 1498 * this call should never block, independent of the 1499 * blocking state of the socket. 1500 * Mike <pall@rz.uni-karlsruhe.de> 1501 */ 1502 return -EAGAIN; 1503 } 1504 1505 static int tcp_peek_sndq(struct sock *sk, struct msghdr *msg, int len) 1506 { 1507 struct sk_buff *skb; 1508 int copied = 0, err = 0; 1509 1510 /* XXX -- need to support SO_PEEK_OFF */ 1511 1512 skb_rbtree_walk(skb, &sk->tcp_rtx_queue) { 1513 err = skb_copy_datagram_msg(skb, 0, msg, skb->len); 1514 if (err) 1515 return err; 1516 copied += skb->len; 1517 } 1518 1519 skb_queue_walk(&sk->sk_write_queue, skb) { 1520 err = skb_copy_datagram_msg(skb, 0, msg, skb->len); 1521 if (err) 1522 break; 1523 1524 copied += skb->len; 1525 } 1526 1527 return err ?: copied; 1528 } 1529 1530 /* Clean up the receive buffer for full frames taken by the user, 1531 * then send an ACK if necessary. COPIED is the number of bytes 1532 * tcp_recvmsg has given to the user so far, it speeds up the 1533 * calculation of whether or not we must ACK for the sake of 1534 * a window update. 1535 */ 1536 void tcp_cleanup_rbuf(struct sock *sk, int copied) 1537 { 1538 struct tcp_sock *tp = tcp_sk(sk); 1539 bool time_to_ack = false; 1540 1541 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue); 1542 1543 WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq), 1544 "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n", 1545 tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt); 1546 1547 if (inet_csk_ack_scheduled(sk)) { 1548 const struct inet_connection_sock *icsk = inet_csk(sk); 1549 1550 if (/* Once-per-two-segments ACK was not sent by tcp_input.c */ 1551 tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss || 1552 /* 1553 * If this read emptied read buffer, we send ACK, if 1554 * connection is not bidirectional, user drained 1555 * receive buffer and there was a small segment 1556 * in queue. 1557 */ 1558 (copied > 0 && 1559 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) || 1560 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) && 1561 !inet_csk_in_pingpong_mode(sk))) && 1562 !atomic_read(&sk->sk_rmem_alloc))) 1563 time_to_ack = true; 1564 } 1565 1566 /* We send an ACK if we can now advertise a non-zero window 1567 * which has been raised "significantly". 1568 * 1569 * Even if window raised up to infinity, do not send window open ACK 1570 * in states, where we will not receive more. It is useless. 1571 */ 1572 if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) { 1573 __u32 rcv_window_now = tcp_receive_window(tp); 1574 1575 /* Optimize, __tcp_select_window() is not cheap. */ 1576 if (2*rcv_window_now <= tp->window_clamp) { 1577 __u32 new_window = __tcp_select_window(sk); 1578 1579 /* Send ACK now, if this read freed lots of space 1580 * in our buffer. Certainly, new_window is new window. 1581 * We can advertise it now, if it is not less than current one. 1582 * "Lots" means "at least twice" here. 1583 */ 1584 if (new_window && new_window >= 2 * rcv_window_now) 1585 time_to_ack = true; 1586 } 1587 } 1588 if (time_to_ack) 1589 tcp_send_ack(sk); 1590 } 1591 1592 void __sk_defer_free_flush(struct sock *sk) 1593 { 1594 struct llist_node *head; 1595 struct sk_buff *skb, *n; 1596 1597 head = llist_del_all(&sk->defer_list); 1598 llist_for_each_entry_safe(skb, n, head, ll_node) { 1599 prefetch(n); 1600 skb_mark_not_on_list(skb); 1601 __kfree_skb(skb); 1602 } 1603 } 1604 EXPORT_SYMBOL(__sk_defer_free_flush); 1605 1606 static void tcp_eat_recv_skb(struct sock *sk, struct sk_buff *skb) 1607 { 1608 __skb_unlink(skb, &sk->sk_receive_queue); 1609 if (likely(skb->destructor == sock_rfree)) { 1610 sock_rfree(skb); 1611 skb->destructor = NULL; 1612 skb->sk = NULL; 1613 if (!skb_queue_empty(&sk->sk_receive_queue) || 1614 !llist_empty(&sk->defer_list)) { 1615 llist_add(&skb->ll_node, &sk->defer_list); 1616 return; 1617 } 1618 } 1619 __kfree_skb(skb); 1620 } 1621 1622 static struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off) 1623 { 1624 struct sk_buff *skb; 1625 u32 offset; 1626 1627 while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) { 1628 offset = seq - TCP_SKB_CB(skb)->seq; 1629 if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) { 1630 pr_err_once("%s: found a SYN, please report !\n", __func__); 1631 offset--; 1632 } 1633 if (offset < skb->len || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) { 1634 *off = offset; 1635 return skb; 1636 } 1637 /* This looks weird, but this can happen if TCP collapsing 1638 * splitted a fat GRO packet, while we released socket lock 1639 * in skb_splice_bits() 1640 */ 1641 tcp_eat_recv_skb(sk, skb); 1642 } 1643 return NULL; 1644 } 1645 1646 /* 1647 * This routine provides an alternative to tcp_recvmsg() for routines 1648 * that would like to handle copying from skbuffs directly in 'sendfile' 1649 * fashion. 1650 * Note: 1651 * - It is assumed that the socket was locked by the caller. 1652 * - The routine does not block. 1653 * - At present, there is no support for reading OOB data 1654 * or for 'peeking' the socket using this routine 1655 * (although both would be easy to implement). 1656 */ 1657 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, 1658 sk_read_actor_t recv_actor) 1659 { 1660 struct sk_buff *skb; 1661 struct tcp_sock *tp = tcp_sk(sk); 1662 u32 seq = tp->copied_seq; 1663 u32 offset; 1664 int copied = 0; 1665 1666 if (sk->sk_state == TCP_LISTEN) 1667 return -ENOTCONN; 1668 while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) { 1669 if (offset < skb->len) { 1670 int used; 1671 size_t len; 1672 1673 len = skb->len - offset; 1674 /* Stop reading if we hit a patch of urgent data */ 1675 if (unlikely(tp->urg_data)) { 1676 u32 urg_offset = tp->urg_seq - seq; 1677 if (urg_offset < len) 1678 len = urg_offset; 1679 if (!len) 1680 break; 1681 } 1682 used = recv_actor(desc, skb, offset, len); 1683 if (used <= 0) { 1684 if (!copied) 1685 copied = used; 1686 break; 1687 } 1688 if (WARN_ON_ONCE(used > len)) 1689 used = len; 1690 seq += used; 1691 copied += used; 1692 offset += used; 1693 1694 /* If recv_actor drops the lock (e.g. TCP splice 1695 * receive) the skb pointer might be invalid when 1696 * getting here: tcp_collapse might have deleted it 1697 * while aggregating skbs from the socket queue. 1698 */ 1699 skb = tcp_recv_skb(sk, seq - 1, &offset); 1700 if (!skb) 1701 break; 1702 /* TCP coalescing might have appended data to the skb. 1703 * Try to splice more frags 1704 */ 1705 if (offset + 1 != skb->len) 1706 continue; 1707 } 1708 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) { 1709 tcp_eat_recv_skb(sk, skb); 1710 ++seq; 1711 break; 1712 } 1713 tcp_eat_recv_skb(sk, skb); 1714 if (!desc->count) 1715 break; 1716 WRITE_ONCE(tp->copied_seq, seq); 1717 } 1718 WRITE_ONCE(tp->copied_seq, seq); 1719 1720 tcp_rcv_space_adjust(sk); 1721 1722 /* Clean up data we have read: This will do ACK frames. */ 1723 if (copied > 0) { 1724 tcp_recv_skb(sk, seq, &offset); 1725 tcp_cleanup_rbuf(sk, copied); 1726 } 1727 return copied; 1728 } 1729 EXPORT_SYMBOL(tcp_read_sock); 1730 1731 int tcp_peek_len(struct socket *sock) 1732 { 1733 return tcp_inq(sock->sk); 1734 } 1735 EXPORT_SYMBOL(tcp_peek_len); 1736 1737 /* Make sure sk_rcvbuf is big enough to satisfy SO_RCVLOWAT hint */ 1738 int tcp_set_rcvlowat(struct sock *sk, int val) 1739 { 1740 int cap; 1741 1742 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK) 1743 cap = sk->sk_rcvbuf >> 1; 1744 else 1745 cap = sock_net(sk)->ipv4.sysctl_tcp_rmem[2] >> 1; 1746 val = min(val, cap); 1747 WRITE_ONCE(sk->sk_rcvlowat, val ? : 1); 1748 1749 /* Check if we need to signal EPOLLIN right now */ 1750 tcp_data_ready(sk); 1751 1752 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK) 1753 return 0; 1754 1755 val <<= 1; 1756 if (val > sk->sk_rcvbuf) { 1757 WRITE_ONCE(sk->sk_rcvbuf, val); 1758 tcp_sk(sk)->window_clamp = tcp_win_from_space(sk, val); 1759 } 1760 return 0; 1761 } 1762 EXPORT_SYMBOL(tcp_set_rcvlowat); 1763 1764 void tcp_update_recv_tstamps(struct sk_buff *skb, 1765 struct scm_timestamping_internal *tss) 1766 { 1767 if (skb->tstamp) 1768 tss->ts[0] = ktime_to_timespec64(skb->tstamp); 1769 else 1770 tss->ts[0] = (struct timespec64) {0}; 1771 1772 if (skb_hwtstamps(skb)->hwtstamp) 1773 tss->ts[2] = ktime_to_timespec64(skb_hwtstamps(skb)->hwtstamp); 1774 else 1775 tss->ts[2] = (struct timespec64) {0}; 1776 } 1777 1778 #ifdef CONFIG_MMU 1779 static const struct vm_operations_struct tcp_vm_ops = { 1780 }; 1781 1782 int tcp_mmap(struct file *file, struct socket *sock, 1783 struct vm_area_struct *vma) 1784 { 1785 if (vma->vm_flags & (VM_WRITE | VM_EXEC)) 1786 return -EPERM; 1787 vma->vm_flags &= ~(VM_MAYWRITE | VM_MAYEXEC); 1788 1789 /* Instruct vm_insert_page() to not mmap_read_lock(mm) */ 1790 vma->vm_flags |= VM_MIXEDMAP; 1791 1792 vma->vm_ops = &tcp_vm_ops; 1793 return 0; 1794 } 1795 EXPORT_SYMBOL(tcp_mmap); 1796 1797 static skb_frag_t *skb_advance_to_frag(struct sk_buff *skb, u32 offset_skb, 1798 u32 *offset_frag) 1799 { 1800 skb_frag_t *frag; 1801 1802 if (unlikely(offset_skb >= skb->len)) 1803 return NULL; 1804 1805 offset_skb -= skb_headlen(skb); 1806 if ((int)offset_skb < 0 || skb_has_frag_list(skb)) 1807 return NULL; 1808 1809 frag = skb_shinfo(skb)->frags; 1810 while (offset_skb) { 1811 if (skb_frag_size(frag) > offset_skb) { 1812 *offset_frag = offset_skb; 1813 return frag; 1814 } 1815 offset_skb -= skb_frag_size(frag); 1816 ++frag; 1817 } 1818 *offset_frag = 0; 1819 return frag; 1820 } 1821 1822 static bool can_map_frag(const skb_frag_t *frag) 1823 { 1824 return skb_frag_size(frag) == PAGE_SIZE && !skb_frag_off(frag); 1825 } 1826 1827 static int find_next_mappable_frag(const skb_frag_t *frag, 1828 int remaining_in_skb) 1829 { 1830 int offset = 0; 1831 1832 if (likely(can_map_frag(frag))) 1833 return 0; 1834 1835 while (offset < remaining_in_skb && !can_map_frag(frag)) { 1836 offset += skb_frag_size(frag); 1837 ++frag; 1838 } 1839 return offset; 1840 } 1841 1842 static void tcp_zerocopy_set_hint_for_skb(struct sock *sk, 1843 struct tcp_zerocopy_receive *zc, 1844 struct sk_buff *skb, u32 offset) 1845 { 1846 u32 frag_offset, partial_frag_remainder = 0; 1847 int mappable_offset; 1848 skb_frag_t *frag; 1849 1850 /* worst case: skip to next skb. try to improve on this case below */ 1851 zc->recv_skip_hint = skb->len - offset; 1852 1853 /* Find the frag containing this offset (and how far into that frag) */ 1854 frag = skb_advance_to_frag(skb, offset, &frag_offset); 1855 if (!frag) 1856 return; 1857 1858 if (frag_offset) { 1859 struct skb_shared_info *info = skb_shinfo(skb); 1860 1861 /* We read part of the last frag, must recvmsg() rest of skb. */ 1862 if (frag == &info->frags[info->nr_frags - 1]) 1863 return; 1864 1865 /* Else, we must at least read the remainder in this frag. */ 1866 partial_frag_remainder = skb_frag_size(frag) - frag_offset; 1867 zc->recv_skip_hint -= partial_frag_remainder; 1868 ++frag; 1869 } 1870 1871 /* partial_frag_remainder: If part way through a frag, must read rest. 1872 * mappable_offset: Bytes till next mappable frag, *not* counting bytes 1873 * in partial_frag_remainder. 1874 */ 1875 mappable_offset = find_next_mappable_frag(frag, zc->recv_skip_hint); 1876 zc->recv_skip_hint = mappable_offset + partial_frag_remainder; 1877 } 1878 1879 static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len, 1880 int flags, struct scm_timestamping_internal *tss, 1881 int *cmsg_flags); 1882 static int receive_fallback_to_copy(struct sock *sk, 1883 struct tcp_zerocopy_receive *zc, int inq, 1884 struct scm_timestamping_internal *tss) 1885 { 1886 unsigned long copy_address = (unsigned long)zc->copybuf_address; 1887 struct msghdr msg = {}; 1888 struct iovec iov; 1889 int err; 1890 1891 zc->length = 0; 1892 zc->recv_skip_hint = 0; 1893 1894 if (copy_address != zc->copybuf_address) 1895 return -EINVAL; 1896 1897 err = import_single_range(READ, (void __user *)copy_address, 1898 inq, &iov, &msg.msg_iter); 1899 if (err) 1900 return err; 1901 1902 err = tcp_recvmsg_locked(sk, &msg, inq, MSG_DONTWAIT, 1903 tss, &zc->msg_flags); 1904 if (err < 0) 1905 return err; 1906 1907 zc->copybuf_len = err; 1908 if (likely(zc->copybuf_len)) { 1909 struct sk_buff *skb; 1910 u32 offset; 1911 1912 skb = tcp_recv_skb(sk, tcp_sk(sk)->copied_seq, &offset); 1913 if (skb) 1914 tcp_zerocopy_set_hint_for_skb(sk, zc, skb, offset); 1915 } 1916 return 0; 1917 } 1918 1919 static int tcp_copy_straggler_data(struct tcp_zerocopy_receive *zc, 1920 struct sk_buff *skb, u32 copylen, 1921 u32 *offset, u32 *seq) 1922 { 1923 unsigned long copy_address = (unsigned long)zc->copybuf_address; 1924 struct msghdr msg = {}; 1925 struct iovec iov; 1926 int err; 1927 1928 if (copy_address != zc->copybuf_address) 1929 return -EINVAL; 1930 1931 err = import_single_range(READ, (void __user *)copy_address, 1932 copylen, &iov, &msg.msg_iter); 1933 if (err) 1934 return err; 1935 err = skb_copy_datagram_msg(skb, *offset, &msg, copylen); 1936 if (err) 1937 return err; 1938 zc->recv_skip_hint -= copylen; 1939 *offset += copylen; 1940 *seq += copylen; 1941 return (__s32)copylen; 1942 } 1943 1944 static int tcp_zc_handle_leftover(struct tcp_zerocopy_receive *zc, 1945 struct sock *sk, 1946 struct sk_buff *skb, 1947 u32 *seq, 1948 s32 copybuf_len, 1949 struct scm_timestamping_internal *tss) 1950 { 1951 u32 offset, copylen = min_t(u32, copybuf_len, zc->recv_skip_hint); 1952 1953 if (!copylen) 1954 return 0; 1955 /* skb is null if inq < PAGE_SIZE. */ 1956 if (skb) { 1957 offset = *seq - TCP_SKB_CB(skb)->seq; 1958 } else { 1959 skb = tcp_recv_skb(sk, *seq, &offset); 1960 if (TCP_SKB_CB(skb)->has_rxtstamp) { 1961 tcp_update_recv_tstamps(skb, tss); 1962 zc->msg_flags |= TCP_CMSG_TS; 1963 } 1964 } 1965 1966 zc->copybuf_len = tcp_copy_straggler_data(zc, skb, copylen, &offset, 1967 seq); 1968 return zc->copybuf_len < 0 ? 0 : copylen; 1969 } 1970 1971 static int tcp_zerocopy_vm_insert_batch_error(struct vm_area_struct *vma, 1972 struct page **pending_pages, 1973 unsigned long pages_remaining, 1974 unsigned long *address, 1975 u32 *length, 1976 u32 *seq, 1977 struct tcp_zerocopy_receive *zc, 1978 u32 total_bytes_to_map, 1979 int err) 1980 { 1981 /* At least one page did not map. Try zapping if we skipped earlier. */ 1982 if (err == -EBUSY && 1983 zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT) { 1984 u32 maybe_zap_len; 1985 1986 maybe_zap_len = total_bytes_to_map - /* All bytes to map */ 1987 *length + /* Mapped or pending */ 1988 (pages_remaining * PAGE_SIZE); /* Failed map. */ 1989 zap_page_range(vma, *address, maybe_zap_len); 1990 err = 0; 1991 } 1992 1993 if (!err) { 1994 unsigned long leftover_pages = pages_remaining; 1995 int bytes_mapped; 1996 1997 /* We called zap_page_range, try to reinsert. */ 1998 err = vm_insert_pages(vma, *address, 1999 pending_pages, 2000 &pages_remaining); 2001 bytes_mapped = PAGE_SIZE * (leftover_pages - pages_remaining); 2002 *seq += bytes_mapped; 2003 *address += bytes_mapped; 2004 } 2005 if (err) { 2006 /* Either we were unable to zap, OR we zapped, retried an 2007 * insert, and still had an issue. Either ways, pages_remaining 2008 * is the number of pages we were unable to map, and we unroll 2009 * some state we speculatively touched before. 2010 */ 2011 const int bytes_not_mapped = PAGE_SIZE * pages_remaining; 2012 2013 *length -= bytes_not_mapped; 2014 zc->recv_skip_hint += bytes_not_mapped; 2015 } 2016 return err; 2017 } 2018 2019 static int tcp_zerocopy_vm_insert_batch(struct vm_area_struct *vma, 2020 struct page **pages, 2021 unsigned int pages_to_map, 2022 unsigned long *address, 2023 u32 *length, 2024 u32 *seq, 2025 struct tcp_zerocopy_receive *zc, 2026 u32 total_bytes_to_map) 2027 { 2028 unsigned long pages_remaining = pages_to_map; 2029 unsigned int pages_mapped; 2030 unsigned int bytes_mapped; 2031 int err; 2032 2033 err = vm_insert_pages(vma, *address, pages, &pages_remaining); 2034 pages_mapped = pages_to_map - (unsigned int)pages_remaining; 2035 bytes_mapped = PAGE_SIZE * pages_mapped; 2036 /* Even if vm_insert_pages fails, it may have partially succeeded in 2037 * mapping (some but not all of the pages). 2038 */ 2039 *seq += bytes_mapped; 2040 *address += bytes_mapped; 2041 2042 if (likely(!err)) 2043 return 0; 2044 2045 /* Error: maybe zap and retry + rollback state for failed inserts. */ 2046 return tcp_zerocopy_vm_insert_batch_error(vma, pages + pages_mapped, 2047 pages_remaining, address, length, seq, zc, total_bytes_to_map, 2048 err); 2049 } 2050 2051 #define TCP_VALID_ZC_MSG_FLAGS (TCP_CMSG_TS) 2052 static void tcp_zc_finalize_rx_tstamp(struct sock *sk, 2053 struct tcp_zerocopy_receive *zc, 2054 struct scm_timestamping_internal *tss) 2055 { 2056 unsigned long msg_control_addr; 2057 struct msghdr cmsg_dummy; 2058 2059 msg_control_addr = (unsigned long)zc->msg_control; 2060 cmsg_dummy.msg_control = (void *)msg_control_addr; 2061 cmsg_dummy.msg_controllen = 2062 (__kernel_size_t)zc->msg_controllen; 2063 cmsg_dummy.msg_flags = in_compat_syscall() 2064 ? MSG_CMSG_COMPAT : 0; 2065 cmsg_dummy.msg_control_is_user = true; 2066 zc->msg_flags = 0; 2067 if (zc->msg_control == msg_control_addr && 2068 zc->msg_controllen == cmsg_dummy.msg_controllen) { 2069 tcp_recv_timestamp(&cmsg_dummy, sk, tss); 2070 zc->msg_control = (__u64) 2071 ((uintptr_t)cmsg_dummy.msg_control); 2072 zc->msg_controllen = 2073 (__u64)cmsg_dummy.msg_controllen; 2074 zc->msg_flags = (__u32)cmsg_dummy.msg_flags; 2075 } 2076 } 2077 2078 #define TCP_ZEROCOPY_PAGE_BATCH_SIZE 32 2079 static int tcp_zerocopy_receive(struct sock *sk, 2080 struct tcp_zerocopy_receive *zc, 2081 struct scm_timestamping_internal *tss) 2082 { 2083 u32 length = 0, offset, vma_len, avail_len, copylen = 0; 2084 unsigned long address = (unsigned long)zc->address; 2085 struct page *pages[TCP_ZEROCOPY_PAGE_BATCH_SIZE]; 2086 s32 copybuf_len = zc->copybuf_len; 2087 struct tcp_sock *tp = tcp_sk(sk); 2088 const skb_frag_t *frags = NULL; 2089 unsigned int pages_to_map = 0; 2090 struct vm_area_struct *vma; 2091 struct sk_buff *skb = NULL; 2092 u32 seq = tp->copied_seq; 2093 u32 total_bytes_to_map; 2094 int inq = tcp_inq(sk); 2095 int ret; 2096 2097 zc->copybuf_len = 0; 2098 zc->msg_flags = 0; 2099 2100 if (address & (PAGE_SIZE - 1) || address != zc->address) 2101 return -EINVAL; 2102 2103 if (sk->sk_state == TCP_LISTEN) 2104 return -ENOTCONN; 2105 2106 sock_rps_record_flow(sk); 2107 2108 if (inq && inq <= copybuf_len) 2109 return receive_fallback_to_copy(sk, zc, inq, tss); 2110 2111 if (inq < PAGE_SIZE) { 2112 zc->length = 0; 2113 zc->recv_skip_hint = inq; 2114 if (!inq && sock_flag(sk, SOCK_DONE)) 2115 return -EIO; 2116 return 0; 2117 } 2118 2119 mmap_read_lock(current->mm); 2120 2121 vma = vma_lookup(current->mm, address); 2122 if (!vma || vma->vm_ops != &tcp_vm_ops) { 2123 mmap_read_unlock(current->mm); 2124 return -EINVAL; 2125 } 2126 vma_len = min_t(unsigned long, zc->length, vma->vm_end - address); 2127 avail_len = min_t(u32, vma_len, inq); 2128 total_bytes_to_map = avail_len & ~(PAGE_SIZE - 1); 2129 if (total_bytes_to_map) { 2130 if (!(zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT)) 2131 zap_page_range(vma, address, total_bytes_to_map); 2132 zc->length = total_bytes_to_map; 2133 zc->recv_skip_hint = 0; 2134 } else { 2135 zc->length = avail_len; 2136 zc->recv_skip_hint = avail_len; 2137 } 2138 ret = 0; 2139 while (length + PAGE_SIZE <= zc->length) { 2140 int mappable_offset; 2141 struct page *page; 2142 2143 if (zc->recv_skip_hint < PAGE_SIZE) { 2144 u32 offset_frag; 2145 2146 if (skb) { 2147 if (zc->recv_skip_hint > 0) 2148 break; 2149 skb = skb->next; 2150 offset = seq - TCP_SKB_CB(skb)->seq; 2151 } else { 2152 skb = tcp_recv_skb(sk, seq, &offset); 2153 } 2154 2155 if (TCP_SKB_CB(skb)->has_rxtstamp) { 2156 tcp_update_recv_tstamps(skb, tss); 2157 zc->msg_flags |= TCP_CMSG_TS; 2158 } 2159 zc->recv_skip_hint = skb->len - offset; 2160 frags = skb_advance_to_frag(skb, offset, &offset_frag); 2161 if (!frags || offset_frag) 2162 break; 2163 } 2164 2165 mappable_offset = find_next_mappable_frag(frags, 2166 zc->recv_skip_hint); 2167 if (mappable_offset) { 2168 zc->recv_skip_hint = mappable_offset; 2169 break; 2170 } 2171 page = skb_frag_page(frags); 2172 prefetchw(page); 2173 pages[pages_to_map++] = page; 2174 length += PAGE_SIZE; 2175 zc->recv_skip_hint -= PAGE_SIZE; 2176 frags++; 2177 if (pages_to_map == TCP_ZEROCOPY_PAGE_BATCH_SIZE || 2178 zc->recv_skip_hint < PAGE_SIZE) { 2179 /* Either full batch, or we're about to go to next skb 2180 * (and we cannot unroll failed ops across skbs). 2181 */ 2182 ret = tcp_zerocopy_vm_insert_batch(vma, pages, 2183 pages_to_map, 2184 &address, &length, 2185 &seq, zc, 2186 total_bytes_to_map); 2187 if (ret) 2188 goto out; 2189 pages_to_map = 0; 2190 } 2191 } 2192 if (pages_to_map) { 2193 ret = tcp_zerocopy_vm_insert_batch(vma, pages, pages_to_map, 2194 &address, &length, &seq, 2195 zc, total_bytes_to_map); 2196 } 2197 out: 2198 mmap_read_unlock(current->mm); 2199 /* Try to copy straggler data. */ 2200 if (!ret) 2201 copylen = tcp_zc_handle_leftover(zc, sk, skb, &seq, copybuf_len, tss); 2202 2203 if (length + copylen) { 2204 WRITE_ONCE(tp->copied_seq, seq); 2205 tcp_rcv_space_adjust(sk); 2206 2207 /* Clean up data we have read: This will do ACK frames. */ 2208 tcp_recv_skb(sk, seq, &offset); 2209 tcp_cleanup_rbuf(sk, length + copylen); 2210 ret = 0; 2211 if (length == zc->length) 2212 zc->recv_skip_hint = 0; 2213 } else { 2214 if (!zc->recv_skip_hint && sock_flag(sk, SOCK_DONE)) 2215 ret = -EIO; 2216 } 2217 zc->length = length; 2218 return ret; 2219 } 2220 #endif 2221 2222 /* Similar to __sock_recv_timestamp, but does not require an skb */ 2223 void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk, 2224 struct scm_timestamping_internal *tss) 2225 { 2226 int new_tstamp = sock_flag(sk, SOCK_TSTAMP_NEW); 2227 bool has_timestamping = false; 2228 2229 if (tss->ts[0].tv_sec || tss->ts[0].tv_nsec) { 2230 if (sock_flag(sk, SOCK_RCVTSTAMP)) { 2231 if (sock_flag(sk, SOCK_RCVTSTAMPNS)) { 2232 if (new_tstamp) { 2233 struct __kernel_timespec kts = { 2234 .tv_sec = tss->ts[0].tv_sec, 2235 .tv_nsec = tss->ts[0].tv_nsec, 2236 }; 2237 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_NEW, 2238 sizeof(kts), &kts); 2239 } else { 2240 struct __kernel_old_timespec ts_old = { 2241 .tv_sec = tss->ts[0].tv_sec, 2242 .tv_nsec = tss->ts[0].tv_nsec, 2243 }; 2244 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_OLD, 2245 sizeof(ts_old), &ts_old); 2246 } 2247 } else { 2248 if (new_tstamp) { 2249 struct __kernel_sock_timeval stv = { 2250 .tv_sec = tss->ts[0].tv_sec, 2251 .tv_usec = tss->ts[0].tv_nsec / 1000, 2252 }; 2253 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_NEW, 2254 sizeof(stv), &stv); 2255 } else { 2256 struct __kernel_old_timeval tv = { 2257 .tv_sec = tss->ts[0].tv_sec, 2258 .tv_usec = tss->ts[0].tv_nsec / 1000, 2259 }; 2260 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_OLD, 2261 sizeof(tv), &tv); 2262 } 2263 } 2264 } 2265 2266 if (sk->sk_tsflags & SOF_TIMESTAMPING_SOFTWARE) 2267 has_timestamping = true; 2268 else 2269 tss->ts[0] = (struct timespec64) {0}; 2270 } 2271 2272 if (tss->ts[2].tv_sec || tss->ts[2].tv_nsec) { 2273 if (sk->sk_tsflags & SOF_TIMESTAMPING_RAW_HARDWARE) 2274 has_timestamping = true; 2275 else 2276 tss->ts[2] = (struct timespec64) {0}; 2277 } 2278 2279 if (has_timestamping) { 2280 tss->ts[1] = (struct timespec64) {0}; 2281 if (sock_flag(sk, SOCK_TSTAMP_NEW)) 2282 put_cmsg_scm_timestamping64(msg, tss); 2283 else 2284 put_cmsg_scm_timestamping(msg, tss); 2285 } 2286 } 2287 2288 static int tcp_inq_hint(struct sock *sk) 2289 { 2290 const struct tcp_sock *tp = tcp_sk(sk); 2291 u32 copied_seq = READ_ONCE(tp->copied_seq); 2292 u32 rcv_nxt = READ_ONCE(tp->rcv_nxt); 2293 int inq; 2294 2295 inq = rcv_nxt - copied_seq; 2296 if (unlikely(inq < 0 || copied_seq != READ_ONCE(tp->copied_seq))) { 2297 lock_sock(sk); 2298 inq = tp->rcv_nxt - tp->copied_seq; 2299 release_sock(sk); 2300 } 2301 /* After receiving a FIN, tell the user-space to continue reading 2302 * by returning a non-zero inq. 2303 */ 2304 if (inq == 0 && sock_flag(sk, SOCK_DONE)) 2305 inq = 1; 2306 return inq; 2307 } 2308 2309 /* 2310 * This routine copies from a sock struct into the user buffer. 2311 * 2312 * Technical note: in 2.3 we work on _locked_ socket, so that 2313 * tricks with *seq access order and skb->users are not required. 2314 * Probably, code can be easily improved even more. 2315 */ 2316 2317 static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len, 2318 int flags, struct scm_timestamping_internal *tss, 2319 int *cmsg_flags) 2320 { 2321 struct tcp_sock *tp = tcp_sk(sk); 2322 int copied = 0; 2323 u32 peek_seq; 2324 u32 *seq; 2325 unsigned long used; 2326 int err; 2327 int target; /* Read at least this many bytes */ 2328 long timeo; 2329 struct sk_buff *skb, *last; 2330 u32 urg_hole = 0; 2331 2332 err = -ENOTCONN; 2333 if (sk->sk_state == TCP_LISTEN) 2334 goto out; 2335 2336 if (tp->recvmsg_inq) 2337 *cmsg_flags = TCP_CMSG_INQ; 2338 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 2339 2340 /* Urgent data needs to be handled specially. */ 2341 if (flags & MSG_OOB) 2342 goto recv_urg; 2343 2344 if (unlikely(tp->repair)) { 2345 err = -EPERM; 2346 if (!(flags & MSG_PEEK)) 2347 goto out; 2348 2349 if (tp->repair_queue == TCP_SEND_QUEUE) 2350 goto recv_sndq; 2351 2352 err = -EINVAL; 2353 if (tp->repair_queue == TCP_NO_QUEUE) 2354 goto out; 2355 2356 /* 'common' recv queue MSG_PEEK-ing */ 2357 } 2358 2359 seq = &tp->copied_seq; 2360 if (flags & MSG_PEEK) { 2361 peek_seq = tp->copied_seq; 2362 seq = &peek_seq; 2363 } 2364 2365 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 2366 2367 do { 2368 u32 offset; 2369 2370 /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */ 2371 if (unlikely(tp->urg_data) && tp->urg_seq == *seq) { 2372 if (copied) 2373 break; 2374 if (signal_pending(current)) { 2375 copied = timeo ? sock_intr_errno(timeo) : -EAGAIN; 2376 break; 2377 } 2378 } 2379 2380 /* Next get a buffer. */ 2381 2382 last = skb_peek_tail(&sk->sk_receive_queue); 2383 skb_queue_walk(&sk->sk_receive_queue, skb) { 2384 last = skb; 2385 /* Now that we have two receive queues this 2386 * shouldn't happen. 2387 */ 2388 if (WARN(before(*seq, TCP_SKB_CB(skb)->seq), 2389 "TCP recvmsg seq # bug: copied %X, seq %X, rcvnxt %X, fl %X\n", 2390 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, 2391 flags)) 2392 break; 2393 2394 offset = *seq - TCP_SKB_CB(skb)->seq; 2395 if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) { 2396 pr_err_once("%s: found a SYN, please report !\n", __func__); 2397 offset--; 2398 } 2399 if (offset < skb->len) 2400 goto found_ok_skb; 2401 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 2402 goto found_fin_ok; 2403 WARN(!(flags & MSG_PEEK), 2404 "TCP recvmsg seq # bug 2: copied %X, seq %X, rcvnxt %X, fl %X\n", 2405 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags); 2406 } 2407 2408 /* Well, if we have backlog, try to process it now yet. */ 2409 2410 if (copied >= target && !READ_ONCE(sk->sk_backlog.tail)) 2411 break; 2412 2413 if (copied) { 2414 if (!timeo || 2415 sk->sk_err || 2416 sk->sk_state == TCP_CLOSE || 2417 (sk->sk_shutdown & RCV_SHUTDOWN) || 2418 signal_pending(current)) 2419 break; 2420 } else { 2421 if (sock_flag(sk, SOCK_DONE)) 2422 break; 2423 2424 if (sk->sk_err) { 2425 copied = sock_error(sk); 2426 break; 2427 } 2428 2429 if (sk->sk_shutdown & RCV_SHUTDOWN) 2430 break; 2431 2432 if (sk->sk_state == TCP_CLOSE) { 2433 /* This occurs when user tries to read 2434 * from never connected socket. 2435 */ 2436 copied = -ENOTCONN; 2437 break; 2438 } 2439 2440 if (!timeo) { 2441 copied = -EAGAIN; 2442 break; 2443 } 2444 2445 if (signal_pending(current)) { 2446 copied = sock_intr_errno(timeo); 2447 break; 2448 } 2449 } 2450 2451 if (copied >= target) { 2452 /* Do not sleep, just process backlog. */ 2453 __sk_flush_backlog(sk); 2454 } else { 2455 tcp_cleanup_rbuf(sk, copied); 2456 sk_defer_free_flush(sk); 2457 sk_wait_data(sk, &timeo, last); 2458 } 2459 2460 if ((flags & MSG_PEEK) && 2461 (peek_seq - copied - urg_hole != tp->copied_seq)) { 2462 net_dbg_ratelimited("TCP(%s:%d): Application bug, race in MSG_PEEK\n", 2463 current->comm, 2464 task_pid_nr(current)); 2465 peek_seq = tp->copied_seq; 2466 } 2467 continue; 2468 2469 found_ok_skb: 2470 /* Ok so how much can we use? */ 2471 used = skb->len - offset; 2472 if (len < used) 2473 used = len; 2474 2475 /* Do we have urgent data here? */ 2476 if (unlikely(tp->urg_data)) { 2477 u32 urg_offset = tp->urg_seq - *seq; 2478 if (urg_offset < used) { 2479 if (!urg_offset) { 2480 if (!sock_flag(sk, SOCK_URGINLINE)) { 2481 WRITE_ONCE(*seq, *seq + 1); 2482 urg_hole++; 2483 offset++; 2484 used--; 2485 if (!used) 2486 goto skip_copy; 2487 } 2488 } else 2489 used = urg_offset; 2490 } 2491 } 2492 2493 if (!(flags & MSG_TRUNC)) { 2494 err = skb_copy_datagram_msg(skb, offset, msg, used); 2495 if (err) { 2496 /* Exception. Bailout! */ 2497 if (!copied) 2498 copied = -EFAULT; 2499 break; 2500 } 2501 } 2502 2503 WRITE_ONCE(*seq, *seq + used); 2504 copied += used; 2505 len -= used; 2506 2507 tcp_rcv_space_adjust(sk); 2508 2509 skip_copy: 2510 if (unlikely(tp->urg_data) && after(tp->copied_seq, tp->urg_seq)) { 2511 WRITE_ONCE(tp->urg_data, 0); 2512 tcp_fast_path_check(sk); 2513 } 2514 2515 if (TCP_SKB_CB(skb)->has_rxtstamp) { 2516 tcp_update_recv_tstamps(skb, tss); 2517 *cmsg_flags |= TCP_CMSG_TS; 2518 } 2519 2520 if (used + offset < skb->len) 2521 continue; 2522 2523 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 2524 goto found_fin_ok; 2525 if (!(flags & MSG_PEEK)) 2526 tcp_eat_recv_skb(sk, skb); 2527 continue; 2528 2529 found_fin_ok: 2530 /* Process the FIN. */ 2531 WRITE_ONCE(*seq, *seq + 1); 2532 if (!(flags & MSG_PEEK)) 2533 tcp_eat_recv_skb(sk, skb); 2534 break; 2535 } while (len > 0); 2536 2537 /* According to UNIX98, msg_name/msg_namelen are ignored 2538 * on connected socket. I was just happy when found this 8) --ANK 2539 */ 2540 2541 /* Clean up data we have read: This will do ACK frames. */ 2542 tcp_cleanup_rbuf(sk, copied); 2543 return copied; 2544 2545 out: 2546 return err; 2547 2548 recv_urg: 2549 err = tcp_recv_urg(sk, msg, len, flags); 2550 goto out; 2551 2552 recv_sndq: 2553 err = tcp_peek_sndq(sk, msg, len); 2554 goto out; 2555 } 2556 2557 int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags, 2558 int *addr_len) 2559 { 2560 int cmsg_flags = 0, ret, inq; 2561 struct scm_timestamping_internal tss; 2562 2563 if (unlikely(flags & MSG_ERRQUEUE)) 2564 return inet_recv_error(sk, msg, len, addr_len); 2565 2566 if (sk_can_busy_loop(sk) && 2567 skb_queue_empty_lockless(&sk->sk_receive_queue) && 2568 sk->sk_state == TCP_ESTABLISHED) 2569 sk_busy_loop(sk, flags & MSG_DONTWAIT); 2570 2571 lock_sock(sk); 2572 ret = tcp_recvmsg_locked(sk, msg, len, flags, &tss, &cmsg_flags); 2573 release_sock(sk); 2574 sk_defer_free_flush(sk); 2575 2576 if (cmsg_flags && ret >= 0) { 2577 if (cmsg_flags & TCP_CMSG_TS) 2578 tcp_recv_timestamp(msg, sk, &tss); 2579 if (cmsg_flags & TCP_CMSG_INQ) { 2580 inq = tcp_inq_hint(sk); 2581 put_cmsg(msg, SOL_TCP, TCP_CM_INQ, sizeof(inq), &inq); 2582 } 2583 } 2584 return ret; 2585 } 2586 EXPORT_SYMBOL(tcp_recvmsg); 2587 2588 void tcp_set_state(struct sock *sk, int state) 2589 { 2590 int oldstate = sk->sk_state; 2591 2592 /* We defined a new enum for TCP states that are exported in BPF 2593 * so as not force the internal TCP states to be frozen. The 2594 * following checks will detect if an internal state value ever 2595 * differs from the BPF value. If this ever happens, then we will 2596 * need to remap the internal value to the BPF value before calling 2597 * tcp_call_bpf_2arg. 2598 */ 2599 BUILD_BUG_ON((int)BPF_TCP_ESTABLISHED != (int)TCP_ESTABLISHED); 2600 BUILD_BUG_ON((int)BPF_TCP_SYN_SENT != (int)TCP_SYN_SENT); 2601 BUILD_BUG_ON((int)BPF_TCP_SYN_RECV != (int)TCP_SYN_RECV); 2602 BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT1 != (int)TCP_FIN_WAIT1); 2603 BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT2 != (int)TCP_FIN_WAIT2); 2604 BUILD_BUG_ON((int)BPF_TCP_TIME_WAIT != (int)TCP_TIME_WAIT); 2605 BUILD_BUG_ON((int)BPF_TCP_CLOSE != (int)TCP_CLOSE); 2606 BUILD_BUG_ON((int)BPF_TCP_CLOSE_WAIT != (int)TCP_CLOSE_WAIT); 2607 BUILD_BUG_ON((int)BPF_TCP_LAST_ACK != (int)TCP_LAST_ACK); 2608 BUILD_BUG_ON((int)BPF_TCP_LISTEN != (int)TCP_LISTEN); 2609 BUILD_BUG_ON((int)BPF_TCP_CLOSING != (int)TCP_CLOSING); 2610 BUILD_BUG_ON((int)BPF_TCP_NEW_SYN_RECV != (int)TCP_NEW_SYN_RECV); 2611 BUILD_BUG_ON((int)BPF_TCP_MAX_STATES != (int)TCP_MAX_STATES); 2612 2613 /* bpf uapi header bpf.h defines an anonymous enum with values 2614 * BPF_TCP_* used by bpf programs. Currently gcc built vmlinux 2615 * is able to emit this enum in DWARF due to the above BUILD_BUG_ON. 2616 * But clang built vmlinux does not have this enum in DWARF 2617 * since clang removes the above code before generating IR/debuginfo. 2618 * Let us explicitly emit the type debuginfo to ensure the 2619 * above-mentioned anonymous enum in the vmlinux DWARF and hence BTF 2620 * regardless of which compiler is used. 2621 */ 2622 BTF_TYPE_EMIT_ENUM(BPF_TCP_ESTABLISHED); 2623 2624 if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_STATE_CB_FLAG)) 2625 tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state); 2626 2627 switch (state) { 2628 case TCP_ESTABLISHED: 2629 if (oldstate != TCP_ESTABLISHED) 2630 TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB); 2631 break; 2632 2633 case TCP_CLOSE: 2634 if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED) 2635 TCP_INC_STATS(sock_net(sk), TCP_MIB_ESTABRESETS); 2636 2637 sk->sk_prot->unhash(sk); 2638 if (inet_csk(sk)->icsk_bind_hash && 2639 !(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) 2640 inet_put_port(sk); 2641 fallthrough; 2642 default: 2643 if (oldstate == TCP_ESTABLISHED) 2644 TCP_DEC_STATS(sock_net(sk), TCP_MIB_CURRESTAB); 2645 } 2646 2647 /* Change state AFTER socket is unhashed to avoid closed 2648 * socket sitting in hash tables. 2649 */ 2650 inet_sk_state_store(sk, state); 2651 } 2652 EXPORT_SYMBOL_GPL(tcp_set_state); 2653 2654 /* 2655 * State processing on a close. This implements the state shift for 2656 * sending our FIN frame. Note that we only send a FIN for some 2657 * states. A shutdown() may have already sent the FIN, or we may be 2658 * closed. 2659 */ 2660 2661 static const unsigned char new_state[16] = { 2662 /* current state: new state: action: */ 2663 [0 /* (Invalid) */] = TCP_CLOSE, 2664 [TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 2665 [TCP_SYN_SENT] = TCP_CLOSE, 2666 [TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 2667 [TCP_FIN_WAIT1] = TCP_FIN_WAIT1, 2668 [TCP_FIN_WAIT2] = TCP_FIN_WAIT2, 2669 [TCP_TIME_WAIT] = TCP_CLOSE, 2670 [TCP_CLOSE] = TCP_CLOSE, 2671 [TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN, 2672 [TCP_LAST_ACK] = TCP_LAST_ACK, 2673 [TCP_LISTEN] = TCP_CLOSE, 2674 [TCP_CLOSING] = TCP_CLOSING, 2675 [TCP_NEW_SYN_RECV] = TCP_CLOSE, /* should not happen ! */ 2676 }; 2677 2678 static int tcp_close_state(struct sock *sk) 2679 { 2680 int next = (int)new_state[sk->sk_state]; 2681 int ns = next & TCP_STATE_MASK; 2682 2683 tcp_set_state(sk, ns); 2684 2685 return next & TCP_ACTION_FIN; 2686 } 2687 2688 /* 2689 * Shutdown the sending side of a connection. Much like close except 2690 * that we don't receive shut down or sock_set_flag(sk, SOCK_DEAD). 2691 */ 2692 2693 void tcp_shutdown(struct sock *sk, int how) 2694 { 2695 /* We need to grab some memory, and put together a FIN, 2696 * and then put it into the queue to be sent. 2697 * Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92. 2698 */ 2699 if (!(how & SEND_SHUTDOWN)) 2700 return; 2701 2702 /* If we've already sent a FIN, or it's a closed state, skip this. */ 2703 if ((1 << sk->sk_state) & 2704 (TCPF_ESTABLISHED | TCPF_SYN_SENT | 2705 TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) { 2706 /* Clear out any half completed packets. FIN if needed. */ 2707 if (tcp_close_state(sk)) 2708 tcp_send_fin(sk); 2709 } 2710 } 2711 EXPORT_SYMBOL(tcp_shutdown); 2712 2713 int tcp_orphan_count_sum(void) 2714 { 2715 int i, total = 0; 2716 2717 for_each_possible_cpu(i) 2718 total += per_cpu(tcp_orphan_count, i); 2719 2720 return max(total, 0); 2721 } 2722 2723 static int tcp_orphan_cache; 2724 static struct timer_list tcp_orphan_timer; 2725 #define TCP_ORPHAN_TIMER_PERIOD msecs_to_jiffies(100) 2726 2727 static void tcp_orphan_update(struct timer_list *unused) 2728 { 2729 WRITE_ONCE(tcp_orphan_cache, tcp_orphan_count_sum()); 2730 mod_timer(&tcp_orphan_timer, jiffies + TCP_ORPHAN_TIMER_PERIOD); 2731 } 2732 2733 static bool tcp_too_many_orphans(int shift) 2734 { 2735 return READ_ONCE(tcp_orphan_cache) << shift > sysctl_tcp_max_orphans; 2736 } 2737 2738 bool tcp_check_oom(struct sock *sk, int shift) 2739 { 2740 bool too_many_orphans, out_of_socket_memory; 2741 2742 too_many_orphans = tcp_too_many_orphans(shift); 2743 out_of_socket_memory = tcp_out_of_memory(sk); 2744 2745 if (too_many_orphans) 2746 net_info_ratelimited("too many orphaned sockets\n"); 2747 if (out_of_socket_memory) 2748 net_info_ratelimited("out of memory -- consider tuning tcp_mem\n"); 2749 return too_many_orphans || out_of_socket_memory; 2750 } 2751 2752 void __tcp_close(struct sock *sk, long timeout) 2753 { 2754 struct sk_buff *skb; 2755 int data_was_unread = 0; 2756 int state; 2757 2758 sk->sk_shutdown = SHUTDOWN_MASK; 2759 2760 if (sk->sk_state == TCP_LISTEN) { 2761 tcp_set_state(sk, TCP_CLOSE); 2762 2763 /* Special case. */ 2764 inet_csk_listen_stop(sk); 2765 2766 goto adjudge_to_death; 2767 } 2768 2769 /* We need to flush the recv. buffs. We do this only on the 2770 * descriptor close, not protocol-sourced closes, because the 2771 * reader process may not have drained the data yet! 2772 */ 2773 while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) { 2774 u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq; 2775 2776 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 2777 len--; 2778 data_was_unread += len; 2779 __kfree_skb(skb); 2780 } 2781 2782 sk_mem_reclaim(sk); 2783 2784 /* If socket has been already reset (e.g. in tcp_reset()) - kill it. */ 2785 if (sk->sk_state == TCP_CLOSE) 2786 goto adjudge_to_death; 2787 2788 /* As outlined in RFC 2525, section 2.17, we send a RST here because 2789 * data was lost. To witness the awful effects of the old behavior of 2790 * always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk 2791 * GET in an FTP client, suspend the process, wait for the client to 2792 * advertise a zero window, then kill -9 the FTP client, wheee... 2793 * Note: timeout is always zero in such a case. 2794 */ 2795 if (unlikely(tcp_sk(sk)->repair)) { 2796 sk->sk_prot->disconnect(sk, 0); 2797 } else if (data_was_unread) { 2798 /* Unread data was tossed, zap the connection. */ 2799 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE); 2800 tcp_set_state(sk, TCP_CLOSE); 2801 tcp_send_active_reset(sk, sk->sk_allocation); 2802 } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) { 2803 /* Check zero linger _after_ checking for unread data. */ 2804 sk->sk_prot->disconnect(sk, 0); 2805 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA); 2806 } else if (tcp_close_state(sk)) { 2807 /* We FIN if the application ate all the data before 2808 * zapping the connection. 2809 */ 2810 2811 /* RED-PEN. Formally speaking, we have broken TCP state 2812 * machine. State transitions: 2813 * 2814 * TCP_ESTABLISHED -> TCP_FIN_WAIT1 2815 * TCP_SYN_RECV -> TCP_FIN_WAIT1 (forget it, it's impossible) 2816 * TCP_CLOSE_WAIT -> TCP_LAST_ACK 2817 * 2818 * are legal only when FIN has been sent (i.e. in window), 2819 * rather than queued out of window. Purists blame. 2820 * 2821 * F.e. "RFC state" is ESTABLISHED, 2822 * if Linux state is FIN-WAIT-1, but FIN is still not sent. 2823 * 2824 * The visible declinations are that sometimes 2825 * we enter time-wait state, when it is not required really 2826 * (harmless), do not send active resets, when they are 2827 * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when 2828 * they look as CLOSING or LAST_ACK for Linux) 2829 * Probably, I missed some more holelets. 2830 * --ANK 2831 * XXX (TFO) - To start off we don't support SYN+ACK+FIN 2832 * in a single packet! (May consider it later but will 2833 * probably need API support or TCP_CORK SYN-ACK until 2834 * data is written and socket is closed.) 2835 */ 2836 tcp_send_fin(sk); 2837 } 2838 2839 sk_stream_wait_close(sk, timeout); 2840 2841 adjudge_to_death: 2842 state = sk->sk_state; 2843 sock_hold(sk); 2844 sock_orphan(sk); 2845 2846 local_bh_disable(); 2847 bh_lock_sock(sk); 2848 /* remove backlog if any, without releasing ownership. */ 2849 __release_sock(sk); 2850 2851 this_cpu_inc(tcp_orphan_count); 2852 2853 /* Have we already been destroyed by a softirq or backlog? */ 2854 if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE) 2855 goto out; 2856 2857 /* This is a (useful) BSD violating of the RFC. There is a 2858 * problem with TCP as specified in that the other end could 2859 * keep a socket open forever with no application left this end. 2860 * We use a 1 minute timeout (about the same as BSD) then kill 2861 * our end. If they send after that then tough - BUT: long enough 2862 * that we won't make the old 4*rto = almost no time - whoops 2863 * reset mistake. 2864 * 2865 * Nope, it was not mistake. It is really desired behaviour 2866 * f.e. on http servers, when such sockets are useless, but 2867 * consume significant resources. Let's do it with special 2868 * linger2 option. --ANK 2869 */ 2870 2871 if (sk->sk_state == TCP_FIN_WAIT2) { 2872 struct tcp_sock *tp = tcp_sk(sk); 2873 if (tp->linger2 < 0) { 2874 tcp_set_state(sk, TCP_CLOSE); 2875 tcp_send_active_reset(sk, GFP_ATOMIC); 2876 __NET_INC_STATS(sock_net(sk), 2877 LINUX_MIB_TCPABORTONLINGER); 2878 } else { 2879 const int tmo = tcp_fin_time(sk); 2880 2881 if (tmo > TCP_TIMEWAIT_LEN) { 2882 inet_csk_reset_keepalive_timer(sk, 2883 tmo - TCP_TIMEWAIT_LEN); 2884 } else { 2885 tcp_time_wait(sk, TCP_FIN_WAIT2, tmo); 2886 goto out; 2887 } 2888 } 2889 } 2890 if (sk->sk_state != TCP_CLOSE) { 2891 sk_mem_reclaim(sk); 2892 if (tcp_check_oom(sk, 0)) { 2893 tcp_set_state(sk, TCP_CLOSE); 2894 tcp_send_active_reset(sk, GFP_ATOMIC); 2895 __NET_INC_STATS(sock_net(sk), 2896 LINUX_MIB_TCPABORTONMEMORY); 2897 } else if (!check_net(sock_net(sk))) { 2898 /* Not possible to send reset; just close */ 2899 tcp_set_state(sk, TCP_CLOSE); 2900 } 2901 } 2902 2903 if (sk->sk_state == TCP_CLOSE) { 2904 struct request_sock *req; 2905 2906 req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk, 2907 lockdep_sock_is_held(sk)); 2908 /* We could get here with a non-NULL req if the socket is 2909 * aborted (e.g., closed with unread data) before 3WHS 2910 * finishes. 2911 */ 2912 if (req) 2913 reqsk_fastopen_remove(sk, req, false); 2914 inet_csk_destroy_sock(sk); 2915 } 2916 /* Otherwise, socket is reprieved until protocol close. */ 2917 2918 out: 2919 bh_unlock_sock(sk); 2920 local_bh_enable(); 2921 } 2922 2923 void tcp_close(struct sock *sk, long timeout) 2924 { 2925 lock_sock(sk); 2926 __tcp_close(sk, timeout); 2927 release_sock(sk); 2928 sock_put(sk); 2929 } 2930 EXPORT_SYMBOL(tcp_close); 2931 2932 /* These states need RST on ABORT according to RFC793 */ 2933 2934 static inline bool tcp_need_reset(int state) 2935 { 2936 return (1 << state) & 2937 (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 | 2938 TCPF_FIN_WAIT2 | TCPF_SYN_RECV); 2939 } 2940 2941 static void tcp_rtx_queue_purge(struct sock *sk) 2942 { 2943 struct rb_node *p = rb_first(&sk->tcp_rtx_queue); 2944 2945 tcp_sk(sk)->highest_sack = NULL; 2946 while (p) { 2947 struct sk_buff *skb = rb_to_skb(p); 2948 2949 p = rb_next(p); 2950 /* Since we are deleting whole queue, no need to 2951 * list_del(&skb->tcp_tsorted_anchor) 2952 */ 2953 tcp_rtx_queue_unlink(skb, sk); 2954 tcp_wmem_free_skb(sk, skb); 2955 } 2956 } 2957 2958 void tcp_write_queue_purge(struct sock *sk) 2959 { 2960 struct sk_buff *skb; 2961 2962 tcp_chrono_stop(sk, TCP_CHRONO_BUSY); 2963 while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) { 2964 tcp_skb_tsorted_anchor_cleanup(skb); 2965 tcp_wmem_free_skb(sk, skb); 2966 } 2967 tcp_rtx_queue_purge(sk); 2968 INIT_LIST_HEAD(&tcp_sk(sk)->tsorted_sent_queue); 2969 sk_mem_reclaim(sk); 2970 tcp_clear_all_retrans_hints(tcp_sk(sk)); 2971 tcp_sk(sk)->packets_out = 0; 2972 inet_csk(sk)->icsk_backoff = 0; 2973 } 2974 2975 int tcp_disconnect(struct sock *sk, int flags) 2976 { 2977 struct inet_sock *inet = inet_sk(sk); 2978 struct inet_connection_sock *icsk = inet_csk(sk); 2979 struct tcp_sock *tp = tcp_sk(sk); 2980 int old_state = sk->sk_state; 2981 u32 seq; 2982 2983 if (old_state != TCP_CLOSE) 2984 tcp_set_state(sk, TCP_CLOSE); 2985 2986 /* ABORT function of RFC793 */ 2987 if (old_state == TCP_LISTEN) { 2988 inet_csk_listen_stop(sk); 2989 } else if (unlikely(tp->repair)) { 2990 sk->sk_err = ECONNABORTED; 2991 } else if (tcp_need_reset(old_state) || 2992 (tp->snd_nxt != tp->write_seq && 2993 (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) { 2994 /* The last check adjusts for discrepancy of Linux wrt. RFC 2995 * states 2996 */ 2997 tcp_send_active_reset(sk, gfp_any()); 2998 sk->sk_err = ECONNRESET; 2999 } else if (old_state == TCP_SYN_SENT) 3000 sk->sk_err = ECONNRESET; 3001 3002 tcp_clear_xmit_timers(sk); 3003 __skb_queue_purge(&sk->sk_receive_queue); 3004 WRITE_ONCE(tp->copied_seq, tp->rcv_nxt); 3005 WRITE_ONCE(tp->urg_data, 0); 3006 tcp_write_queue_purge(sk); 3007 tcp_fastopen_active_disable_ofo_check(sk); 3008 skb_rbtree_purge(&tp->out_of_order_queue); 3009 3010 inet->inet_dport = 0; 3011 3012 if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK)) 3013 inet_reset_saddr(sk); 3014 3015 sk->sk_shutdown = 0; 3016 sock_reset_flag(sk, SOCK_DONE); 3017 tp->srtt_us = 0; 3018 tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT); 3019 tp->rcv_rtt_last_tsecr = 0; 3020 3021 seq = tp->write_seq + tp->max_window + 2; 3022 if (!seq) 3023 seq = 1; 3024 WRITE_ONCE(tp->write_seq, seq); 3025 3026 icsk->icsk_backoff = 0; 3027 icsk->icsk_probes_out = 0; 3028 icsk->icsk_probes_tstamp = 0; 3029 icsk->icsk_rto = TCP_TIMEOUT_INIT; 3030 icsk->icsk_rto_min = TCP_RTO_MIN; 3031 icsk->icsk_delack_max = TCP_DELACK_MAX; 3032 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; 3033 tcp_snd_cwnd_set(tp, TCP_INIT_CWND); 3034 tp->snd_cwnd_cnt = 0; 3035 tp->window_clamp = 0; 3036 tp->delivered = 0; 3037 tp->delivered_ce = 0; 3038 if (icsk->icsk_ca_ops->release) 3039 icsk->icsk_ca_ops->release(sk); 3040 memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv)); 3041 icsk->icsk_ca_initialized = 0; 3042 tcp_set_ca_state(sk, TCP_CA_Open); 3043 tp->is_sack_reneg = 0; 3044 tcp_clear_retrans(tp); 3045 tp->total_retrans = 0; 3046 inet_csk_delack_init(sk); 3047 /* Initialize rcv_mss to TCP_MIN_MSS to avoid division by 0 3048 * issue in __tcp_select_window() 3049 */ 3050 icsk->icsk_ack.rcv_mss = TCP_MIN_MSS; 3051 memset(&tp->rx_opt, 0, sizeof(tp->rx_opt)); 3052 __sk_dst_reset(sk); 3053 dst_release(xchg((__force struct dst_entry **)&sk->sk_rx_dst, NULL)); 3054 tcp_saved_syn_free(tp); 3055 tp->compressed_ack = 0; 3056 tp->segs_in = 0; 3057 tp->segs_out = 0; 3058 tp->bytes_sent = 0; 3059 tp->bytes_acked = 0; 3060 tp->bytes_received = 0; 3061 tp->bytes_retrans = 0; 3062 tp->data_segs_in = 0; 3063 tp->data_segs_out = 0; 3064 tp->duplicate_sack[0].start_seq = 0; 3065 tp->duplicate_sack[0].end_seq = 0; 3066 tp->dsack_dups = 0; 3067 tp->reord_seen = 0; 3068 tp->retrans_out = 0; 3069 tp->sacked_out = 0; 3070 tp->tlp_high_seq = 0; 3071 tp->last_oow_ack_time = 0; 3072 /* There's a bubble in the pipe until at least the first ACK. */ 3073 tp->app_limited = ~0U; 3074 tp->rack.mstamp = 0; 3075 tp->rack.advanced = 0; 3076 tp->rack.reo_wnd_steps = 1; 3077 tp->rack.last_delivered = 0; 3078 tp->rack.reo_wnd_persist = 0; 3079 tp->rack.dsack_seen = 0; 3080 tp->syn_data_acked = 0; 3081 tp->rx_opt.saw_tstamp = 0; 3082 tp->rx_opt.dsack = 0; 3083 tp->rx_opt.num_sacks = 0; 3084 tp->rcv_ooopack = 0; 3085 3086 3087 /* Clean up fastopen related fields */ 3088 tcp_free_fastopen_req(tp); 3089 inet->defer_connect = 0; 3090 tp->fastopen_client_fail = 0; 3091 3092 WARN_ON(inet->inet_num && !icsk->icsk_bind_hash); 3093 3094 if (sk->sk_frag.page) { 3095 put_page(sk->sk_frag.page); 3096 sk->sk_frag.page = NULL; 3097 sk->sk_frag.offset = 0; 3098 } 3099 sk_defer_free_flush(sk); 3100 sk_error_report(sk); 3101 return 0; 3102 } 3103 EXPORT_SYMBOL(tcp_disconnect); 3104 3105 static inline bool tcp_can_repair_sock(const struct sock *sk) 3106 { 3107 return ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) && 3108 (sk->sk_state != TCP_LISTEN); 3109 } 3110 3111 static int tcp_repair_set_window(struct tcp_sock *tp, sockptr_t optbuf, int len) 3112 { 3113 struct tcp_repair_window opt; 3114 3115 if (!tp->repair) 3116 return -EPERM; 3117 3118 if (len != sizeof(opt)) 3119 return -EINVAL; 3120 3121 if (copy_from_sockptr(&opt, optbuf, sizeof(opt))) 3122 return -EFAULT; 3123 3124 if (opt.max_window < opt.snd_wnd) 3125 return -EINVAL; 3126 3127 if (after(opt.snd_wl1, tp->rcv_nxt + opt.rcv_wnd)) 3128 return -EINVAL; 3129 3130 if (after(opt.rcv_wup, tp->rcv_nxt)) 3131 return -EINVAL; 3132 3133 tp->snd_wl1 = opt.snd_wl1; 3134 tp->snd_wnd = opt.snd_wnd; 3135 tp->max_window = opt.max_window; 3136 3137 tp->rcv_wnd = opt.rcv_wnd; 3138 tp->rcv_wup = opt.rcv_wup; 3139 3140 return 0; 3141 } 3142 3143 static int tcp_repair_options_est(struct sock *sk, sockptr_t optbuf, 3144 unsigned int len) 3145 { 3146 struct tcp_sock *tp = tcp_sk(sk); 3147 struct tcp_repair_opt opt; 3148 size_t offset = 0; 3149 3150 while (len >= sizeof(opt)) { 3151 if (copy_from_sockptr_offset(&opt, optbuf, offset, sizeof(opt))) 3152 return -EFAULT; 3153 3154 offset += sizeof(opt); 3155 len -= sizeof(opt); 3156 3157 switch (opt.opt_code) { 3158 case TCPOPT_MSS: 3159 tp->rx_opt.mss_clamp = opt.opt_val; 3160 tcp_mtup_init(sk); 3161 break; 3162 case TCPOPT_WINDOW: 3163 { 3164 u16 snd_wscale = opt.opt_val & 0xFFFF; 3165 u16 rcv_wscale = opt.opt_val >> 16; 3166 3167 if (snd_wscale > TCP_MAX_WSCALE || rcv_wscale > TCP_MAX_WSCALE) 3168 return -EFBIG; 3169 3170 tp->rx_opt.snd_wscale = snd_wscale; 3171 tp->rx_opt.rcv_wscale = rcv_wscale; 3172 tp->rx_opt.wscale_ok = 1; 3173 } 3174 break; 3175 case TCPOPT_SACK_PERM: 3176 if (opt.opt_val != 0) 3177 return -EINVAL; 3178 3179 tp->rx_opt.sack_ok |= TCP_SACK_SEEN; 3180 break; 3181 case TCPOPT_TIMESTAMP: 3182 if (opt.opt_val != 0) 3183 return -EINVAL; 3184 3185 tp->rx_opt.tstamp_ok = 1; 3186 break; 3187 } 3188 } 3189 3190 return 0; 3191 } 3192 3193 DEFINE_STATIC_KEY_FALSE(tcp_tx_delay_enabled); 3194 EXPORT_SYMBOL(tcp_tx_delay_enabled); 3195 3196 static void tcp_enable_tx_delay(void) 3197 { 3198 if (!static_branch_unlikely(&tcp_tx_delay_enabled)) { 3199 static int __tcp_tx_delay_enabled = 0; 3200 3201 if (cmpxchg(&__tcp_tx_delay_enabled, 0, 1) == 0) { 3202 static_branch_enable(&tcp_tx_delay_enabled); 3203 pr_info("TCP_TX_DELAY enabled\n"); 3204 } 3205 } 3206 } 3207 3208 /* When set indicates to always queue non-full frames. Later the user clears 3209 * this option and we transmit any pending partial frames in the queue. This is 3210 * meant to be used alongside sendfile() to get properly filled frames when the 3211 * user (for example) must write out headers with a write() call first and then 3212 * use sendfile to send out the data parts. 3213 * 3214 * TCP_CORK can be set together with TCP_NODELAY and it is stronger than 3215 * TCP_NODELAY. 3216 */ 3217 void __tcp_sock_set_cork(struct sock *sk, bool on) 3218 { 3219 struct tcp_sock *tp = tcp_sk(sk); 3220 3221 if (on) { 3222 tp->nonagle |= TCP_NAGLE_CORK; 3223 } else { 3224 tp->nonagle &= ~TCP_NAGLE_CORK; 3225 if (tp->nonagle & TCP_NAGLE_OFF) 3226 tp->nonagle |= TCP_NAGLE_PUSH; 3227 tcp_push_pending_frames(sk); 3228 } 3229 } 3230 3231 void tcp_sock_set_cork(struct sock *sk, bool on) 3232 { 3233 lock_sock(sk); 3234 __tcp_sock_set_cork(sk, on); 3235 release_sock(sk); 3236 } 3237 EXPORT_SYMBOL(tcp_sock_set_cork); 3238 3239 /* TCP_NODELAY is weaker than TCP_CORK, so that this option on corked socket is 3240 * remembered, but it is not activated until cork is cleared. 3241 * 3242 * However, when TCP_NODELAY is set we make an explicit push, which overrides 3243 * even TCP_CORK for currently queued segments. 3244 */ 3245 void __tcp_sock_set_nodelay(struct sock *sk, bool on) 3246 { 3247 if (on) { 3248 tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH; 3249 tcp_push_pending_frames(sk); 3250 } else { 3251 tcp_sk(sk)->nonagle &= ~TCP_NAGLE_OFF; 3252 } 3253 } 3254 3255 void tcp_sock_set_nodelay(struct sock *sk) 3256 { 3257 lock_sock(sk); 3258 __tcp_sock_set_nodelay(sk, true); 3259 release_sock(sk); 3260 } 3261 EXPORT_SYMBOL(tcp_sock_set_nodelay); 3262 3263 static void __tcp_sock_set_quickack(struct sock *sk, int val) 3264 { 3265 if (!val) { 3266 inet_csk_enter_pingpong_mode(sk); 3267 return; 3268 } 3269 3270 inet_csk_exit_pingpong_mode(sk); 3271 if ((1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) && 3272 inet_csk_ack_scheduled(sk)) { 3273 inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_PUSHED; 3274 tcp_cleanup_rbuf(sk, 1); 3275 if (!(val & 1)) 3276 inet_csk_enter_pingpong_mode(sk); 3277 } 3278 } 3279 3280 void tcp_sock_set_quickack(struct sock *sk, int val) 3281 { 3282 lock_sock(sk); 3283 __tcp_sock_set_quickack(sk, val); 3284 release_sock(sk); 3285 } 3286 EXPORT_SYMBOL(tcp_sock_set_quickack); 3287 3288 int tcp_sock_set_syncnt(struct sock *sk, int val) 3289 { 3290 if (val < 1 || val > MAX_TCP_SYNCNT) 3291 return -EINVAL; 3292 3293 lock_sock(sk); 3294 inet_csk(sk)->icsk_syn_retries = val; 3295 release_sock(sk); 3296 return 0; 3297 } 3298 EXPORT_SYMBOL(tcp_sock_set_syncnt); 3299 3300 void tcp_sock_set_user_timeout(struct sock *sk, u32 val) 3301 { 3302 lock_sock(sk); 3303 inet_csk(sk)->icsk_user_timeout = val; 3304 release_sock(sk); 3305 } 3306 EXPORT_SYMBOL(tcp_sock_set_user_timeout); 3307 3308 int tcp_sock_set_keepidle_locked(struct sock *sk, int val) 3309 { 3310 struct tcp_sock *tp = tcp_sk(sk); 3311 3312 if (val < 1 || val > MAX_TCP_KEEPIDLE) 3313 return -EINVAL; 3314 3315 tp->keepalive_time = val * HZ; 3316 if (sock_flag(sk, SOCK_KEEPOPEN) && 3317 !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) { 3318 u32 elapsed = keepalive_time_elapsed(tp); 3319 3320 if (tp->keepalive_time > elapsed) 3321 elapsed = tp->keepalive_time - elapsed; 3322 else 3323 elapsed = 0; 3324 inet_csk_reset_keepalive_timer(sk, elapsed); 3325 } 3326 3327 return 0; 3328 } 3329 3330 int tcp_sock_set_keepidle(struct sock *sk, int val) 3331 { 3332 int err; 3333 3334 lock_sock(sk); 3335 err = tcp_sock_set_keepidle_locked(sk, val); 3336 release_sock(sk); 3337 return err; 3338 } 3339 EXPORT_SYMBOL(tcp_sock_set_keepidle); 3340 3341 int tcp_sock_set_keepintvl(struct sock *sk, int val) 3342 { 3343 if (val < 1 || val > MAX_TCP_KEEPINTVL) 3344 return -EINVAL; 3345 3346 lock_sock(sk); 3347 tcp_sk(sk)->keepalive_intvl = val * HZ; 3348 release_sock(sk); 3349 return 0; 3350 } 3351 EXPORT_SYMBOL(tcp_sock_set_keepintvl); 3352 3353 int tcp_sock_set_keepcnt(struct sock *sk, int val) 3354 { 3355 if (val < 1 || val > MAX_TCP_KEEPCNT) 3356 return -EINVAL; 3357 3358 lock_sock(sk); 3359 tcp_sk(sk)->keepalive_probes = val; 3360 release_sock(sk); 3361 return 0; 3362 } 3363 EXPORT_SYMBOL(tcp_sock_set_keepcnt); 3364 3365 int tcp_set_window_clamp(struct sock *sk, int val) 3366 { 3367 struct tcp_sock *tp = tcp_sk(sk); 3368 3369 if (!val) { 3370 if (sk->sk_state != TCP_CLOSE) 3371 return -EINVAL; 3372 tp->window_clamp = 0; 3373 } else { 3374 tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ? 3375 SOCK_MIN_RCVBUF / 2 : val; 3376 tp->rcv_ssthresh = min(tp->rcv_wnd, tp->window_clamp); 3377 } 3378 return 0; 3379 } 3380 3381 /* 3382 * Socket option code for TCP. 3383 */ 3384 static int do_tcp_setsockopt(struct sock *sk, int level, int optname, 3385 sockptr_t optval, unsigned int optlen) 3386 { 3387 struct tcp_sock *tp = tcp_sk(sk); 3388 struct inet_connection_sock *icsk = inet_csk(sk); 3389 struct net *net = sock_net(sk); 3390 int val; 3391 int err = 0; 3392 3393 /* These are data/string values, all the others are ints */ 3394 switch (optname) { 3395 case TCP_CONGESTION: { 3396 char name[TCP_CA_NAME_MAX]; 3397 3398 if (optlen < 1) 3399 return -EINVAL; 3400 3401 val = strncpy_from_sockptr(name, optval, 3402 min_t(long, TCP_CA_NAME_MAX-1, optlen)); 3403 if (val < 0) 3404 return -EFAULT; 3405 name[val] = 0; 3406 3407 lock_sock(sk); 3408 err = tcp_set_congestion_control(sk, name, true, 3409 ns_capable(sock_net(sk)->user_ns, 3410 CAP_NET_ADMIN)); 3411 release_sock(sk); 3412 return err; 3413 } 3414 case TCP_ULP: { 3415 char name[TCP_ULP_NAME_MAX]; 3416 3417 if (optlen < 1) 3418 return -EINVAL; 3419 3420 val = strncpy_from_sockptr(name, optval, 3421 min_t(long, TCP_ULP_NAME_MAX - 1, 3422 optlen)); 3423 if (val < 0) 3424 return -EFAULT; 3425 name[val] = 0; 3426 3427 lock_sock(sk); 3428 err = tcp_set_ulp(sk, name); 3429 release_sock(sk); 3430 return err; 3431 } 3432 case TCP_FASTOPEN_KEY: { 3433 __u8 key[TCP_FASTOPEN_KEY_BUF_LENGTH]; 3434 __u8 *backup_key = NULL; 3435 3436 /* Allow a backup key as well to facilitate key rotation 3437 * First key is the active one. 3438 */ 3439 if (optlen != TCP_FASTOPEN_KEY_LENGTH && 3440 optlen != TCP_FASTOPEN_KEY_BUF_LENGTH) 3441 return -EINVAL; 3442 3443 if (copy_from_sockptr(key, optval, optlen)) 3444 return -EFAULT; 3445 3446 if (optlen == TCP_FASTOPEN_KEY_BUF_LENGTH) 3447 backup_key = key + TCP_FASTOPEN_KEY_LENGTH; 3448 3449 return tcp_fastopen_reset_cipher(net, sk, key, backup_key); 3450 } 3451 default: 3452 /* fallthru */ 3453 break; 3454 } 3455 3456 if (optlen < sizeof(int)) 3457 return -EINVAL; 3458 3459 if (copy_from_sockptr(&val, optval, sizeof(val))) 3460 return -EFAULT; 3461 3462 lock_sock(sk); 3463 3464 switch (optname) { 3465 case TCP_MAXSEG: 3466 /* Values greater than interface MTU won't take effect. However 3467 * at the point when this call is done we typically don't yet 3468 * know which interface is going to be used 3469 */ 3470 if (val && (val < TCP_MIN_MSS || val > MAX_TCP_WINDOW)) { 3471 err = -EINVAL; 3472 break; 3473 } 3474 tp->rx_opt.user_mss = val; 3475 break; 3476 3477 case TCP_NODELAY: 3478 __tcp_sock_set_nodelay(sk, val); 3479 break; 3480 3481 case TCP_THIN_LINEAR_TIMEOUTS: 3482 if (val < 0 || val > 1) 3483 err = -EINVAL; 3484 else 3485 tp->thin_lto = val; 3486 break; 3487 3488 case TCP_THIN_DUPACK: 3489 if (val < 0 || val > 1) 3490 err = -EINVAL; 3491 break; 3492 3493 case TCP_REPAIR: 3494 if (!tcp_can_repair_sock(sk)) 3495 err = -EPERM; 3496 else if (val == TCP_REPAIR_ON) { 3497 tp->repair = 1; 3498 sk->sk_reuse = SK_FORCE_REUSE; 3499 tp->repair_queue = TCP_NO_QUEUE; 3500 } else if (val == TCP_REPAIR_OFF) { 3501 tp->repair = 0; 3502 sk->sk_reuse = SK_NO_REUSE; 3503 tcp_send_window_probe(sk); 3504 } else if (val == TCP_REPAIR_OFF_NO_WP) { 3505 tp->repair = 0; 3506 sk->sk_reuse = SK_NO_REUSE; 3507 } else 3508 err = -EINVAL; 3509 3510 break; 3511 3512 case TCP_REPAIR_QUEUE: 3513 if (!tp->repair) 3514 err = -EPERM; 3515 else if ((unsigned int)val < TCP_QUEUES_NR) 3516 tp->repair_queue = val; 3517 else 3518 err = -EINVAL; 3519 break; 3520 3521 case TCP_QUEUE_SEQ: 3522 if (sk->sk_state != TCP_CLOSE) { 3523 err = -EPERM; 3524 } else if (tp->repair_queue == TCP_SEND_QUEUE) { 3525 if (!tcp_rtx_queue_empty(sk)) 3526 err = -EPERM; 3527 else 3528 WRITE_ONCE(tp->write_seq, val); 3529 } else if (tp->repair_queue == TCP_RECV_QUEUE) { 3530 if (tp->rcv_nxt != tp->copied_seq) { 3531 err = -EPERM; 3532 } else { 3533 WRITE_ONCE(tp->rcv_nxt, val); 3534 WRITE_ONCE(tp->copied_seq, val); 3535 } 3536 } else { 3537 err = -EINVAL; 3538 } 3539 break; 3540 3541 case TCP_REPAIR_OPTIONS: 3542 if (!tp->repair) 3543 err = -EINVAL; 3544 else if (sk->sk_state == TCP_ESTABLISHED) 3545 err = tcp_repair_options_est(sk, optval, optlen); 3546 else 3547 err = -EPERM; 3548 break; 3549 3550 case TCP_CORK: 3551 __tcp_sock_set_cork(sk, val); 3552 break; 3553 3554 case TCP_KEEPIDLE: 3555 err = tcp_sock_set_keepidle_locked(sk, val); 3556 break; 3557 case TCP_KEEPINTVL: 3558 if (val < 1 || val > MAX_TCP_KEEPINTVL) 3559 err = -EINVAL; 3560 else 3561 tp->keepalive_intvl = val * HZ; 3562 break; 3563 case TCP_KEEPCNT: 3564 if (val < 1 || val > MAX_TCP_KEEPCNT) 3565 err = -EINVAL; 3566 else 3567 tp->keepalive_probes = val; 3568 break; 3569 case TCP_SYNCNT: 3570 if (val < 1 || val > MAX_TCP_SYNCNT) 3571 err = -EINVAL; 3572 else 3573 icsk->icsk_syn_retries = val; 3574 break; 3575 3576 case TCP_SAVE_SYN: 3577 /* 0: disable, 1: enable, 2: start from ether_header */ 3578 if (val < 0 || val > 2) 3579 err = -EINVAL; 3580 else 3581 tp->save_syn = val; 3582 break; 3583 3584 case TCP_LINGER2: 3585 if (val < 0) 3586 tp->linger2 = -1; 3587 else if (val > TCP_FIN_TIMEOUT_MAX / HZ) 3588 tp->linger2 = TCP_FIN_TIMEOUT_MAX; 3589 else 3590 tp->linger2 = val * HZ; 3591 break; 3592 3593 case TCP_DEFER_ACCEPT: 3594 /* Translate value in seconds to number of retransmits */ 3595 icsk->icsk_accept_queue.rskq_defer_accept = 3596 secs_to_retrans(val, TCP_TIMEOUT_INIT / HZ, 3597 TCP_RTO_MAX / HZ); 3598 break; 3599 3600 case TCP_WINDOW_CLAMP: 3601 err = tcp_set_window_clamp(sk, val); 3602 break; 3603 3604 case TCP_QUICKACK: 3605 __tcp_sock_set_quickack(sk, val); 3606 break; 3607 3608 #ifdef CONFIG_TCP_MD5SIG 3609 case TCP_MD5SIG: 3610 case TCP_MD5SIG_EXT: 3611 err = tp->af_specific->md5_parse(sk, optname, optval, optlen); 3612 break; 3613 #endif 3614 case TCP_USER_TIMEOUT: 3615 /* Cap the max time in ms TCP will retry or probe the window 3616 * before giving up and aborting (ETIMEDOUT) a connection. 3617 */ 3618 if (val < 0) 3619 err = -EINVAL; 3620 else 3621 icsk->icsk_user_timeout = val; 3622 break; 3623 3624 case TCP_FASTOPEN: 3625 if (val >= 0 && ((1 << sk->sk_state) & (TCPF_CLOSE | 3626 TCPF_LISTEN))) { 3627 tcp_fastopen_init_key_once(net); 3628 3629 fastopen_queue_tune(sk, val); 3630 } else { 3631 err = -EINVAL; 3632 } 3633 break; 3634 case TCP_FASTOPEN_CONNECT: 3635 if (val > 1 || val < 0) { 3636 err = -EINVAL; 3637 } else if (net->ipv4.sysctl_tcp_fastopen & TFO_CLIENT_ENABLE) { 3638 if (sk->sk_state == TCP_CLOSE) 3639 tp->fastopen_connect = val; 3640 else 3641 err = -EINVAL; 3642 } else { 3643 err = -EOPNOTSUPP; 3644 } 3645 break; 3646 case TCP_FASTOPEN_NO_COOKIE: 3647 if (val > 1 || val < 0) 3648 err = -EINVAL; 3649 else if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) 3650 err = -EINVAL; 3651 else 3652 tp->fastopen_no_cookie = val; 3653 break; 3654 case TCP_TIMESTAMP: 3655 if (!tp->repair) 3656 err = -EPERM; 3657 else 3658 tp->tsoffset = val - tcp_time_stamp_raw(); 3659 break; 3660 case TCP_REPAIR_WINDOW: 3661 err = tcp_repair_set_window(tp, optval, optlen); 3662 break; 3663 case TCP_NOTSENT_LOWAT: 3664 tp->notsent_lowat = val; 3665 sk->sk_write_space(sk); 3666 break; 3667 case TCP_INQ: 3668 if (val > 1 || val < 0) 3669 err = -EINVAL; 3670 else 3671 tp->recvmsg_inq = val; 3672 break; 3673 case TCP_TX_DELAY: 3674 if (val) 3675 tcp_enable_tx_delay(); 3676 tp->tcp_tx_delay = val; 3677 break; 3678 default: 3679 err = -ENOPROTOOPT; 3680 break; 3681 } 3682 3683 release_sock(sk); 3684 return err; 3685 } 3686 3687 int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, 3688 unsigned int optlen) 3689 { 3690 const struct inet_connection_sock *icsk = inet_csk(sk); 3691 3692 if (level != SOL_TCP) 3693 return icsk->icsk_af_ops->setsockopt(sk, level, optname, 3694 optval, optlen); 3695 return do_tcp_setsockopt(sk, level, optname, optval, optlen); 3696 } 3697 EXPORT_SYMBOL(tcp_setsockopt); 3698 3699 static void tcp_get_info_chrono_stats(const struct tcp_sock *tp, 3700 struct tcp_info *info) 3701 { 3702 u64 stats[__TCP_CHRONO_MAX], total = 0; 3703 enum tcp_chrono i; 3704 3705 for (i = TCP_CHRONO_BUSY; i < __TCP_CHRONO_MAX; ++i) { 3706 stats[i] = tp->chrono_stat[i - 1]; 3707 if (i == tp->chrono_type) 3708 stats[i] += tcp_jiffies32 - tp->chrono_start; 3709 stats[i] *= USEC_PER_SEC / HZ; 3710 total += stats[i]; 3711 } 3712 3713 info->tcpi_busy_time = total; 3714 info->tcpi_rwnd_limited = stats[TCP_CHRONO_RWND_LIMITED]; 3715 info->tcpi_sndbuf_limited = stats[TCP_CHRONO_SNDBUF_LIMITED]; 3716 } 3717 3718 /* Return information about state of tcp endpoint in API format. */ 3719 void tcp_get_info(struct sock *sk, struct tcp_info *info) 3720 { 3721 const struct tcp_sock *tp = tcp_sk(sk); /* iff sk_type == SOCK_STREAM */ 3722 const struct inet_connection_sock *icsk = inet_csk(sk); 3723 unsigned long rate; 3724 u32 now; 3725 u64 rate64; 3726 bool slow; 3727 3728 memset(info, 0, sizeof(*info)); 3729 if (sk->sk_type != SOCK_STREAM) 3730 return; 3731 3732 info->tcpi_state = inet_sk_state_load(sk); 3733 3734 /* Report meaningful fields for all TCP states, including listeners */ 3735 rate = READ_ONCE(sk->sk_pacing_rate); 3736 rate64 = (rate != ~0UL) ? rate : ~0ULL; 3737 info->tcpi_pacing_rate = rate64; 3738 3739 rate = READ_ONCE(sk->sk_max_pacing_rate); 3740 rate64 = (rate != ~0UL) ? rate : ~0ULL; 3741 info->tcpi_max_pacing_rate = rate64; 3742 3743 info->tcpi_reordering = tp->reordering; 3744 info->tcpi_snd_cwnd = tcp_snd_cwnd(tp); 3745 3746 if (info->tcpi_state == TCP_LISTEN) { 3747 /* listeners aliased fields : 3748 * tcpi_unacked -> Number of children ready for accept() 3749 * tcpi_sacked -> max backlog 3750 */ 3751 info->tcpi_unacked = READ_ONCE(sk->sk_ack_backlog); 3752 info->tcpi_sacked = READ_ONCE(sk->sk_max_ack_backlog); 3753 return; 3754 } 3755 3756 slow = lock_sock_fast(sk); 3757 3758 info->tcpi_ca_state = icsk->icsk_ca_state; 3759 info->tcpi_retransmits = icsk->icsk_retransmits; 3760 info->tcpi_probes = icsk->icsk_probes_out; 3761 info->tcpi_backoff = icsk->icsk_backoff; 3762 3763 if (tp->rx_opt.tstamp_ok) 3764 info->tcpi_options |= TCPI_OPT_TIMESTAMPS; 3765 if (tcp_is_sack(tp)) 3766 info->tcpi_options |= TCPI_OPT_SACK; 3767 if (tp->rx_opt.wscale_ok) { 3768 info->tcpi_options |= TCPI_OPT_WSCALE; 3769 info->tcpi_snd_wscale = tp->rx_opt.snd_wscale; 3770 info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale; 3771 } 3772 3773 if (tp->ecn_flags & TCP_ECN_OK) 3774 info->tcpi_options |= TCPI_OPT_ECN; 3775 if (tp->ecn_flags & TCP_ECN_SEEN) 3776 info->tcpi_options |= TCPI_OPT_ECN_SEEN; 3777 if (tp->syn_data_acked) 3778 info->tcpi_options |= TCPI_OPT_SYN_DATA; 3779 3780 info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto); 3781 info->tcpi_ato = jiffies_to_usecs(icsk->icsk_ack.ato); 3782 info->tcpi_snd_mss = tp->mss_cache; 3783 info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss; 3784 3785 info->tcpi_unacked = tp->packets_out; 3786 info->tcpi_sacked = tp->sacked_out; 3787 3788 info->tcpi_lost = tp->lost_out; 3789 info->tcpi_retrans = tp->retrans_out; 3790 3791 now = tcp_jiffies32; 3792 info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime); 3793 info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime); 3794 info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp); 3795 3796 info->tcpi_pmtu = icsk->icsk_pmtu_cookie; 3797 info->tcpi_rcv_ssthresh = tp->rcv_ssthresh; 3798 info->tcpi_rtt = tp->srtt_us >> 3; 3799 info->tcpi_rttvar = tp->mdev_us >> 2; 3800 info->tcpi_snd_ssthresh = tp->snd_ssthresh; 3801 info->tcpi_advmss = tp->advmss; 3802 3803 info->tcpi_rcv_rtt = tp->rcv_rtt_est.rtt_us >> 3; 3804 info->tcpi_rcv_space = tp->rcvq_space.space; 3805 3806 info->tcpi_total_retrans = tp->total_retrans; 3807 3808 info->tcpi_bytes_acked = tp->bytes_acked; 3809 info->tcpi_bytes_received = tp->bytes_received; 3810 info->tcpi_notsent_bytes = max_t(int, 0, tp->write_seq - tp->snd_nxt); 3811 tcp_get_info_chrono_stats(tp, info); 3812 3813 info->tcpi_segs_out = tp->segs_out; 3814 3815 /* segs_in and data_segs_in can be updated from tcp_segs_in() from BH */ 3816 info->tcpi_segs_in = READ_ONCE(tp->segs_in); 3817 info->tcpi_data_segs_in = READ_ONCE(tp->data_segs_in); 3818 3819 info->tcpi_min_rtt = tcp_min_rtt(tp); 3820 info->tcpi_data_segs_out = tp->data_segs_out; 3821 3822 info->tcpi_delivery_rate_app_limited = tp->rate_app_limited ? 1 : 0; 3823 rate64 = tcp_compute_delivery_rate(tp); 3824 if (rate64) 3825 info->tcpi_delivery_rate = rate64; 3826 info->tcpi_delivered = tp->delivered; 3827 info->tcpi_delivered_ce = tp->delivered_ce; 3828 info->tcpi_bytes_sent = tp->bytes_sent; 3829 info->tcpi_bytes_retrans = tp->bytes_retrans; 3830 info->tcpi_dsack_dups = tp->dsack_dups; 3831 info->tcpi_reord_seen = tp->reord_seen; 3832 info->tcpi_rcv_ooopack = tp->rcv_ooopack; 3833 info->tcpi_snd_wnd = tp->snd_wnd; 3834 info->tcpi_fastopen_client_fail = tp->fastopen_client_fail; 3835 unlock_sock_fast(sk, slow); 3836 } 3837 EXPORT_SYMBOL_GPL(tcp_get_info); 3838 3839 static size_t tcp_opt_stats_get_size(void) 3840 { 3841 return 3842 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BUSY */ 3843 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_RWND_LIMITED */ 3844 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_SNDBUF_LIMITED */ 3845 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_DATA_SEGS_OUT */ 3846 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_TOTAL_RETRANS */ 3847 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_PACING_RATE */ 3848 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_DELIVERY_RATE */ 3849 nla_total_size(sizeof(u32)) + /* TCP_NLA_SND_CWND */ 3850 nla_total_size(sizeof(u32)) + /* TCP_NLA_REORDERING */ 3851 nla_total_size(sizeof(u32)) + /* TCP_NLA_MIN_RTT */ 3852 nla_total_size(sizeof(u8)) + /* TCP_NLA_RECUR_RETRANS */ 3853 nla_total_size(sizeof(u8)) + /* TCP_NLA_DELIVERY_RATE_APP_LMT */ 3854 nla_total_size(sizeof(u32)) + /* TCP_NLA_SNDQ_SIZE */ 3855 nla_total_size(sizeof(u8)) + /* TCP_NLA_CA_STATE */ 3856 nla_total_size(sizeof(u32)) + /* TCP_NLA_SND_SSTHRESH */ 3857 nla_total_size(sizeof(u32)) + /* TCP_NLA_DELIVERED */ 3858 nla_total_size(sizeof(u32)) + /* TCP_NLA_DELIVERED_CE */ 3859 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BYTES_SENT */ 3860 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BYTES_RETRANS */ 3861 nla_total_size(sizeof(u32)) + /* TCP_NLA_DSACK_DUPS */ 3862 nla_total_size(sizeof(u32)) + /* TCP_NLA_REORD_SEEN */ 3863 nla_total_size(sizeof(u32)) + /* TCP_NLA_SRTT */ 3864 nla_total_size(sizeof(u16)) + /* TCP_NLA_TIMEOUT_REHASH */ 3865 nla_total_size(sizeof(u32)) + /* TCP_NLA_BYTES_NOTSENT */ 3866 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_EDT */ 3867 nla_total_size(sizeof(u8)) + /* TCP_NLA_TTL */ 3868 0; 3869 } 3870 3871 /* Returns TTL or hop limit of an incoming packet from skb. */ 3872 static u8 tcp_skb_ttl_or_hop_limit(const struct sk_buff *skb) 3873 { 3874 if (skb->protocol == htons(ETH_P_IP)) 3875 return ip_hdr(skb)->ttl; 3876 else if (skb->protocol == htons(ETH_P_IPV6)) 3877 return ipv6_hdr(skb)->hop_limit; 3878 else 3879 return 0; 3880 } 3881 3882 struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk, 3883 const struct sk_buff *orig_skb, 3884 const struct sk_buff *ack_skb) 3885 { 3886 const struct tcp_sock *tp = tcp_sk(sk); 3887 struct sk_buff *stats; 3888 struct tcp_info info; 3889 unsigned long rate; 3890 u64 rate64; 3891 3892 stats = alloc_skb(tcp_opt_stats_get_size(), GFP_ATOMIC); 3893 if (!stats) 3894 return NULL; 3895 3896 tcp_get_info_chrono_stats(tp, &info); 3897 nla_put_u64_64bit(stats, TCP_NLA_BUSY, 3898 info.tcpi_busy_time, TCP_NLA_PAD); 3899 nla_put_u64_64bit(stats, TCP_NLA_RWND_LIMITED, 3900 info.tcpi_rwnd_limited, TCP_NLA_PAD); 3901 nla_put_u64_64bit(stats, TCP_NLA_SNDBUF_LIMITED, 3902 info.tcpi_sndbuf_limited, TCP_NLA_PAD); 3903 nla_put_u64_64bit(stats, TCP_NLA_DATA_SEGS_OUT, 3904 tp->data_segs_out, TCP_NLA_PAD); 3905 nla_put_u64_64bit(stats, TCP_NLA_TOTAL_RETRANS, 3906 tp->total_retrans, TCP_NLA_PAD); 3907 3908 rate = READ_ONCE(sk->sk_pacing_rate); 3909 rate64 = (rate != ~0UL) ? rate : ~0ULL; 3910 nla_put_u64_64bit(stats, TCP_NLA_PACING_RATE, rate64, TCP_NLA_PAD); 3911 3912 rate64 = tcp_compute_delivery_rate(tp); 3913 nla_put_u64_64bit(stats, TCP_NLA_DELIVERY_RATE, rate64, TCP_NLA_PAD); 3914 3915 nla_put_u32(stats, TCP_NLA_SND_CWND, tcp_snd_cwnd(tp)); 3916 nla_put_u32(stats, TCP_NLA_REORDERING, tp->reordering); 3917 nla_put_u32(stats, TCP_NLA_MIN_RTT, tcp_min_rtt(tp)); 3918 3919 nla_put_u8(stats, TCP_NLA_RECUR_RETRANS, inet_csk(sk)->icsk_retransmits); 3920 nla_put_u8(stats, TCP_NLA_DELIVERY_RATE_APP_LMT, !!tp->rate_app_limited); 3921 nla_put_u32(stats, TCP_NLA_SND_SSTHRESH, tp->snd_ssthresh); 3922 nla_put_u32(stats, TCP_NLA_DELIVERED, tp->delivered); 3923 nla_put_u32(stats, TCP_NLA_DELIVERED_CE, tp->delivered_ce); 3924 3925 nla_put_u32(stats, TCP_NLA_SNDQ_SIZE, tp->write_seq - tp->snd_una); 3926 nla_put_u8(stats, TCP_NLA_CA_STATE, inet_csk(sk)->icsk_ca_state); 3927 3928 nla_put_u64_64bit(stats, TCP_NLA_BYTES_SENT, tp->bytes_sent, 3929 TCP_NLA_PAD); 3930 nla_put_u64_64bit(stats, TCP_NLA_BYTES_RETRANS, tp->bytes_retrans, 3931 TCP_NLA_PAD); 3932 nla_put_u32(stats, TCP_NLA_DSACK_DUPS, tp->dsack_dups); 3933 nla_put_u32(stats, TCP_NLA_REORD_SEEN, tp->reord_seen); 3934 nla_put_u32(stats, TCP_NLA_SRTT, tp->srtt_us >> 3); 3935 nla_put_u16(stats, TCP_NLA_TIMEOUT_REHASH, tp->timeout_rehash); 3936 nla_put_u32(stats, TCP_NLA_BYTES_NOTSENT, 3937 max_t(int, 0, tp->write_seq - tp->snd_nxt)); 3938 nla_put_u64_64bit(stats, TCP_NLA_EDT, orig_skb->skb_mstamp_ns, 3939 TCP_NLA_PAD); 3940 if (ack_skb) 3941 nla_put_u8(stats, TCP_NLA_TTL, 3942 tcp_skb_ttl_or_hop_limit(ack_skb)); 3943 3944 return stats; 3945 } 3946 3947 static int do_tcp_getsockopt(struct sock *sk, int level, 3948 int optname, char __user *optval, int __user *optlen) 3949 { 3950 struct inet_connection_sock *icsk = inet_csk(sk); 3951 struct tcp_sock *tp = tcp_sk(sk); 3952 struct net *net = sock_net(sk); 3953 int val, len; 3954 3955 if (get_user(len, optlen)) 3956 return -EFAULT; 3957 3958 len = min_t(unsigned int, len, sizeof(int)); 3959 3960 if (len < 0) 3961 return -EINVAL; 3962 3963 switch (optname) { 3964 case TCP_MAXSEG: 3965 val = tp->mss_cache; 3966 if (!val && ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) 3967 val = tp->rx_opt.user_mss; 3968 if (tp->repair) 3969 val = tp->rx_opt.mss_clamp; 3970 break; 3971 case TCP_NODELAY: 3972 val = !!(tp->nonagle&TCP_NAGLE_OFF); 3973 break; 3974 case TCP_CORK: 3975 val = !!(tp->nonagle&TCP_NAGLE_CORK); 3976 break; 3977 case TCP_KEEPIDLE: 3978 val = keepalive_time_when(tp) / HZ; 3979 break; 3980 case TCP_KEEPINTVL: 3981 val = keepalive_intvl_when(tp) / HZ; 3982 break; 3983 case TCP_KEEPCNT: 3984 val = keepalive_probes(tp); 3985 break; 3986 case TCP_SYNCNT: 3987 val = icsk->icsk_syn_retries ? : net->ipv4.sysctl_tcp_syn_retries; 3988 break; 3989 case TCP_LINGER2: 3990 val = tp->linger2; 3991 if (val >= 0) 3992 val = (val ? : net->ipv4.sysctl_tcp_fin_timeout) / HZ; 3993 break; 3994 case TCP_DEFER_ACCEPT: 3995 val = retrans_to_secs(icsk->icsk_accept_queue.rskq_defer_accept, 3996 TCP_TIMEOUT_INIT / HZ, TCP_RTO_MAX / HZ); 3997 break; 3998 case TCP_WINDOW_CLAMP: 3999 val = tp->window_clamp; 4000 break; 4001 case TCP_INFO: { 4002 struct tcp_info info; 4003 4004 if (get_user(len, optlen)) 4005 return -EFAULT; 4006 4007 tcp_get_info(sk, &info); 4008 4009 len = min_t(unsigned int, len, sizeof(info)); 4010 if (put_user(len, optlen)) 4011 return -EFAULT; 4012 if (copy_to_user(optval, &info, len)) 4013 return -EFAULT; 4014 return 0; 4015 } 4016 case TCP_CC_INFO: { 4017 const struct tcp_congestion_ops *ca_ops; 4018 union tcp_cc_info info; 4019 size_t sz = 0; 4020 int attr; 4021 4022 if (get_user(len, optlen)) 4023 return -EFAULT; 4024 4025 ca_ops = icsk->icsk_ca_ops; 4026 if (ca_ops && ca_ops->get_info) 4027 sz = ca_ops->get_info(sk, ~0U, &attr, &info); 4028 4029 len = min_t(unsigned int, len, sz); 4030 if (put_user(len, optlen)) 4031 return -EFAULT; 4032 if (copy_to_user(optval, &info, len)) 4033 return -EFAULT; 4034 return 0; 4035 } 4036 case TCP_QUICKACK: 4037 val = !inet_csk_in_pingpong_mode(sk); 4038 break; 4039 4040 case TCP_CONGESTION: 4041 if (get_user(len, optlen)) 4042 return -EFAULT; 4043 len = min_t(unsigned int, len, TCP_CA_NAME_MAX); 4044 if (put_user(len, optlen)) 4045 return -EFAULT; 4046 if (copy_to_user(optval, icsk->icsk_ca_ops->name, len)) 4047 return -EFAULT; 4048 return 0; 4049 4050 case TCP_ULP: 4051 if (get_user(len, optlen)) 4052 return -EFAULT; 4053 len = min_t(unsigned int, len, TCP_ULP_NAME_MAX); 4054 if (!icsk->icsk_ulp_ops) { 4055 if (put_user(0, optlen)) 4056 return -EFAULT; 4057 return 0; 4058 } 4059 if (put_user(len, optlen)) 4060 return -EFAULT; 4061 if (copy_to_user(optval, icsk->icsk_ulp_ops->name, len)) 4062 return -EFAULT; 4063 return 0; 4064 4065 case TCP_FASTOPEN_KEY: { 4066 u64 key[TCP_FASTOPEN_KEY_BUF_LENGTH / sizeof(u64)]; 4067 unsigned int key_len; 4068 4069 if (get_user(len, optlen)) 4070 return -EFAULT; 4071 4072 key_len = tcp_fastopen_get_cipher(net, icsk, key) * 4073 TCP_FASTOPEN_KEY_LENGTH; 4074 len = min_t(unsigned int, len, key_len); 4075 if (put_user(len, optlen)) 4076 return -EFAULT; 4077 if (copy_to_user(optval, key, len)) 4078 return -EFAULT; 4079 return 0; 4080 } 4081 case TCP_THIN_LINEAR_TIMEOUTS: 4082 val = tp->thin_lto; 4083 break; 4084 4085 case TCP_THIN_DUPACK: 4086 val = 0; 4087 break; 4088 4089 case TCP_REPAIR: 4090 val = tp->repair; 4091 break; 4092 4093 case TCP_REPAIR_QUEUE: 4094 if (tp->repair) 4095 val = tp->repair_queue; 4096 else 4097 return -EINVAL; 4098 break; 4099 4100 case TCP_REPAIR_WINDOW: { 4101 struct tcp_repair_window opt; 4102 4103 if (get_user(len, optlen)) 4104 return -EFAULT; 4105 4106 if (len != sizeof(opt)) 4107 return -EINVAL; 4108 4109 if (!tp->repair) 4110 return -EPERM; 4111 4112 opt.snd_wl1 = tp->snd_wl1; 4113 opt.snd_wnd = tp->snd_wnd; 4114 opt.max_window = tp->max_window; 4115 opt.rcv_wnd = tp->rcv_wnd; 4116 opt.rcv_wup = tp->rcv_wup; 4117 4118 if (copy_to_user(optval, &opt, len)) 4119 return -EFAULT; 4120 return 0; 4121 } 4122 case TCP_QUEUE_SEQ: 4123 if (tp->repair_queue == TCP_SEND_QUEUE) 4124 val = tp->write_seq; 4125 else if (tp->repair_queue == TCP_RECV_QUEUE) 4126 val = tp->rcv_nxt; 4127 else 4128 return -EINVAL; 4129 break; 4130 4131 case TCP_USER_TIMEOUT: 4132 val = icsk->icsk_user_timeout; 4133 break; 4134 4135 case TCP_FASTOPEN: 4136 val = icsk->icsk_accept_queue.fastopenq.max_qlen; 4137 break; 4138 4139 case TCP_FASTOPEN_CONNECT: 4140 val = tp->fastopen_connect; 4141 break; 4142 4143 case TCP_FASTOPEN_NO_COOKIE: 4144 val = tp->fastopen_no_cookie; 4145 break; 4146 4147 case TCP_TX_DELAY: 4148 val = tp->tcp_tx_delay; 4149 break; 4150 4151 case TCP_TIMESTAMP: 4152 val = tcp_time_stamp_raw() + tp->tsoffset; 4153 break; 4154 case TCP_NOTSENT_LOWAT: 4155 val = tp->notsent_lowat; 4156 break; 4157 case TCP_INQ: 4158 val = tp->recvmsg_inq; 4159 break; 4160 case TCP_SAVE_SYN: 4161 val = tp->save_syn; 4162 break; 4163 case TCP_SAVED_SYN: { 4164 if (get_user(len, optlen)) 4165 return -EFAULT; 4166 4167 lock_sock(sk); 4168 if (tp->saved_syn) { 4169 if (len < tcp_saved_syn_len(tp->saved_syn)) { 4170 if (put_user(tcp_saved_syn_len(tp->saved_syn), 4171 optlen)) { 4172 release_sock(sk); 4173 return -EFAULT; 4174 } 4175 release_sock(sk); 4176 return -EINVAL; 4177 } 4178 len = tcp_saved_syn_len(tp->saved_syn); 4179 if (put_user(len, optlen)) { 4180 release_sock(sk); 4181 return -EFAULT; 4182 } 4183 if (copy_to_user(optval, tp->saved_syn->data, len)) { 4184 release_sock(sk); 4185 return -EFAULT; 4186 } 4187 tcp_saved_syn_free(tp); 4188 release_sock(sk); 4189 } else { 4190 release_sock(sk); 4191 len = 0; 4192 if (put_user(len, optlen)) 4193 return -EFAULT; 4194 } 4195 return 0; 4196 } 4197 #ifdef CONFIG_MMU 4198 case TCP_ZEROCOPY_RECEIVE: { 4199 struct scm_timestamping_internal tss; 4200 struct tcp_zerocopy_receive zc = {}; 4201 int err; 4202 4203 if (get_user(len, optlen)) 4204 return -EFAULT; 4205 if (len < 0 || 4206 len < offsetofend(struct tcp_zerocopy_receive, length)) 4207 return -EINVAL; 4208 if (unlikely(len > sizeof(zc))) { 4209 err = check_zeroed_user(optval + sizeof(zc), 4210 len - sizeof(zc)); 4211 if (err < 1) 4212 return err == 0 ? -EINVAL : err; 4213 len = sizeof(zc); 4214 if (put_user(len, optlen)) 4215 return -EFAULT; 4216 } 4217 if (copy_from_user(&zc, optval, len)) 4218 return -EFAULT; 4219 if (zc.reserved) 4220 return -EINVAL; 4221 if (zc.msg_flags & ~(TCP_VALID_ZC_MSG_FLAGS)) 4222 return -EINVAL; 4223 lock_sock(sk); 4224 err = tcp_zerocopy_receive(sk, &zc, &tss); 4225 err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname, 4226 &zc, &len, err); 4227 release_sock(sk); 4228 sk_defer_free_flush(sk); 4229 if (len >= offsetofend(struct tcp_zerocopy_receive, msg_flags)) 4230 goto zerocopy_rcv_cmsg; 4231 switch (len) { 4232 case offsetofend(struct tcp_zerocopy_receive, msg_flags): 4233 goto zerocopy_rcv_cmsg; 4234 case offsetofend(struct tcp_zerocopy_receive, msg_controllen): 4235 case offsetofend(struct tcp_zerocopy_receive, msg_control): 4236 case offsetofend(struct tcp_zerocopy_receive, flags): 4237 case offsetofend(struct tcp_zerocopy_receive, copybuf_len): 4238 case offsetofend(struct tcp_zerocopy_receive, copybuf_address): 4239 case offsetofend(struct tcp_zerocopy_receive, err): 4240 goto zerocopy_rcv_sk_err; 4241 case offsetofend(struct tcp_zerocopy_receive, inq): 4242 goto zerocopy_rcv_inq; 4243 case offsetofend(struct tcp_zerocopy_receive, length): 4244 default: 4245 goto zerocopy_rcv_out; 4246 } 4247 zerocopy_rcv_cmsg: 4248 if (zc.msg_flags & TCP_CMSG_TS) 4249 tcp_zc_finalize_rx_tstamp(sk, &zc, &tss); 4250 else 4251 zc.msg_flags = 0; 4252 zerocopy_rcv_sk_err: 4253 if (!err) 4254 zc.err = sock_error(sk); 4255 zerocopy_rcv_inq: 4256 zc.inq = tcp_inq_hint(sk); 4257 zerocopy_rcv_out: 4258 if (!err && copy_to_user(optval, &zc, len)) 4259 err = -EFAULT; 4260 return err; 4261 } 4262 #endif 4263 default: 4264 return -ENOPROTOOPT; 4265 } 4266 4267 if (put_user(len, optlen)) 4268 return -EFAULT; 4269 if (copy_to_user(optval, &val, len)) 4270 return -EFAULT; 4271 return 0; 4272 } 4273 4274 bool tcp_bpf_bypass_getsockopt(int level, int optname) 4275 { 4276 /* TCP do_tcp_getsockopt has optimized getsockopt implementation 4277 * to avoid extra socket lock for TCP_ZEROCOPY_RECEIVE. 4278 */ 4279 if (level == SOL_TCP && optname == TCP_ZEROCOPY_RECEIVE) 4280 return true; 4281 4282 return false; 4283 } 4284 EXPORT_SYMBOL(tcp_bpf_bypass_getsockopt); 4285 4286 int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval, 4287 int __user *optlen) 4288 { 4289 struct inet_connection_sock *icsk = inet_csk(sk); 4290 4291 if (level != SOL_TCP) 4292 return icsk->icsk_af_ops->getsockopt(sk, level, optname, 4293 optval, optlen); 4294 return do_tcp_getsockopt(sk, level, optname, optval, optlen); 4295 } 4296 EXPORT_SYMBOL(tcp_getsockopt); 4297 4298 #ifdef CONFIG_TCP_MD5SIG 4299 static DEFINE_PER_CPU(struct tcp_md5sig_pool, tcp_md5sig_pool); 4300 static DEFINE_MUTEX(tcp_md5sig_mutex); 4301 static bool tcp_md5sig_pool_populated = false; 4302 4303 static void __tcp_alloc_md5sig_pool(void) 4304 { 4305 struct crypto_ahash *hash; 4306 int cpu; 4307 4308 hash = crypto_alloc_ahash("md5", 0, CRYPTO_ALG_ASYNC); 4309 if (IS_ERR(hash)) 4310 return; 4311 4312 for_each_possible_cpu(cpu) { 4313 void *scratch = per_cpu(tcp_md5sig_pool, cpu).scratch; 4314 struct ahash_request *req; 4315 4316 if (!scratch) { 4317 scratch = kmalloc_node(sizeof(union tcp_md5sum_block) + 4318 sizeof(struct tcphdr), 4319 GFP_KERNEL, 4320 cpu_to_node(cpu)); 4321 if (!scratch) 4322 return; 4323 per_cpu(tcp_md5sig_pool, cpu).scratch = scratch; 4324 } 4325 if (per_cpu(tcp_md5sig_pool, cpu).md5_req) 4326 continue; 4327 4328 req = ahash_request_alloc(hash, GFP_KERNEL); 4329 if (!req) 4330 return; 4331 4332 ahash_request_set_callback(req, 0, NULL, NULL); 4333 4334 per_cpu(tcp_md5sig_pool, cpu).md5_req = req; 4335 } 4336 /* before setting tcp_md5sig_pool_populated, we must commit all writes 4337 * to memory. See smp_rmb() in tcp_get_md5sig_pool() 4338 */ 4339 smp_wmb(); 4340 tcp_md5sig_pool_populated = true; 4341 } 4342 4343 bool tcp_alloc_md5sig_pool(void) 4344 { 4345 if (unlikely(!tcp_md5sig_pool_populated)) { 4346 mutex_lock(&tcp_md5sig_mutex); 4347 4348 if (!tcp_md5sig_pool_populated) { 4349 __tcp_alloc_md5sig_pool(); 4350 if (tcp_md5sig_pool_populated) 4351 static_branch_inc(&tcp_md5_needed); 4352 } 4353 4354 mutex_unlock(&tcp_md5sig_mutex); 4355 } 4356 return tcp_md5sig_pool_populated; 4357 } 4358 EXPORT_SYMBOL(tcp_alloc_md5sig_pool); 4359 4360 4361 /** 4362 * tcp_get_md5sig_pool - get md5sig_pool for this user 4363 * 4364 * We use percpu structure, so if we succeed, we exit with preemption 4365 * and BH disabled, to make sure another thread or softirq handling 4366 * wont try to get same context. 4367 */ 4368 struct tcp_md5sig_pool *tcp_get_md5sig_pool(void) 4369 { 4370 local_bh_disable(); 4371 4372 if (tcp_md5sig_pool_populated) { 4373 /* coupled with smp_wmb() in __tcp_alloc_md5sig_pool() */ 4374 smp_rmb(); 4375 return this_cpu_ptr(&tcp_md5sig_pool); 4376 } 4377 local_bh_enable(); 4378 return NULL; 4379 } 4380 EXPORT_SYMBOL(tcp_get_md5sig_pool); 4381 4382 int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *hp, 4383 const struct sk_buff *skb, unsigned int header_len) 4384 { 4385 struct scatterlist sg; 4386 const struct tcphdr *tp = tcp_hdr(skb); 4387 struct ahash_request *req = hp->md5_req; 4388 unsigned int i; 4389 const unsigned int head_data_len = skb_headlen(skb) > header_len ? 4390 skb_headlen(skb) - header_len : 0; 4391 const struct skb_shared_info *shi = skb_shinfo(skb); 4392 struct sk_buff *frag_iter; 4393 4394 sg_init_table(&sg, 1); 4395 4396 sg_set_buf(&sg, ((u8 *) tp) + header_len, head_data_len); 4397 ahash_request_set_crypt(req, &sg, NULL, head_data_len); 4398 if (crypto_ahash_update(req)) 4399 return 1; 4400 4401 for (i = 0; i < shi->nr_frags; ++i) { 4402 const skb_frag_t *f = &shi->frags[i]; 4403 unsigned int offset = skb_frag_off(f); 4404 struct page *page = skb_frag_page(f) + (offset >> PAGE_SHIFT); 4405 4406 sg_set_page(&sg, page, skb_frag_size(f), 4407 offset_in_page(offset)); 4408 ahash_request_set_crypt(req, &sg, NULL, skb_frag_size(f)); 4409 if (crypto_ahash_update(req)) 4410 return 1; 4411 } 4412 4413 skb_walk_frags(skb, frag_iter) 4414 if (tcp_md5_hash_skb_data(hp, frag_iter, 0)) 4415 return 1; 4416 4417 return 0; 4418 } 4419 EXPORT_SYMBOL(tcp_md5_hash_skb_data); 4420 4421 int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, const struct tcp_md5sig_key *key) 4422 { 4423 u8 keylen = READ_ONCE(key->keylen); /* paired with WRITE_ONCE() in tcp_md5_do_add */ 4424 struct scatterlist sg; 4425 4426 sg_init_one(&sg, key->key, keylen); 4427 ahash_request_set_crypt(hp->md5_req, &sg, NULL, keylen); 4428 4429 /* We use data_race() because tcp_md5_do_add() might change key->key under us */ 4430 return data_race(crypto_ahash_update(hp->md5_req)); 4431 } 4432 EXPORT_SYMBOL(tcp_md5_hash_key); 4433 4434 /* Called with rcu_read_lock() */ 4435 enum skb_drop_reason 4436 tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb, 4437 const void *saddr, const void *daddr, 4438 int family, int dif, int sdif) 4439 { 4440 /* 4441 * This gets called for each TCP segment that arrives 4442 * so we want to be efficient. 4443 * We have 3 drop cases: 4444 * o No MD5 hash and one expected. 4445 * o MD5 hash and we're not expecting one. 4446 * o MD5 hash and its wrong. 4447 */ 4448 const __u8 *hash_location = NULL; 4449 struct tcp_md5sig_key *hash_expected; 4450 const struct tcphdr *th = tcp_hdr(skb); 4451 struct tcp_sock *tp = tcp_sk(sk); 4452 int genhash, l3index; 4453 u8 newhash[16]; 4454 4455 /* sdif set, means packet ingressed via a device 4456 * in an L3 domain and dif is set to the l3mdev 4457 */ 4458 l3index = sdif ? dif : 0; 4459 4460 hash_expected = tcp_md5_do_lookup(sk, l3index, saddr, family); 4461 hash_location = tcp_parse_md5sig_option(th); 4462 4463 /* We've parsed the options - do we have a hash? */ 4464 if (!hash_expected && !hash_location) 4465 return SKB_NOT_DROPPED_YET; 4466 4467 if (hash_expected && !hash_location) { 4468 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5NOTFOUND); 4469 return SKB_DROP_REASON_TCP_MD5NOTFOUND; 4470 } 4471 4472 if (!hash_expected && hash_location) { 4473 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5UNEXPECTED); 4474 return SKB_DROP_REASON_TCP_MD5UNEXPECTED; 4475 } 4476 4477 /* check the signature */ 4478 genhash = tp->af_specific->calc_md5_hash(newhash, hash_expected, 4479 NULL, skb); 4480 4481 if (genhash || memcmp(hash_location, newhash, 16) != 0) { 4482 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5FAILURE); 4483 if (family == AF_INET) { 4484 net_info_ratelimited("MD5 Hash failed for (%pI4, %d)->(%pI4, %d)%s L3 index %d\n", 4485 saddr, ntohs(th->source), 4486 daddr, ntohs(th->dest), 4487 genhash ? " tcp_v4_calc_md5_hash failed" 4488 : "", l3index); 4489 } else { 4490 net_info_ratelimited("MD5 Hash %s for [%pI6c]:%u->[%pI6c]:%u L3 index %d\n", 4491 genhash ? "failed" : "mismatch", 4492 saddr, ntohs(th->source), 4493 daddr, ntohs(th->dest), l3index); 4494 } 4495 return SKB_DROP_REASON_TCP_MD5FAILURE; 4496 } 4497 return SKB_NOT_DROPPED_YET; 4498 } 4499 EXPORT_SYMBOL(tcp_inbound_md5_hash); 4500 4501 #endif 4502 4503 void tcp_done(struct sock *sk) 4504 { 4505 struct request_sock *req; 4506 4507 /* We might be called with a new socket, after 4508 * inet_csk_prepare_forced_close() has been called 4509 * so we can not use lockdep_sock_is_held(sk) 4510 */ 4511 req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk, 1); 4512 4513 if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV) 4514 TCP_INC_STATS(sock_net(sk), TCP_MIB_ATTEMPTFAILS); 4515 4516 tcp_set_state(sk, TCP_CLOSE); 4517 tcp_clear_xmit_timers(sk); 4518 if (req) 4519 reqsk_fastopen_remove(sk, req, false); 4520 4521 sk->sk_shutdown = SHUTDOWN_MASK; 4522 4523 if (!sock_flag(sk, SOCK_DEAD)) 4524 sk->sk_state_change(sk); 4525 else 4526 inet_csk_destroy_sock(sk); 4527 } 4528 EXPORT_SYMBOL_GPL(tcp_done); 4529 4530 int tcp_abort(struct sock *sk, int err) 4531 { 4532 if (!sk_fullsock(sk)) { 4533 if (sk->sk_state == TCP_NEW_SYN_RECV) { 4534 struct request_sock *req = inet_reqsk(sk); 4535 4536 local_bh_disable(); 4537 inet_csk_reqsk_queue_drop(req->rsk_listener, req); 4538 local_bh_enable(); 4539 return 0; 4540 } 4541 return -EOPNOTSUPP; 4542 } 4543 4544 /* Don't race with userspace socket closes such as tcp_close. */ 4545 lock_sock(sk); 4546 4547 if (sk->sk_state == TCP_LISTEN) { 4548 tcp_set_state(sk, TCP_CLOSE); 4549 inet_csk_listen_stop(sk); 4550 } 4551 4552 /* Don't race with BH socket closes such as inet_csk_listen_stop. */ 4553 local_bh_disable(); 4554 bh_lock_sock(sk); 4555 4556 if (!sock_flag(sk, SOCK_DEAD)) { 4557 sk->sk_err = err; 4558 /* This barrier is coupled with smp_rmb() in tcp_poll() */ 4559 smp_wmb(); 4560 sk_error_report(sk); 4561 if (tcp_need_reset(sk->sk_state)) 4562 tcp_send_active_reset(sk, GFP_ATOMIC); 4563 tcp_done(sk); 4564 } 4565 4566 bh_unlock_sock(sk); 4567 local_bh_enable(); 4568 tcp_write_queue_purge(sk); 4569 release_sock(sk); 4570 return 0; 4571 } 4572 EXPORT_SYMBOL_GPL(tcp_abort); 4573 4574 extern struct tcp_congestion_ops tcp_reno; 4575 4576 static __initdata unsigned long thash_entries; 4577 static int __init set_thash_entries(char *str) 4578 { 4579 ssize_t ret; 4580 4581 if (!str) 4582 return 0; 4583 4584 ret = kstrtoul(str, 0, &thash_entries); 4585 if (ret) 4586 return 0; 4587 4588 return 1; 4589 } 4590 __setup("thash_entries=", set_thash_entries); 4591 4592 static void __init tcp_init_mem(void) 4593 { 4594 unsigned long limit = nr_free_buffer_pages() / 16; 4595 4596 limit = max(limit, 128UL); 4597 sysctl_tcp_mem[0] = limit / 4 * 3; /* 4.68 % */ 4598 sysctl_tcp_mem[1] = limit; /* 6.25 % */ 4599 sysctl_tcp_mem[2] = sysctl_tcp_mem[0] * 2; /* 9.37 % */ 4600 } 4601 4602 void __init tcp_init(void) 4603 { 4604 int max_rshare, max_wshare, cnt; 4605 unsigned long limit; 4606 unsigned int i; 4607 4608 BUILD_BUG_ON(TCP_MIN_SND_MSS <= MAX_TCP_OPTION_SPACE); 4609 BUILD_BUG_ON(sizeof(struct tcp_skb_cb) > 4610 sizeof_field(struct sk_buff, cb)); 4611 4612 percpu_counter_init(&tcp_sockets_allocated, 0, GFP_KERNEL); 4613 4614 timer_setup(&tcp_orphan_timer, tcp_orphan_update, TIMER_DEFERRABLE); 4615 mod_timer(&tcp_orphan_timer, jiffies + TCP_ORPHAN_TIMER_PERIOD); 4616 4617 inet_hashinfo_init(&tcp_hashinfo); 4618 inet_hashinfo2_init(&tcp_hashinfo, "tcp_listen_portaddr_hash", 4619 thash_entries, 21, /* one slot per 2 MB*/ 4620 0, 64 * 1024); 4621 tcp_hashinfo.bind_bucket_cachep = 4622 kmem_cache_create("tcp_bind_bucket", 4623 sizeof(struct inet_bind_bucket), 0, 4624 SLAB_HWCACHE_ALIGN | SLAB_PANIC | 4625 SLAB_ACCOUNT, 4626 NULL); 4627 4628 /* Size and allocate the main established and bind bucket 4629 * hash tables. 4630 * 4631 * The methodology is similar to that of the buffer cache. 4632 */ 4633 tcp_hashinfo.ehash = 4634 alloc_large_system_hash("TCP established", 4635 sizeof(struct inet_ehash_bucket), 4636 thash_entries, 4637 17, /* one slot per 128 KB of memory */ 4638 0, 4639 NULL, 4640 &tcp_hashinfo.ehash_mask, 4641 0, 4642 thash_entries ? 0 : 512 * 1024); 4643 for (i = 0; i <= tcp_hashinfo.ehash_mask; i++) 4644 INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].chain, i); 4645 4646 if (inet_ehash_locks_alloc(&tcp_hashinfo)) 4647 panic("TCP: failed to alloc ehash_locks"); 4648 tcp_hashinfo.bhash = 4649 alloc_large_system_hash("TCP bind", 4650 sizeof(struct inet_bind_hashbucket), 4651 tcp_hashinfo.ehash_mask + 1, 4652 17, /* one slot per 128 KB of memory */ 4653 0, 4654 &tcp_hashinfo.bhash_size, 4655 NULL, 4656 0, 4657 64 * 1024); 4658 tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size; 4659 for (i = 0; i < tcp_hashinfo.bhash_size; i++) { 4660 spin_lock_init(&tcp_hashinfo.bhash[i].lock); 4661 INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain); 4662 } 4663 4664 4665 cnt = tcp_hashinfo.ehash_mask + 1; 4666 sysctl_tcp_max_orphans = cnt / 2; 4667 4668 tcp_init_mem(); 4669 /* Set per-socket limits to no more than 1/128 the pressure threshold */ 4670 limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7); 4671 max_wshare = min(4UL*1024*1024, limit); 4672 max_rshare = min(6UL*1024*1024, limit); 4673 4674 init_net.ipv4.sysctl_tcp_wmem[0] = SK_MEM_QUANTUM; 4675 init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024; 4676 init_net.ipv4.sysctl_tcp_wmem[2] = max(64*1024, max_wshare); 4677 4678 init_net.ipv4.sysctl_tcp_rmem[0] = SK_MEM_QUANTUM; 4679 init_net.ipv4.sysctl_tcp_rmem[1] = 131072; 4680 init_net.ipv4.sysctl_tcp_rmem[2] = max(131072, max_rshare); 4681 4682 pr_info("Hash tables configured (established %u bind %u)\n", 4683 tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size); 4684 4685 tcp_v4_init(); 4686 tcp_metrics_init(); 4687 BUG_ON(tcp_register_congestion_control(&tcp_reno) != 0); 4688 tcp_tasklet_init(); 4689 mptcp_init(); 4690 } 4691