1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * Implementation of the Transmission Control Protocol(TCP). 8 * 9 * Authors: Ross Biro 10 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 11 * Mark Evans, <evansmp@uhura.aston.ac.uk> 12 * Corey Minyard <wf-rch!minyard@relay.EU.net> 13 * Florian La Roche, <flla@stud.uni-sb.de> 14 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu> 15 * Linus Torvalds, <torvalds@cs.helsinki.fi> 16 * Alan Cox, <gw4pts@gw4pts.ampr.org> 17 * Matthew Dillon, <dillon@apollo.west.oic.com> 18 * Arnt Gulbrandsen, <agulbra@nvg.unit.no> 19 * Jorge Cwik, <jorge@laser.satlink.net> 20 * 21 * Fixes: 22 * Alan Cox : Numerous verify_area() calls 23 * Alan Cox : Set the ACK bit on a reset 24 * Alan Cox : Stopped it crashing if it closed while 25 * sk->inuse=1 and was trying to connect 26 * (tcp_err()). 27 * Alan Cox : All icmp error handling was broken 28 * pointers passed where wrong and the 29 * socket was looked up backwards. Nobody 30 * tested any icmp error code obviously. 31 * Alan Cox : tcp_err() now handled properly. It 32 * wakes people on errors. poll 33 * behaves and the icmp error race 34 * has gone by moving it into sock.c 35 * Alan Cox : tcp_send_reset() fixed to work for 36 * everything not just packets for 37 * unknown sockets. 38 * Alan Cox : tcp option processing. 39 * Alan Cox : Reset tweaked (still not 100%) [Had 40 * syn rule wrong] 41 * Herp Rosmanith : More reset fixes 42 * Alan Cox : No longer acks invalid rst frames. 43 * Acking any kind of RST is right out. 44 * Alan Cox : Sets an ignore me flag on an rst 45 * receive otherwise odd bits of prattle 46 * escape still 47 * Alan Cox : Fixed another acking RST frame bug. 48 * Should stop LAN workplace lockups. 49 * Alan Cox : Some tidyups using the new skb list 50 * facilities 51 * Alan Cox : sk->keepopen now seems to work 52 * Alan Cox : Pulls options out correctly on accepts 53 * Alan Cox : Fixed assorted sk->rqueue->next errors 54 * Alan Cox : PSH doesn't end a TCP read. Switched a 55 * bit to skb ops. 56 * Alan Cox : Tidied tcp_data to avoid a potential 57 * nasty. 58 * Alan Cox : Added some better commenting, as the 59 * tcp is hard to follow 60 * Alan Cox : Removed incorrect check for 20 * psh 61 * Michael O'Reilly : ack < copied bug fix. 62 * Johannes Stille : Misc tcp fixes (not all in yet). 63 * Alan Cox : FIN with no memory -> CRASH 64 * Alan Cox : Added socket option proto entries. 65 * Also added awareness of them to accept. 66 * Alan Cox : Added TCP options (SOL_TCP) 67 * Alan Cox : Switched wakeup calls to callbacks, 68 * so the kernel can layer network 69 * sockets. 70 * Alan Cox : Use ip_tos/ip_ttl settings. 71 * Alan Cox : Handle FIN (more) properly (we hope). 72 * Alan Cox : RST frames sent on unsynchronised 73 * state ack error. 74 * Alan Cox : Put in missing check for SYN bit. 75 * Alan Cox : Added tcp_select_window() aka NET2E 76 * window non shrink trick. 77 * Alan Cox : Added a couple of small NET2E timer 78 * fixes 79 * Charles Hedrick : TCP fixes 80 * Toomas Tamm : TCP window fixes 81 * Alan Cox : Small URG fix to rlogin ^C ack fight 82 * Charles Hedrick : Rewrote most of it to actually work 83 * Linus : Rewrote tcp_read() and URG handling 84 * completely 85 * Gerhard Koerting: Fixed some missing timer handling 86 * Matthew Dillon : Reworked TCP machine states as per RFC 87 * Gerhard Koerting: PC/TCP workarounds 88 * Adam Caldwell : Assorted timer/timing errors 89 * Matthew Dillon : Fixed another RST bug 90 * Alan Cox : Move to kernel side addressing changes. 91 * Alan Cox : Beginning work on TCP fastpathing 92 * (not yet usable) 93 * Arnt Gulbrandsen: Turbocharged tcp_check() routine. 94 * Alan Cox : TCP fast path debugging 95 * Alan Cox : Window clamping 96 * Michael Riepe : Bug in tcp_check() 97 * Matt Dillon : More TCP improvements and RST bug fixes 98 * Matt Dillon : Yet more small nasties remove from the 99 * TCP code (Be very nice to this man if 100 * tcp finally works 100%) 8) 101 * Alan Cox : BSD accept semantics. 102 * Alan Cox : Reset on closedown bug. 103 * Peter De Schrijver : ENOTCONN check missing in tcp_sendto(). 104 * Michael Pall : Handle poll() after URG properly in 105 * all cases. 106 * Michael Pall : Undo the last fix in tcp_read_urg() 107 * (multi URG PUSH broke rlogin). 108 * Michael Pall : Fix the multi URG PUSH problem in 109 * tcp_readable(), poll() after URG 110 * works now. 111 * Michael Pall : recv(...,MSG_OOB) never blocks in the 112 * BSD api. 113 * Alan Cox : Changed the semantics of sk->socket to 114 * fix a race and a signal problem with 115 * accept() and async I/O. 116 * Alan Cox : Relaxed the rules on tcp_sendto(). 117 * Yury Shevchuk : Really fixed accept() blocking problem. 118 * Craig I. Hagan : Allow for BSD compatible TIME_WAIT for 119 * clients/servers which listen in on 120 * fixed ports. 121 * Alan Cox : Cleaned the above up and shrank it to 122 * a sensible code size. 123 * Alan Cox : Self connect lockup fix. 124 * Alan Cox : No connect to multicast. 125 * Ross Biro : Close unaccepted children on master 126 * socket close. 127 * Alan Cox : Reset tracing code. 128 * Alan Cox : Spurious resets on shutdown. 129 * Alan Cox : Giant 15 minute/60 second timer error 130 * Alan Cox : Small whoops in polling before an 131 * accept. 132 * Alan Cox : Kept the state trace facility since 133 * it's handy for debugging. 134 * Alan Cox : More reset handler fixes. 135 * Alan Cox : Started rewriting the code based on 136 * the RFC's for other useful protocol 137 * references see: Comer, KA9Q NOS, and 138 * for a reference on the difference 139 * between specifications and how BSD 140 * works see the 4.4lite source. 141 * A.N.Kuznetsov : Don't time wait on completion of tidy 142 * close. 143 * Linus Torvalds : Fin/Shutdown & copied_seq changes. 144 * Linus Torvalds : Fixed BSD port reuse to work first syn 145 * Alan Cox : Reimplemented timers as per the RFC 146 * and using multiple timers for sanity. 147 * Alan Cox : Small bug fixes, and a lot of new 148 * comments. 149 * Alan Cox : Fixed dual reader crash by locking 150 * the buffers (much like datagram.c) 151 * Alan Cox : Fixed stuck sockets in probe. A probe 152 * now gets fed up of retrying without 153 * (even a no space) answer. 154 * Alan Cox : Extracted closing code better 155 * Alan Cox : Fixed the closing state machine to 156 * resemble the RFC. 157 * Alan Cox : More 'per spec' fixes. 158 * Jorge Cwik : Even faster checksumming. 159 * Alan Cox : tcp_data() doesn't ack illegal PSH 160 * only frames. At least one pc tcp stack 161 * generates them. 162 * Alan Cox : Cache last socket. 163 * Alan Cox : Per route irtt. 164 * Matt Day : poll()->select() match BSD precisely on error 165 * Alan Cox : New buffers 166 * Marc Tamsky : Various sk->prot->retransmits and 167 * sk->retransmits misupdating fixed. 168 * Fixed tcp_write_timeout: stuck close, 169 * and TCP syn retries gets used now. 170 * Mark Yarvis : In tcp_read_wakeup(), don't send an 171 * ack if state is TCP_CLOSED. 172 * Alan Cox : Look up device on a retransmit - routes may 173 * change. Doesn't yet cope with MSS shrink right 174 * but it's a start! 175 * Marc Tamsky : Closing in closing fixes. 176 * Mike Shaver : RFC1122 verifications. 177 * Alan Cox : rcv_saddr errors. 178 * Alan Cox : Block double connect(). 179 * Alan Cox : Small hooks for enSKIP. 180 * Alexey Kuznetsov: Path MTU discovery. 181 * Alan Cox : Support soft errors. 182 * Alan Cox : Fix MTU discovery pathological case 183 * when the remote claims no mtu! 184 * Marc Tamsky : TCP_CLOSE fix. 185 * Colin (G3TNE) : Send a reset on syn ack replies in 186 * window but wrong (fixes NT lpd problems) 187 * Pedro Roque : Better TCP window handling, delayed ack. 188 * Joerg Reuter : No modification of locked buffers in 189 * tcp_do_retransmit() 190 * Eric Schenk : Changed receiver side silly window 191 * avoidance algorithm to BSD style 192 * algorithm. This doubles throughput 193 * against machines running Solaris, 194 * and seems to result in general 195 * improvement. 196 * Stefan Magdalinski : adjusted tcp_readable() to fix FIONREAD 197 * Willy Konynenberg : Transparent proxying support. 198 * Mike McLagan : Routing by source 199 * Keith Owens : Do proper merging with partial SKB's in 200 * tcp_do_sendmsg to avoid burstiness. 201 * Eric Schenk : Fix fast close down bug with 202 * shutdown() followed by close(). 203 * Andi Kleen : Make poll agree with SIGIO 204 * Salvatore Sanfilippo : Support SO_LINGER with linger == 1 and 205 * lingertime == 0 (RFC 793 ABORT Call) 206 * Hirokazu Takahashi : Use copy_from_user() instead of 207 * csum_and_copy_from_user() if possible. 208 * 209 * Description of States: 210 * 211 * TCP_SYN_SENT sent a connection request, waiting for ack 212 * 213 * TCP_SYN_RECV received a connection request, sent ack, 214 * waiting for final ack in three-way handshake. 215 * 216 * TCP_ESTABLISHED connection established 217 * 218 * TCP_FIN_WAIT1 our side has shutdown, waiting to complete 219 * transmission of remaining buffered data 220 * 221 * TCP_FIN_WAIT2 all buffered data sent, waiting for remote 222 * to shutdown 223 * 224 * TCP_CLOSING both sides have shutdown but we still have 225 * data we have to finish sending 226 * 227 * TCP_TIME_WAIT timeout to catch resent junk before entering 228 * closed, can only be entered from FIN_WAIT2 229 * or CLOSING. Required because the other end 230 * may not have gotten our last ACK causing it 231 * to retransmit the data packet (which we ignore) 232 * 233 * TCP_CLOSE_WAIT remote side has shutdown and is waiting for 234 * us to finish writing our data and to shutdown 235 * (we have to close() to move on to LAST_ACK) 236 * 237 * TCP_LAST_ACK out side has shutdown after remote has 238 * shutdown. There may still be data in our 239 * buffer that we have to finish sending 240 * 241 * TCP_CLOSE socket is finished 242 */ 243 244 #define pr_fmt(fmt) "TCP: " fmt 245 246 #include <crypto/hash.h> 247 #include <linux/kernel.h> 248 #include <linux/module.h> 249 #include <linux/types.h> 250 #include <linux/fcntl.h> 251 #include <linux/poll.h> 252 #include <linux/inet_diag.h> 253 #include <linux/init.h> 254 #include <linux/fs.h> 255 #include <linux/skbuff.h> 256 #include <linux/scatterlist.h> 257 #include <linux/splice.h> 258 #include <linux/net.h> 259 #include <linux/socket.h> 260 #include <linux/random.h> 261 #include <linux/memblock.h> 262 #include <linux/highmem.h> 263 #include <linux/cache.h> 264 #include <linux/err.h> 265 #include <linux/time.h> 266 #include <linux/slab.h> 267 #include <linux/errqueue.h> 268 #include <linux/static_key.h> 269 #include <linux/btf.h> 270 271 #include <net/icmp.h> 272 #include <net/inet_common.h> 273 #include <net/tcp.h> 274 #include <net/mptcp.h> 275 #include <net/proto_memory.h> 276 #include <net/xfrm.h> 277 #include <net/ip.h> 278 #include <net/sock.h> 279 #include <net/rstreason.h> 280 281 #include <linux/uaccess.h> 282 #include <asm/ioctls.h> 283 #include <net/busy_poll.h> 284 #include <net/hotdata.h> 285 #include <trace/events/tcp.h> 286 #include <net/rps.h> 287 288 #include "../core/devmem.h" 289 290 /* Track pending CMSGs. */ 291 enum { 292 TCP_CMSG_INQ = 1, 293 TCP_CMSG_TS = 2 294 }; 295 296 DEFINE_PER_CPU(unsigned int, tcp_orphan_count); 297 EXPORT_PER_CPU_SYMBOL_GPL(tcp_orphan_count); 298 299 DEFINE_PER_CPU(u32, tcp_tw_isn); 300 EXPORT_PER_CPU_SYMBOL_GPL(tcp_tw_isn); 301 302 long sysctl_tcp_mem[3] __read_mostly; 303 EXPORT_IPV6_MOD(sysctl_tcp_mem); 304 305 atomic_long_t tcp_memory_allocated ____cacheline_aligned_in_smp; /* Current allocated memory. */ 306 EXPORT_IPV6_MOD(tcp_memory_allocated); 307 DEFINE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc); 308 EXPORT_PER_CPU_SYMBOL_GPL(tcp_memory_per_cpu_fw_alloc); 309 310 #if IS_ENABLED(CONFIG_SMC) 311 DEFINE_STATIC_KEY_FALSE(tcp_have_smc); 312 EXPORT_SYMBOL(tcp_have_smc); 313 #endif 314 315 /* 316 * Current number of TCP sockets. 317 */ 318 struct percpu_counter tcp_sockets_allocated ____cacheline_aligned_in_smp; 319 EXPORT_IPV6_MOD(tcp_sockets_allocated); 320 321 /* 322 * TCP splice context 323 */ 324 struct tcp_splice_state { 325 struct pipe_inode_info *pipe; 326 size_t len; 327 unsigned int flags; 328 }; 329 330 /* 331 * Pressure flag: try to collapse. 332 * Technical note: it is used by multiple contexts non atomically. 333 * All the __sk_mem_schedule() is of this nature: accounting 334 * is strict, actions are advisory and have some latency. 335 */ 336 unsigned long tcp_memory_pressure __read_mostly; 337 EXPORT_SYMBOL_GPL(tcp_memory_pressure); 338 339 void tcp_enter_memory_pressure(struct sock *sk) 340 { 341 unsigned long val; 342 343 if (READ_ONCE(tcp_memory_pressure)) 344 return; 345 val = jiffies; 346 347 if (!val) 348 val--; 349 if (!cmpxchg(&tcp_memory_pressure, 0, val)) 350 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURES); 351 } 352 EXPORT_IPV6_MOD_GPL(tcp_enter_memory_pressure); 353 354 void tcp_leave_memory_pressure(struct sock *sk) 355 { 356 unsigned long val; 357 358 if (!READ_ONCE(tcp_memory_pressure)) 359 return; 360 val = xchg(&tcp_memory_pressure, 0); 361 if (val) 362 NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURESCHRONO, 363 jiffies_to_msecs(jiffies - val)); 364 } 365 EXPORT_IPV6_MOD_GPL(tcp_leave_memory_pressure); 366 367 /* Convert seconds to retransmits based on initial and max timeout */ 368 static u8 secs_to_retrans(int seconds, int timeout, int rto_max) 369 { 370 u8 res = 0; 371 372 if (seconds > 0) { 373 int period = timeout; 374 375 res = 1; 376 while (seconds > period && res < 255) { 377 res++; 378 timeout <<= 1; 379 if (timeout > rto_max) 380 timeout = rto_max; 381 period += timeout; 382 } 383 } 384 return res; 385 } 386 387 /* Convert retransmits to seconds based on initial and max timeout */ 388 static int retrans_to_secs(u8 retrans, int timeout, int rto_max) 389 { 390 int period = 0; 391 392 if (retrans > 0) { 393 period = timeout; 394 while (--retrans) { 395 timeout <<= 1; 396 if (timeout > rto_max) 397 timeout = rto_max; 398 period += timeout; 399 } 400 } 401 return period; 402 } 403 404 static u64 tcp_compute_delivery_rate(const struct tcp_sock *tp) 405 { 406 u32 rate = READ_ONCE(tp->rate_delivered); 407 u32 intv = READ_ONCE(tp->rate_interval_us); 408 u64 rate64 = 0; 409 410 if (rate && intv) { 411 rate64 = (u64)rate * tp->mss_cache * USEC_PER_SEC; 412 do_div(rate64, intv); 413 } 414 return rate64; 415 } 416 417 /* Address-family independent initialization for a tcp_sock. 418 * 419 * NOTE: A lot of things set to zero explicitly by call to 420 * sk_alloc() so need not be done here. 421 */ 422 void tcp_init_sock(struct sock *sk) 423 { 424 struct inet_connection_sock *icsk = inet_csk(sk); 425 struct tcp_sock *tp = tcp_sk(sk); 426 int rto_min_us, rto_max_ms; 427 428 tp->out_of_order_queue = RB_ROOT; 429 sk->tcp_rtx_queue = RB_ROOT; 430 tcp_init_xmit_timers(sk); 431 INIT_LIST_HEAD(&tp->tsq_node); 432 INIT_LIST_HEAD(&tp->tsorted_sent_queue); 433 434 icsk->icsk_rto = TCP_TIMEOUT_INIT; 435 436 rto_max_ms = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rto_max_ms); 437 icsk->icsk_rto_max = msecs_to_jiffies(rto_max_ms); 438 439 rto_min_us = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rto_min_us); 440 icsk->icsk_rto_min = usecs_to_jiffies(rto_min_us); 441 icsk->icsk_delack_max = TCP_DELACK_MAX; 442 tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT); 443 minmax_reset(&tp->rtt_min, tcp_jiffies32, ~0U); 444 445 /* So many TCP implementations out there (incorrectly) count the 446 * initial SYN frame in their delayed-ACK and congestion control 447 * algorithms that we must have the following bandaid to talk 448 * efficiently to them. -DaveM 449 */ 450 tcp_snd_cwnd_set(tp, TCP_INIT_CWND); 451 452 /* There's a bubble in the pipe until at least the first ACK. */ 453 tp->app_limited = ~0U; 454 tp->rate_app_limited = 1; 455 456 /* See draft-stevens-tcpca-spec-01 for discussion of the 457 * initialization of these values. 458 */ 459 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; 460 tp->snd_cwnd_clamp = ~0; 461 tp->mss_cache = TCP_MSS_DEFAULT; 462 463 tp->reordering = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reordering); 464 tcp_assign_congestion_control(sk); 465 466 tp->tsoffset = 0; 467 tp->rack.reo_wnd_steps = 1; 468 469 sk->sk_write_space = sk_stream_write_space; 470 sock_set_flag(sk, SOCK_USE_WRITE_QUEUE); 471 472 icsk->icsk_sync_mss = tcp_sync_mss; 473 474 WRITE_ONCE(sk->sk_sndbuf, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_wmem[1])); 475 WRITE_ONCE(sk->sk_rcvbuf, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[1])); 476 tcp_scaling_ratio_init(sk); 477 478 set_bit(SOCK_SUPPORT_ZC, &sk->sk_socket->flags); 479 sk_sockets_allocated_inc(sk); 480 xa_init_flags(&sk->sk_user_frags, XA_FLAGS_ALLOC1); 481 } 482 EXPORT_IPV6_MOD(tcp_init_sock); 483 484 static void tcp_tx_timestamp(struct sock *sk, struct sockcm_cookie *sockc) 485 { 486 struct sk_buff *skb = tcp_write_queue_tail(sk); 487 u32 tsflags = sockc->tsflags; 488 489 if (tsflags && skb) { 490 struct skb_shared_info *shinfo = skb_shinfo(skb); 491 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb); 492 493 sock_tx_timestamp(sk, sockc, &shinfo->tx_flags); 494 if (tsflags & SOF_TIMESTAMPING_TX_ACK) 495 tcb->txstamp_ack |= TSTAMP_ACK_SK; 496 if (tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK) 497 shinfo->tskey = TCP_SKB_CB(skb)->seq + skb->len - 1; 498 } 499 500 if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) && 501 SK_BPF_CB_FLAG_TEST(sk, SK_BPF_CB_TX_TIMESTAMPING) && skb) 502 bpf_skops_tx_timestamping(sk, skb, BPF_SOCK_OPS_TSTAMP_SENDMSG_CB); 503 } 504 505 static bool tcp_stream_is_readable(struct sock *sk, int target) 506 { 507 if (tcp_epollin_ready(sk, target)) 508 return true; 509 return sk_is_readable(sk); 510 } 511 512 /* 513 * Wait for a TCP event. 514 * 515 * Note that we don't need to lock the socket, as the upper poll layers 516 * take care of normal races (between the test and the event) and we don't 517 * go look at any of the socket buffers directly. 518 */ 519 __poll_t tcp_poll(struct file *file, struct socket *sock, poll_table *wait) 520 { 521 __poll_t mask; 522 struct sock *sk = sock->sk; 523 const struct tcp_sock *tp = tcp_sk(sk); 524 u8 shutdown; 525 int state; 526 527 sock_poll_wait(file, sock, wait); 528 529 state = inet_sk_state_load(sk); 530 if (state == TCP_LISTEN) 531 return inet_csk_listen_poll(sk); 532 533 /* Socket is not locked. We are protected from async events 534 * by poll logic and correct handling of state changes 535 * made by other threads is impossible in any case. 536 */ 537 538 mask = 0; 539 540 /* 541 * EPOLLHUP is certainly not done right. But poll() doesn't 542 * have a notion of HUP in just one direction, and for a 543 * socket the read side is more interesting. 544 * 545 * Some poll() documentation says that EPOLLHUP is incompatible 546 * with the EPOLLOUT/POLLWR flags, so somebody should check this 547 * all. But careful, it tends to be safer to return too many 548 * bits than too few, and you can easily break real applications 549 * if you don't tell them that something has hung up! 550 * 551 * Check-me. 552 * 553 * Check number 1. EPOLLHUP is _UNMASKABLE_ event (see UNIX98 and 554 * our fs/select.c). It means that after we received EOF, 555 * poll always returns immediately, making impossible poll() on write() 556 * in state CLOSE_WAIT. One solution is evident --- to set EPOLLHUP 557 * if and only if shutdown has been made in both directions. 558 * Actually, it is interesting to look how Solaris and DUX 559 * solve this dilemma. I would prefer, if EPOLLHUP were maskable, 560 * then we could set it on SND_SHUTDOWN. BTW examples given 561 * in Stevens' books assume exactly this behaviour, it explains 562 * why EPOLLHUP is incompatible with EPOLLOUT. --ANK 563 * 564 * NOTE. Check for TCP_CLOSE is added. The goal is to prevent 565 * blocking on fresh not-connected or disconnected socket. --ANK 566 */ 567 shutdown = READ_ONCE(sk->sk_shutdown); 568 if (shutdown == SHUTDOWN_MASK || state == TCP_CLOSE) 569 mask |= EPOLLHUP; 570 if (shutdown & RCV_SHUTDOWN) 571 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 572 573 /* Connected or passive Fast Open socket? */ 574 if (state != TCP_SYN_SENT && 575 (state != TCP_SYN_RECV || rcu_access_pointer(tp->fastopen_rsk))) { 576 int target = sock_rcvlowat(sk, 0, INT_MAX); 577 u16 urg_data = READ_ONCE(tp->urg_data); 578 579 if (unlikely(urg_data) && 580 READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq) && 581 !sock_flag(sk, SOCK_URGINLINE)) 582 target++; 583 584 if (tcp_stream_is_readable(sk, target)) 585 mask |= EPOLLIN | EPOLLRDNORM; 586 587 if (!(shutdown & SEND_SHUTDOWN)) { 588 if (__sk_stream_is_writeable(sk, 1)) { 589 mask |= EPOLLOUT | EPOLLWRNORM; 590 } else { /* send SIGIO later */ 591 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 592 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 593 594 /* Race breaker. If space is freed after 595 * wspace test but before the flags are set, 596 * IO signal will be lost. Memory barrier 597 * pairs with the input side. 598 */ 599 smp_mb__after_atomic(); 600 if (__sk_stream_is_writeable(sk, 1)) 601 mask |= EPOLLOUT | EPOLLWRNORM; 602 } 603 } else 604 mask |= EPOLLOUT | EPOLLWRNORM; 605 606 if (urg_data & TCP_URG_VALID) 607 mask |= EPOLLPRI; 608 } else if (state == TCP_SYN_SENT && 609 inet_test_bit(DEFER_CONNECT, sk)) { 610 /* Active TCP fastopen socket with defer_connect 611 * Return EPOLLOUT so application can call write() 612 * in order for kernel to generate SYN+data 613 */ 614 mask |= EPOLLOUT | EPOLLWRNORM; 615 } 616 /* This barrier is coupled with smp_wmb() in tcp_done_with_error() */ 617 smp_rmb(); 618 if (READ_ONCE(sk->sk_err) || 619 !skb_queue_empty_lockless(&sk->sk_error_queue)) 620 mask |= EPOLLERR; 621 622 return mask; 623 } 624 EXPORT_SYMBOL(tcp_poll); 625 626 int tcp_ioctl(struct sock *sk, int cmd, int *karg) 627 { 628 struct tcp_sock *tp = tcp_sk(sk); 629 int answ; 630 bool slow; 631 632 switch (cmd) { 633 case SIOCINQ: 634 if (sk->sk_state == TCP_LISTEN) 635 return -EINVAL; 636 637 slow = lock_sock_fast(sk); 638 answ = tcp_inq(sk); 639 unlock_sock_fast(sk, slow); 640 break; 641 case SIOCATMARK: 642 answ = READ_ONCE(tp->urg_data) && 643 READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq); 644 break; 645 case SIOCOUTQ: 646 if (sk->sk_state == TCP_LISTEN) 647 return -EINVAL; 648 649 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) 650 answ = 0; 651 else 652 answ = READ_ONCE(tp->write_seq) - tp->snd_una; 653 break; 654 case SIOCOUTQNSD: 655 if (sk->sk_state == TCP_LISTEN) 656 return -EINVAL; 657 658 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) 659 answ = 0; 660 else 661 answ = READ_ONCE(tp->write_seq) - 662 READ_ONCE(tp->snd_nxt); 663 break; 664 default: 665 return -ENOIOCTLCMD; 666 } 667 668 *karg = answ; 669 return 0; 670 } 671 EXPORT_IPV6_MOD(tcp_ioctl); 672 673 void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb) 674 { 675 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH; 676 tp->pushed_seq = tp->write_seq; 677 } 678 679 static inline bool forced_push(const struct tcp_sock *tp) 680 { 681 return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1)); 682 } 683 684 void tcp_skb_entail(struct sock *sk, struct sk_buff *skb) 685 { 686 struct tcp_sock *tp = tcp_sk(sk); 687 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb); 688 689 tcb->seq = tcb->end_seq = tp->write_seq; 690 tcb->tcp_flags = TCPHDR_ACK; 691 __skb_header_release(skb); 692 tcp_add_write_queue_tail(sk, skb); 693 sk_wmem_queued_add(sk, skb->truesize); 694 sk_mem_charge(sk, skb->truesize); 695 if (tp->nonagle & TCP_NAGLE_PUSH) 696 tp->nonagle &= ~TCP_NAGLE_PUSH; 697 698 tcp_slow_start_after_idle_check(sk); 699 } 700 701 static inline void tcp_mark_urg(struct tcp_sock *tp, int flags) 702 { 703 if (flags & MSG_OOB) 704 tp->snd_up = tp->write_seq; 705 } 706 707 /* If a not yet filled skb is pushed, do not send it if 708 * we have data packets in Qdisc or NIC queues : 709 * Because TX completion will happen shortly, it gives a chance 710 * to coalesce future sendmsg() payload into this skb, without 711 * need for a timer, and with no latency trade off. 712 * As packets containing data payload have a bigger truesize 713 * than pure acks (dataless) packets, the last checks prevent 714 * autocorking if we only have an ACK in Qdisc/NIC queues, 715 * or if TX completion was delayed after we processed ACK packet. 716 */ 717 static bool tcp_should_autocork(struct sock *sk, struct sk_buff *skb, 718 int size_goal) 719 { 720 return skb->len < size_goal && 721 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_autocorking) && 722 !tcp_rtx_queue_empty(sk) && 723 refcount_read(&sk->sk_wmem_alloc) > skb->truesize && 724 tcp_skb_can_collapse_to(skb); 725 } 726 727 void tcp_push(struct sock *sk, int flags, int mss_now, 728 int nonagle, int size_goal) 729 { 730 struct tcp_sock *tp = tcp_sk(sk); 731 struct sk_buff *skb; 732 733 skb = tcp_write_queue_tail(sk); 734 if (!skb) 735 return; 736 if (!(flags & MSG_MORE) || forced_push(tp)) 737 tcp_mark_push(tp, skb); 738 739 tcp_mark_urg(tp, flags); 740 741 if (tcp_should_autocork(sk, skb, size_goal)) { 742 743 /* avoid atomic op if TSQ_THROTTLED bit is already set */ 744 if (!test_bit(TSQ_THROTTLED, &sk->sk_tsq_flags)) { 745 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING); 746 set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags); 747 smp_mb__after_atomic(); 748 } 749 /* It is possible TX completion already happened 750 * before we set TSQ_THROTTLED. 751 */ 752 if (refcount_read(&sk->sk_wmem_alloc) > skb->truesize) 753 return; 754 } 755 756 if (flags & MSG_MORE) 757 nonagle = TCP_NAGLE_CORK; 758 759 __tcp_push_pending_frames(sk, mss_now, nonagle); 760 } 761 762 static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb, 763 unsigned int offset, size_t len) 764 { 765 struct tcp_splice_state *tss = rd_desc->arg.data; 766 int ret; 767 768 ret = skb_splice_bits(skb, skb->sk, offset, tss->pipe, 769 min(rd_desc->count, len), tss->flags); 770 if (ret > 0) 771 rd_desc->count -= ret; 772 return ret; 773 } 774 775 static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss) 776 { 777 /* Store TCP splice context information in read_descriptor_t. */ 778 read_descriptor_t rd_desc = { 779 .arg.data = tss, 780 .count = tss->len, 781 }; 782 783 return tcp_read_sock(sk, &rd_desc, tcp_splice_data_recv); 784 } 785 786 /** 787 * tcp_splice_read - splice data from TCP socket to a pipe 788 * @sock: socket to splice from 789 * @ppos: position (not valid) 790 * @pipe: pipe to splice to 791 * @len: number of bytes to splice 792 * @flags: splice modifier flags 793 * 794 * Description: 795 * Will read pages from given socket and fill them into a pipe. 796 * 797 **/ 798 ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos, 799 struct pipe_inode_info *pipe, size_t len, 800 unsigned int flags) 801 { 802 struct sock *sk = sock->sk; 803 struct tcp_splice_state tss = { 804 .pipe = pipe, 805 .len = len, 806 .flags = flags, 807 }; 808 long timeo; 809 ssize_t spliced; 810 int ret; 811 812 sock_rps_record_flow(sk); 813 /* 814 * We can't seek on a socket input 815 */ 816 if (unlikely(*ppos)) 817 return -ESPIPE; 818 819 ret = spliced = 0; 820 821 lock_sock(sk); 822 823 timeo = sock_rcvtimeo(sk, sock->file->f_flags & O_NONBLOCK); 824 while (tss.len) { 825 ret = __tcp_splice_read(sk, &tss); 826 if (ret < 0) 827 break; 828 else if (!ret) { 829 if (spliced) 830 break; 831 if (sock_flag(sk, SOCK_DONE)) 832 break; 833 if (sk->sk_err) { 834 ret = sock_error(sk); 835 break; 836 } 837 if (sk->sk_shutdown & RCV_SHUTDOWN) 838 break; 839 if (sk->sk_state == TCP_CLOSE) { 840 /* 841 * This occurs when user tries to read 842 * from never connected socket. 843 */ 844 ret = -ENOTCONN; 845 break; 846 } 847 if (!timeo) { 848 ret = -EAGAIN; 849 break; 850 } 851 /* if __tcp_splice_read() got nothing while we have 852 * an skb in receive queue, we do not want to loop. 853 * This might happen with URG data. 854 */ 855 if (!skb_queue_empty(&sk->sk_receive_queue)) 856 break; 857 ret = sk_wait_data(sk, &timeo, NULL); 858 if (ret < 0) 859 break; 860 if (signal_pending(current)) { 861 ret = sock_intr_errno(timeo); 862 break; 863 } 864 continue; 865 } 866 tss.len -= ret; 867 spliced += ret; 868 869 if (!tss.len || !timeo) 870 break; 871 release_sock(sk); 872 lock_sock(sk); 873 874 if (sk->sk_err || sk->sk_state == TCP_CLOSE || 875 (sk->sk_shutdown & RCV_SHUTDOWN) || 876 signal_pending(current)) 877 break; 878 } 879 880 release_sock(sk); 881 882 if (spliced) 883 return spliced; 884 885 return ret; 886 } 887 EXPORT_IPV6_MOD(tcp_splice_read); 888 889 struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp, 890 bool force_schedule) 891 { 892 struct sk_buff *skb; 893 894 skb = alloc_skb_fclone(MAX_TCP_HEADER, gfp); 895 if (likely(skb)) { 896 bool mem_scheduled; 897 898 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 899 if (force_schedule) { 900 mem_scheduled = true; 901 sk_forced_mem_schedule(sk, skb->truesize); 902 } else { 903 mem_scheduled = sk_wmem_schedule(sk, skb->truesize); 904 } 905 if (likely(mem_scheduled)) { 906 skb_reserve(skb, MAX_TCP_HEADER); 907 skb->ip_summed = CHECKSUM_PARTIAL; 908 INIT_LIST_HEAD(&skb->tcp_tsorted_anchor); 909 return skb; 910 } 911 __kfree_skb(skb); 912 } else { 913 sk->sk_prot->enter_memory_pressure(sk); 914 sk_stream_moderate_sndbuf(sk); 915 } 916 return NULL; 917 } 918 919 static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now, 920 int large_allowed) 921 { 922 struct tcp_sock *tp = tcp_sk(sk); 923 u32 new_size_goal, size_goal; 924 925 if (!large_allowed) 926 return mss_now; 927 928 /* Note : tcp_tso_autosize() will eventually split this later */ 929 new_size_goal = tcp_bound_to_half_wnd(tp, sk->sk_gso_max_size); 930 931 /* We try hard to avoid divides here */ 932 size_goal = tp->gso_segs * mss_now; 933 if (unlikely(new_size_goal < size_goal || 934 new_size_goal >= size_goal + mss_now)) { 935 tp->gso_segs = min_t(u16, new_size_goal / mss_now, 936 sk->sk_gso_max_segs); 937 size_goal = tp->gso_segs * mss_now; 938 } 939 940 return max(size_goal, mss_now); 941 } 942 943 int tcp_send_mss(struct sock *sk, int *size_goal, int flags) 944 { 945 int mss_now; 946 947 mss_now = tcp_current_mss(sk); 948 *size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB)); 949 950 return mss_now; 951 } 952 953 /* In some cases, sendmsg() could have added an skb to the write queue, 954 * but failed adding payload on it. We need to remove it to consume less 955 * memory, but more importantly be able to generate EPOLLOUT for Edge Trigger 956 * epoll() users. Another reason is that tcp_write_xmit() does not like 957 * finding an empty skb in the write queue. 958 */ 959 void tcp_remove_empty_skb(struct sock *sk) 960 { 961 struct sk_buff *skb = tcp_write_queue_tail(sk); 962 963 if (skb && TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) { 964 tcp_unlink_write_queue(skb, sk); 965 if (tcp_write_queue_empty(sk)) 966 tcp_chrono_stop(sk, TCP_CHRONO_BUSY); 967 tcp_wmem_free_skb(sk, skb); 968 } 969 } 970 971 /* skb changing from pure zc to mixed, must charge zc */ 972 static int tcp_downgrade_zcopy_pure(struct sock *sk, struct sk_buff *skb) 973 { 974 if (unlikely(skb_zcopy_pure(skb))) { 975 u32 extra = skb->truesize - 976 SKB_TRUESIZE(skb_end_offset(skb)); 977 978 if (!sk_wmem_schedule(sk, extra)) 979 return -ENOMEM; 980 981 sk_mem_charge(sk, extra); 982 skb_shinfo(skb)->flags &= ~SKBFL_PURE_ZEROCOPY; 983 } 984 return 0; 985 } 986 987 988 int tcp_wmem_schedule(struct sock *sk, int copy) 989 { 990 int left; 991 992 if (likely(sk_wmem_schedule(sk, copy))) 993 return copy; 994 995 /* We could be in trouble if we have nothing queued. 996 * Use whatever is left in sk->sk_forward_alloc and tcp_wmem[0] 997 * to guarantee some progress. 998 */ 999 left = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_wmem[0]) - sk->sk_wmem_queued; 1000 if (left > 0) 1001 sk_forced_mem_schedule(sk, min(left, copy)); 1002 return min(copy, sk->sk_forward_alloc); 1003 } 1004 1005 void tcp_free_fastopen_req(struct tcp_sock *tp) 1006 { 1007 if (tp->fastopen_req) { 1008 kfree(tp->fastopen_req); 1009 tp->fastopen_req = NULL; 1010 } 1011 } 1012 1013 int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied, 1014 size_t size, struct ubuf_info *uarg) 1015 { 1016 struct tcp_sock *tp = tcp_sk(sk); 1017 struct inet_sock *inet = inet_sk(sk); 1018 struct sockaddr *uaddr = msg->msg_name; 1019 int err, flags; 1020 1021 if (!(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen) & 1022 TFO_CLIENT_ENABLE) || 1023 (uaddr && msg->msg_namelen >= sizeof(uaddr->sa_family) && 1024 uaddr->sa_family == AF_UNSPEC)) 1025 return -EOPNOTSUPP; 1026 if (tp->fastopen_req) 1027 return -EALREADY; /* Another Fast Open is in progress */ 1028 1029 tp->fastopen_req = kzalloc(sizeof(struct tcp_fastopen_request), 1030 sk->sk_allocation); 1031 if (unlikely(!tp->fastopen_req)) 1032 return -ENOBUFS; 1033 tp->fastopen_req->data = msg; 1034 tp->fastopen_req->size = size; 1035 tp->fastopen_req->uarg = uarg; 1036 1037 if (inet_test_bit(DEFER_CONNECT, sk)) { 1038 err = tcp_connect(sk); 1039 /* Same failure procedure as in tcp_v4/6_connect */ 1040 if (err) { 1041 tcp_set_state(sk, TCP_CLOSE); 1042 inet->inet_dport = 0; 1043 sk->sk_route_caps = 0; 1044 } 1045 } 1046 flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0; 1047 err = __inet_stream_connect(sk->sk_socket, uaddr, 1048 msg->msg_namelen, flags, 1); 1049 /* fastopen_req could already be freed in __inet_stream_connect 1050 * if the connection times out or gets rst 1051 */ 1052 if (tp->fastopen_req) { 1053 *copied = tp->fastopen_req->copied; 1054 tcp_free_fastopen_req(tp); 1055 inet_clear_bit(DEFER_CONNECT, sk); 1056 } 1057 return err; 1058 } 1059 1060 int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) 1061 { 1062 struct tcp_sock *tp = tcp_sk(sk); 1063 struct ubuf_info *uarg = NULL; 1064 struct sk_buff *skb; 1065 struct sockcm_cookie sockc; 1066 int flags, err, copied = 0; 1067 int mss_now = 0, size_goal, copied_syn = 0; 1068 int process_backlog = 0; 1069 int zc = 0; 1070 long timeo; 1071 1072 flags = msg->msg_flags; 1073 1074 if ((flags & MSG_ZEROCOPY) && size) { 1075 if (msg->msg_ubuf) { 1076 uarg = msg->msg_ubuf; 1077 if (sk->sk_route_caps & NETIF_F_SG) 1078 zc = MSG_ZEROCOPY; 1079 } else if (sock_flag(sk, SOCK_ZEROCOPY)) { 1080 skb = tcp_write_queue_tail(sk); 1081 uarg = msg_zerocopy_realloc(sk, size, skb_zcopy(skb)); 1082 if (!uarg) { 1083 err = -ENOBUFS; 1084 goto out_err; 1085 } 1086 if (sk->sk_route_caps & NETIF_F_SG) 1087 zc = MSG_ZEROCOPY; 1088 else 1089 uarg_to_msgzc(uarg)->zerocopy = 0; 1090 } 1091 } else if (unlikely(msg->msg_flags & MSG_SPLICE_PAGES) && size) { 1092 if (sk->sk_route_caps & NETIF_F_SG) 1093 zc = MSG_SPLICE_PAGES; 1094 } 1095 1096 if (unlikely(flags & MSG_FASTOPEN || 1097 inet_test_bit(DEFER_CONNECT, sk)) && 1098 !tp->repair) { 1099 err = tcp_sendmsg_fastopen(sk, msg, &copied_syn, size, uarg); 1100 if (err == -EINPROGRESS && copied_syn > 0) 1101 goto out; 1102 else if (err) 1103 goto out_err; 1104 } 1105 1106 timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); 1107 1108 tcp_rate_check_app_limited(sk); /* is sending application-limited? */ 1109 1110 /* Wait for a connection to finish. One exception is TCP Fast Open 1111 * (passive side) where data is allowed to be sent before a connection 1112 * is fully established. 1113 */ 1114 if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) && 1115 !tcp_passive_fastopen(sk)) { 1116 err = sk_stream_wait_connect(sk, &timeo); 1117 if (err != 0) 1118 goto do_error; 1119 } 1120 1121 if (unlikely(tp->repair)) { 1122 if (tp->repair_queue == TCP_RECV_QUEUE) { 1123 copied = tcp_send_rcvq(sk, msg, size); 1124 goto out_nopush; 1125 } 1126 1127 err = -EINVAL; 1128 if (tp->repair_queue == TCP_NO_QUEUE) 1129 goto out_err; 1130 1131 /* 'common' sending to sendq */ 1132 } 1133 1134 sockc = (struct sockcm_cookie) { .tsflags = READ_ONCE(sk->sk_tsflags)}; 1135 if (msg->msg_controllen) { 1136 err = sock_cmsg_send(sk, msg, &sockc); 1137 if (unlikely(err)) { 1138 err = -EINVAL; 1139 goto out_err; 1140 } 1141 } 1142 1143 /* This should be in poll */ 1144 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 1145 1146 /* Ok commence sending. */ 1147 copied = 0; 1148 1149 restart: 1150 mss_now = tcp_send_mss(sk, &size_goal, flags); 1151 1152 err = -EPIPE; 1153 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) 1154 goto do_error; 1155 1156 while (msg_data_left(msg)) { 1157 ssize_t copy = 0; 1158 1159 skb = tcp_write_queue_tail(sk); 1160 if (skb) 1161 copy = size_goal - skb->len; 1162 1163 if (copy <= 0 || !tcp_skb_can_collapse_to(skb)) { 1164 bool first_skb; 1165 1166 new_segment: 1167 if (!sk_stream_memory_free(sk)) 1168 goto wait_for_space; 1169 1170 if (unlikely(process_backlog >= 16)) { 1171 process_backlog = 0; 1172 if (sk_flush_backlog(sk)) 1173 goto restart; 1174 } 1175 first_skb = tcp_rtx_and_write_queues_empty(sk); 1176 skb = tcp_stream_alloc_skb(sk, sk->sk_allocation, 1177 first_skb); 1178 if (!skb) 1179 goto wait_for_space; 1180 1181 process_backlog++; 1182 1183 #ifdef CONFIG_SKB_DECRYPTED 1184 skb->decrypted = !!(flags & MSG_SENDPAGE_DECRYPTED); 1185 #endif 1186 tcp_skb_entail(sk, skb); 1187 copy = size_goal; 1188 1189 /* All packets are restored as if they have 1190 * already been sent. skb_mstamp_ns isn't set to 1191 * avoid wrong rtt estimation. 1192 */ 1193 if (tp->repair) 1194 TCP_SKB_CB(skb)->sacked |= TCPCB_REPAIRED; 1195 } 1196 1197 /* Try to append data to the end of skb. */ 1198 if (copy > msg_data_left(msg)) 1199 copy = msg_data_left(msg); 1200 1201 if (zc == 0) { 1202 bool merge = true; 1203 int i = skb_shinfo(skb)->nr_frags; 1204 struct page_frag *pfrag = sk_page_frag(sk); 1205 1206 if (!sk_page_frag_refill(sk, pfrag)) 1207 goto wait_for_space; 1208 1209 if (!skb_can_coalesce(skb, i, pfrag->page, 1210 pfrag->offset)) { 1211 if (i >= READ_ONCE(net_hotdata.sysctl_max_skb_frags)) { 1212 tcp_mark_push(tp, skb); 1213 goto new_segment; 1214 } 1215 merge = false; 1216 } 1217 1218 copy = min_t(int, copy, pfrag->size - pfrag->offset); 1219 1220 if (unlikely(skb_zcopy_pure(skb) || skb_zcopy_managed(skb))) { 1221 if (tcp_downgrade_zcopy_pure(sk, skb)) 1222 goto wait_for_space; 1223 skb_zcopy_downgrade_managed(skb); 1224 } 1225 1226 copy = tcp_wmem_schedule(sk, copy); 1227 if (!copy) 1228 goto wait_for_space; 1229 1230 err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb, 1231 pfrag->page, 1232 pfrag->offset, 1233 copy); 1234 if (err) 1235 goto do_error; 1236 1237 /* Update the skb. */ 1238 if (merge) { 1239 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 1240 } else { 1241 skb_fill_page_desc(skb, i, pfrag->page, 1242 pfrag->offset, copy); 1243 page_ref_inc(pfrag->page); 1244 } 1245 pfrag->offset += copy; 1246 } else if (zc == MSG_ZEROCOPY) { 1247 /* First append to a fragless skb builds initial 1248 * pure zerocopy skb 1249 */ 1250 if (!skb->len) 1251 skb_shinfo(skb)->flags |= SKBFL_PURE_ZEROCOPY; 1252 1253 if (!skb_zcopy_pure(skb)) { 1254 copy = tcp_wmem_schedule(sk, copy); 1255 if (!copy) 1256 goto wait_for_space; 1257 } 1258 1259 err = skb_zerocopy_iter_stream(sk, skb, msg, copy, uarg); 1260 if (err == -EMSGSIZE || err == -EEXIST) { 1261 tcp_mark_push(tp, skb); 1262 goto new_segment; 1263 } 1264 if (err < 0) 1265 goto do_error; 1266 copy = err; 1267 } else if (zc == MSG_SPLICE_PAGES) { 1268 /* Splice in data if we can; copy if we can't. */ 1269 if (tcp_downgrade_zcopy_pure(sk, skb)) 1270 goto wait_for_space; 1271 copy = tcp_wmem_schedule(sk, copy); 1272 if (!copy) 1273 goto wait_for_space; 1274 1275 err = skb_splice_from_iter(skb, &msg->msg_iter, copy, 1276 sk->sk_allocation); 1277 if (err < 0) { 1278 if (err == -EMSGSIZE) { 1279 tcp_mark_push(tp, skb); 1280 goto new_segment; 1281 } 1282 goto do_error; 1283 } 1284 copy = err; 1285 1286 if (!(flags & MSG_NO_SHARED_FRAGS)) 1287 skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG; 1288 1289 sk_wmem_queued_add(sk, copy); 1290 sk_mem_charge(sk, copy); 1291 } 1292 1293 if (!copied) 1294 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH; 1295 1296 WRITE_ONCE(tp->write_seq, tp->write_seq + copy); 1297 TCP_SKB_CB(skb)->end_seq += copy; 1298 tcp_skb_pcount_set(skb, 0); 1299 1300 copied += copy; 1301 if (!msg_data_left(msg)) { 1302 if (unlikely(flags & MSG_EOR)) 1303 TCP_SKB_CB(skb)->eor = 1; 1304 goto out; 1305 } 1306 1307 if (skb->len < size_goal || (flags & MSG_OOB) || unlikely(tp->repair)) 1308 continue; 1309 1310 if (forced_push(tp)) { 1311 tcp_mark_push(tp, skb); 1312 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH); 1313 } else if (skb == tcp_send_head(sk)) 1314 tcp_push_one(sk, mss_now); 1315 continue; 1316 1317 wait_for_space: 1318 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 1319 tcp_remove_empty_skb(sk); 1320 if (copied) 1321 tcp_push(sk, flags & ~MSG_MORE, mss_now, 1322 TCP_NAGLE_PUSH, size_goal); 1323 1324 err = sk_stream_wait_memory(sk, &timeo); 1325 if (err != 0) 1326 goto do_error; 1327 1328 mss_now = tcp_send_mss(sk, &size_goal, flags); 1329 } 1330 1331 out: 1332 if (copied) { 1333 tcp_tx_timestamp(sk, &sockc); 1334 tcp_push(sk, flags, mss_now, tp->nonagle, size_goal); 1335 } 1336 out_nopush: 1337 /* msg->msg_ubuf is pinned by the caller so we don't take extra refs */ 1338 if (uarg && !msg->msg_ubuf) 1339 net_zcopy_put(uarg); 1340 return copied + copied_syn; 1341 1342 do_error: 1343 tcp_remove_empty_skb(sk); 1344 1345 if (copied + copied_syn) 1346 goto out; 1347 out_err: 1348 /* msg->msg_ubuf is pinned by the caller so we don't take extra refs */ 1349 if (uarg && !msg->msg_ubuf) 1350 net_zcopy_put_abort(uarg, true); 1351 err = sk_stream_error(sk, flags, err); 1352 /* make sure we wake any epoll edge trigger waiter */ 1353 if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) { 1354 sk->sk_write_space(sk); 1355 tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED); 1356 } 1357 return err; 1358 } 1359 EXPORT_SYMBOL_GPL(tcp_sendmsg_locked); 1360 1361 int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) 1362 { 1363 int ret; 1364 1365 lock_sock(sk); 1366 ret = tcp_sendmsg_locked(sk, msg, size); 1367 release_sock(sk); 1368 1369 return ret; 1370 } 1371 EXPORT_SYMBOL(tcp_sendmsg); 1372 1373 void tcp_splice_eof(struct socket *sock) 1374 { 1375 struct sock *sk = sock->sk; 1376 struct tcp_sock *tp = tcp_sk(sk); 1377 int mss_now, size_goal; 1378 1379 if (!tcp_write_queue_tail(sk)) 1380 return; 1381 1382 lock_sock(sk); 1383 mss_now = tcp_send_mss(sk, &size_goal, 0); 1384 tcp_push(sk, 0, mss_now, tp->nonagle, size_goal); 1385 release_sock(sk); 1386 } 1387 EXPORT_IPV6_MOD_GPL(tcp_splice_eof); 1388 1389 /* 1390 * Handle reading urgent data. BSD has very simple semantics for 1391 * this, no blocking and very strange errors 8) 1392 */ 1393 1394 static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags) 1395 { 1396 struct tcp_sock *tp = tcp_sk(sk); 1397 1398 /* No URG data to read. */ 1399 if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data || 1400 tp->urg_data == TCP_URG_READ) 1401 return -EINVAL; /* Yes this is right ! */ 1402 1403 if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE)) 1404 return -ENOTCONN; 1405 1406 if (tp->urg_data & TCP_URG_VALID) { 1407 int err = 0; 1408 char c = tp->urg_data; 1409 1410 if (!(flags & MSG_PEEK)) 1411 WRITE_ONCE(tp->urg_data, TCP_URG_READ); 1412 1413 /* Read urgent data. */ 1414 msg->msg_flags |= MSG_OOB; 1415 1416 if (len > 0) { 1417 if (!(flags & MSG_TRUNC)) 1418 err = memcpy_to_msg(msg, &c, 1); 1419 len = 1; 1420 } else 1421 msg->msg_flags |= MSG_TRUNC; 1422 1423 return err ? -EFAULT : len; 1424 } 1425 1426 if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN)) 1427 return 0; 1428 1429 /* Fixed the recv(..., MSG_OOB) behaviour. BSD docs and 1430 * the available implementations agree in this case: 1431 * this call should never block, independent of the 1432 * blocking state of the socket. 1433 * Mike <pall@rz.uni-karlsruhe.de> 1434 */ 1435 return -EAGAIN; 1436 } 1437 1438 static int tcp_peek_sndq(struct sock *sk, struct msghdr *msg, int len) 1439 { 1440 struct sk_buff *skb; 1441 int copied = 0, err = 0; 1442 1443 skb_rbtree_walk(skb, &sk->tcp_rtx_queue) { 1444 err = skb_copy_datagram_msg(skb, 0, msg, skb->len); 1445 if (err) 1446 return err; 1447 copied += skb->len; 1448 } 1449 1450 skb_queue_walk(&sk->sk_write_queue, skb) { 1451 err = skb_copy_datagram_msg(skb, 0, msg, skb->len); 1452 if (err) 1453 break; 1454 1455 copied += skb->len; 1456 } 1457 1458 return err ?: copied; 1459 } 1460 1461 /* Clean up the receive buffer for full frames taken by the user, 1462 * then send an ACK if necessary. COPIED is the number of bytes 1463 * tcp_recvmsg has given to the user so far, it speeds up the 1464 * calculation of whether or not we must ACK for the sake of 1465 * a window update. 1466 */ 1467 void __tcp_cleanup_rbuf(struct sock *sk, int copied) 1468 { 1469 struct tcp_sock *tp = tcp_sk(sk); 1470 bool time_to_ack = false; 1471 1472 if (inet_csk_ack_scheduled(sk)) { 1473 const struct inet_connection_sock *icsk = inet_csk(sk); 1474 1475 if (/* Once-per-two-segments ACK was not sent by tcp_input.c */ 1476 tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss || 1477 /* 1478 * If this read emptied read buffer, we send ACK, if 1479 * connection is not bidirectional, user drained 1480 * receive buffer and there was a small segment 1481 * in queue. 1482 */ 1483 (copied > 0 && 1484 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) || 1485 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) && 1486 !inet_csk_in_pingpong_mode(sk))) && 1487 !atomic_read(&sk->sk_rmem_alloc))) 1488 time_to_ack = true; 1489 } 1490 1491 /* We send an ACK if we can now advertise a non-zero window 1492 * which has been raised "significantly". 1493 * 1494 * Even if window raised up to infinity, do not send window open ACK 1495 * in states, where we will not receive more. It is useless. 1496 */ 1497 if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) { 1498 __u32 rcv_window_now = tcp_receive_window(tp); 1499 1500 /* Optimize, __tcp_select_window() is not cheap. */ 1501 if (2*rcv_window_now <= tp->window_clamp) { 1502 __u32 new_window = __tcp_select_window(sk); 1503 1504 /* Send ACK now, if this read freed lots of space 1505 * in our buffer. Certainly, new_window is new window. 1506 * We can advertise it now, if it is not less than current one. 1507 * "Lots" means "at least twice" here. 1508 */ 1509 if (new_window && new_window >= 2 * rcv_window_now) 1510 time_to_ack = true; 1511 } 1512 } 1513 if (time_to_ack) 1514 tcp_send_ack(sk); 1515 } 1516 1517 void tcp_cleanup_rbuf(struct sock *sk, int copied) 1518 { 1519 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue); 1520 struct tcp_sock *tp = tcp_sk(sk); 1521 1522 WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq), 1523 "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n", 1524 tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt); 1525 __tcp_cleanup_rbuf(sk, copied); 1526 } 1527 1528 static void tcp_eat_recv_skb(struct sock *sk, struct sk_buff *skb) 1529 { 1530 __skb_unlink(skb, &sk->sk_receive_queue); 1531 if (likely(skb->destructor == sock_rfree)) { 1532 sock_rfree(skb); 1533 skb->destructor = NULL; 1534 skb->sk = NULL; 1535 return skb_attempt_defer_free(skb); 1536 } 1537 __kfree_skb(skb); 1538 } 1539 1540 struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off) 1541 { 1542 struct sk_buff *skb; 1543 u32 offset; 1544 1545 while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) { 1546 offset = seq - TCP_SKB_CB(skb)->seq; 1547 if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) { 1548 pr_err_once("%s: found a SYN, please report !\n", __func__); 1549 offset--; 1550 } 1551 if (offset < skb->len || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) { 1552 *off = offset; 1553 return skb; 1554 } 1555 /* This looks weird, but this can happen if TCP collapsing 1556 * splitted a fat GRO packet, while we released socket lock 1557 * in skb_splice_bits() 1558 */ 1559 tcp_eat_recv_skb(sk, skb); 1560 } 1561 return NULL; 1562 } 1563 EXPORT_SYMBOL(tcp_recv_skb); 1564 1565 /* 1566 * This routine provides an alternative to tcp_recvmsg() for routines 1567 * that would like to handle copying from skbuffs directly in 'sendfile' 1568 * fashion. 1569 * Note: 1570 * - It is assumed that the socket was locked by the caller. 1571 * - The routine does not block. 1572 * - At present, there is no support for reading OOB data 1573 * or for 'peeking' the socket using this routine 1574 * (although both would be easy to implement). 1575 */ 1576 static int __tcp_read_sock(struct sock *sk, read_descriptor_t *desc, 1577 sk_read_actor_t recv_actor, bool noack, 1578 u32 *copied_seq) 1579 { 1580 struct sk_buff *skb; 1581 struct tcp_sock *tp = tcp_sk(sk); 1582 u32 seq = *copied_seq; 1583 u32 offset; 1584 int copied = 0; 1585 1586 if (sk->sk_state == TCP_LISTEN) 1587 return -ENOTCONN; 1588 while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) { 1589 if (offset < skb->len) { 1590 int used; 1591 size_t len; 1592 1593 len = skb->len - offset; 1594 /* Stop reading if we hit a patch of urgent data */ 1595 if (unlikely(tp->urg_data)) { 1596 u32 urg_offset = tp->urg_seq - seq; 1597 if (urg_offset < len) 1598 len = urg_offset; 1599 if (!len) 1600 break; 1601 } 1602 used = recv_actor(desc, skb, offset, len); 1603 if (used <= 0) { 1604 if (!copied) 1605 copied = used; 1606 break; 1607 } 1608 if (WARN_ON_ONCE(used > len)) 1609 used = len; 1610 seq += used; 1611 copied += used; 1612 offset += used; 1613 1614 /* If recv_actor drops the lock (e.g. TCP splice 1615 * receive) the skb pointer might be invalid when 1616 * getting here: tcp_collapse might have deleted it 1617 * while aggregating skbs from the socket queue. 1618 */ 1619 skb = tcp_recv_skb(sk, seq - 1, &offset); 1620 if (!skb) 1621 break; 1622 /* TCP coalescing might have appended data to the skb. 1623 * Try to splice more frags 1624 */ 1625 if (offset + 1 != skb->len) 1626 continue; 1627 } 1628 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) { 1629 tcp_eat_recv_skb(sk, skb); 1630 ++seq; 1631 break; 1632 } 1633 tcp_eat_recv_skb(sk, skb); 1634 if (!desc->count) 1635 break; 1636 WRITE_ONCE(*copied_seq, seq); 1637 } 1638 WRITE_ONCE(*copied_seq, seq); 1639 1640 if (noack) 1641 goto out; 1642 1643 tcp_rcv_space_adjust(sk); 1644 1645 /* Clean up data we have read: This will do ACK frames. */ 1646 if (copied > 0) { 1647 tcp_recv_skb(sk, seq, &offset); 1648 tcp_cleanup_rbuf(sk, copied); 1649 } 1650 out: 1651 return copied; 1652 } 1653 1654 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, 1655 sk_read_actor_t recv_actor) 1656 { 1657 return __tcp_read_sock(sk, desc, recv_actor, false, 1658 &tcp_sk(sk)->copied_seq); 1659 } 1660 EXPORT_SYMBOL(tcp_read_sock); 1661 1662 int tcp_read_sock_noack(struct sock *sk, read_descriptor_t *desc, 1663 sk_read_actor_t recv_actor, bool noack, 1664 u32 *copied_seq) 1665 { 1666 return __tcp_read_sock(sk, desc, recv_actor, noack, copied_seq); 1667 } 1668 1669 int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor) 1670 { 1671 struct sk_buff *skb; 1672 int copied = 0; 1673 1674 if (sk->sk_state == TCP_LISTEN) 1675 return -ENOTCONN; 1676 1677 while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) { 1678 u8 tcp_flags; 1679 int used; 1680 1681 __skb_unlink(skb, &sk->sk_receive_queue); 1682 WARN_ON_ONCE(!skb_set_owner_sk_safe(skb, sk)); 1683 tcp_flags = TCP_SKB_CB(skb)->tcp_flags; 1684 used = recv_actor(sk, skb); 1685 if (used < 0) { 1686 if (!copied) 1687 copied = used; 1688 break; 1689 } 1690 copied += used; 1691 1692 if (tcp_flags & TCPHDR_FIN) 1693 break; 1694 } 1695 return copied; 1696 } 1697 EXPORT_IPV6_MOD(tcp_read_skb); 1698 1699 void tcp_read_done(struct sock *sk, size_t len) 1700 { 1701 struct tcp_sock *tp = tcp_sk(sk); 1702 u32 seq = tp->copied_seq; 1703 struct sk_buff *skb; 1704 size_t left; 1705 u32 offset; 1706 1707 if (sk->sk_state == TCP_LISTEN) 1708 return; 1709 1710 left = len; 1711 while (left && (skb = tcp_recv_skb(sk, seq, &offset)) != NULL) { 1712 int used; 1713 1714 used = min_t(size_t, skb->len - offset, left); 1715 seq += used; 1716 left -= used; 1717 1718 if (skb->len > offset + used) 1719 break; 1720 1721 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) { 1722 tcp_eat_recv_skb(sk, skb); 1723 ++seq; 1724 break; 1725 } 1726 tcp_eat_recv_skb(sk, skb); 1727 } 1728 WRITE_ONCE(tp->copied_seq, seq); 1729 1730 tcp_rcv_space_adjust(sk); 1731 1732 /* Clean up data we have read: This will do ACK frames. */ 1733 if (left != len) 1734 tcp_cleanup_rbuf(sk, len - left); 1735 } 1736 EXPORT_SYMBOL(tcp_read_done); 1737 1738 int tcp_peek_len(struct socket *sock) 1739 { 1740 return tcp_inq(sock->sk); 1741 } 1742 EXPORT_IPV6_MOD(tcp_peek_len); 1743 1744 /* Make sure sk_rcvbuf is big enough to satisfy SO_RCVLOWAT hint */ 1745 int tcp_set_rcvlowat(struct sock *sk, int val) 1746 { 1747 int space, cap; 1748 1749 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK) 1750 cap = sk->sk_rcvbuf >> 1; 1751 else 1752 cap = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2]) >> 1; 1753 val = min(val, cap); 1754 WRITE_ONCE(sk->sk_rcvlowat, val ? : 1); 1755 1756 /* Check if we need to signal EPOLLIN right now */ 1757 tcp_data_ready(sk); 1758 1759 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK) 1760 return 0; 1761 1762 space = tcp_space_from_win(sk, val); 1763 if (space > sk->sk_rcvbuf) { 1764 WRITE_ONCE(sk->sk_rcvbuf, space); 1765 WRITE_ONCE(tcp_sk(sk)->window_clamp, val); 1766 } 1767 return 0; 1768 } 1769 EXPORT_IPV6_MOD(tcp_set_rcvlowat); 1770 1771 void tcp_update_recv_tstamps(struct sk_buff *skb, 1772 struct scm_timestamping_internal *tss) 1773 { 1774 if (skb->tstamp) 1775 tss->ts[0] = ktime_to_timespec64(skb->tstamp); 1776 else 1777 tss->ts[0] = (struct timespec64) {0}; 1778 1779 if (skb_hwtstamps(skb)->hwtstamp) 1780 tss->ts[2] = ktime_to_timespec64(skb_hwtstamps(skb)->hwtstamp); 1781 else 1782 tss->ts[2] = (struct timespec64) {0}; 1783 } 1784 1785 #ifdef CONFIG_MMU 1786 static const struct vm_operations_struct tcp_vm_ops = { 1787 }; 1788 1789 int tcp_mmap(struct file *file, struct socket *sock, 1790 struct vm_area_struct *vma) 1791 { 1792 if (vma->vm_flags & (VM_WRITE | VM_EXEC)) 1793 return -EPERM; 1794 vm_flags_clear(vma, VM_MAYWRITE | VM_MAYEXEC); 1795 1796 /* Instruct vm_insert_page() to not mmap_read_lock(mm) */ 1797 vm_flags_set(vma, VM_MIXEDMAP); 1798 1799 vma->vm_ops = &tcp_vm_ops; 1800 return 0; 1801 } 1802 EXPORT_IPV6_MOD(tcp_mmap); 1803 1804 static skb_frag_t *skb_advance_to_frag(struct sk_buff *skb, u32 offset_skb, 1805 u32 *offset_frag) 1806 { 1807 skb_frag_t *frag; 1808 1809 if (unlikely(offset_skb >= skb->len)) 1810 return NULL; 1811 1812 offset_skb -= skb_headlen(skb); 1813 if ((int)offset_skb < 0 || skb_has_frag_list(skb)) 1814 return NULL; 1815 1816 frag = skb_shinfo(skb)->frags; 1817 while (offset_skb) { 1818 if (skb_frag_size(frag) > offset_skb) { 1819 *offset_frag = offset_skb; 1820 return frag; 1821 } 1822 offset_skb -= skb_frag_size(frag); 1823 ++frag; 1824 } 1825 *offset_frag = 0; 1826 return frag; 1827 } 1828 1829 static bool can_map_frag(const skb_frag_t *frag) 1830 { 1831 struct page *page; 1832 1833 if (skb_frag_size(frag) != PAGE_SIZE || skb_frag_off(frag)) 1834 return false; 1835 1836 page = skb_frag_page(frag); 1837 1838 if (PageCompound(page) || page->mapping) 1839 return false; 1840 1841 return true; 1842 } 1843 1844 static int find_next_mappable_frag(const skb_frag_t *frag, 1845 int remaining_in_skb) 1846 { 1847 int offset = 0; 1848 1849 if (likely(can_map_frag(frag))) 1850 return 0; 1851 1852 while (offset < remaining_in_skb && !can_map_frag(frag)) { 1853 offset += skb_frag_size(frag); 1854 ++frag; 1855 } 1856 return offset; 1857 } 1858 1859 static void tcp_zerocopy_set_hint_for_skb(struct sock *sk, 1860 struct tcp_zerocopy_receive *zc, 1861 struct sk_buff *skb, u32 offset) 1862 { 1863 u32 frag_offset, partial_frag_remainder = 0; 1864 int mappable_offset; 1865 skb_frag_t *frag; 1866 1867 /* worst case: skip to next skb. try to improve on this case below */ 1868 zc->recv_skip_hint = skb->len - offset; 1869 1870 /* Find the frag containing this offset (and how far into that frag) */ 1871 frag = skb_advance_to_frag(skb, offset, &frag_offset); 1872 if (!frag) 1873 return; 1874 1875 if (frag_offset) { 1876 struct skb_shared_info *info = skb_shinfo(skb); 1877 1878 /* We read part of the last frag, must recvmsg() rest of skb. */ 1879 if (frag == &info->frags[info->nr_frags - 1]) 1880 return; 1881 1882 /* Else, we must at least read the remainder in this frag. */ 1883 partial_frag_remainder = skb_frag_size(frag) - frag_offset; 1884 zc->recv_skip_hint -= partial_frag_remainder; 1885 ++frag; 1886 } 1887 1888 /* partial_frag_remainder: If part way through a frag, must read rest. 1889 * mappable_offset: Bytes till next mappable frag, *not* counting bytes 1890 * in partial_frag_remainder. 1891 */ 1892 mappable_offset = find_next_mappable_frag(frag, zc->recv_skip_hint); 1893 zc->recv_skip_hint = mappable_offset + partial_frag_remainder; 1894 } 1895 1896 static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len, 1897 int flags, struct scm_timestamping_internal *tss, 1898 int *cmsg_flags); 1899 static int receive_fallback_to_copy(struct sock *sk, 1900 struct tcp_zerocopy_receive *zc, int inq, 1901 struct scm_timestamping_internal *tss) 1902 { 1903 unsigned long copy_address = (unsigned long)zc->copybuf_address; 1904 struct msghdr msg = {}; 1905 int err; 1906 1907 zc->length = 0; 1908 zc->recv_skip_hint = 0; 1909 1910 if (copy_address != zc->copybuf_address) 1911 return -EINVAL; 1912 1913 err = import_ubuf(ITER_DEST, (void __user *)copy_address, inq, 1914 &msg.msg_iter); 1915 if (err) 1916 return err; 1917 1918 err = tcp_recvmsg_locked(sk, &msg, inq, MSG_DONTWAIT, 1919 tss, &zc->msg_flags); 1920 if (err < 0) 1921 return err; 1922 1923 zc->copybuf_len = err; 1924 if (likely(zc->copybuf_len)) { 1925 struct sk_buff *skb; 1926 u32 offset; 1927 1928 skb = tcp_recv_skb(sk, tcp_sk(sk)->copied_seq, &offset); 1929 if (skb) 1930 tcp_zerocopy_set_hint_for_skb(sk, zc, skb, offset); 1931 } 1932 return 0; 1933 } 1934 1935 static int tcp_copy_straggler_data(struct tcp_zerocopy_receive *zc, 1936 struct sk_buff *skb, u32 copylen, 1937 u32 *offset, u32 *seq) 1938 { 1939 unsigned long copy_address = (unsigned long)zc->copybuf_address; 1940 struct msghdr msg = {}; 1941 int err; 1942 1943 if (copy_address != zc->copybuf_address) 1944 return -EINVAL; 1945 1946 err = import_ubuf(ITER_DEST, (void __user *)copy_address, copylen, 1947 &msg.msg_iter); 1948 if (err) 1949 return err; 1950 err = skb_copy_datagram_msg(skb, *offset, &msg, copylen); 1951 if (err) 1952 return err; 1953 zc->recv_skip_hint -= copylen; 1954 *offset += copylen; 1955 *seq += copylen; 1956 return (__s32)copylen; 1957 } 1958 1959 static int tcp_zc_handle_leftover(struct tcp_zerocopy_receive *zc, 1960 struct sock *sk, 1961 struct sk_buff *skb, 1962 u32 *seq, 1963 s32 copybuf_len, 1964 struct scm_timestamping_internal *tss) 1965 { 1966 u32 offset, copylen = min_t(u32, copybuf_len, zc->recv_skip_hint); 1967 1968 if (!copylen) 1969 return 0; 1970 /* skb is null if inq < PAGE_SIZE. */ 1971 if (skb) { 1972 offset = *seq - TCP_SKB_CB(skb)->seq; 1973 } else { 1974 skb = tcp_recv_skb(sk, *seq, &offset); 1975 if (TCP_SKB_CB(skb)->has_rxtstamp) { 1976 tcp_update_recv_tstamps(skb, tss); 1977 zc->msg_flags |= TCP_CMSG_TS; 1978 } 1979 } 1980 1981 zc->copybuf_len = tcp_copy_straggler_data(zc, skb, copylen, &offset, 1982 seq); 1983 return zc->copybuf_len < 0 ? 0 : copylen; 1984 } 1985 1986 static int tcp_zerocopy_vm_insert_batch_error(struct vm_area_struct *vma, 1987 struct page **pending_pages, 1988 unsigned long pages_remaining, 1989 unsigned long *address, 1990 u32 *length, 1991 u32 *seq, 1992 struct tcp_zerocopy_receive *zc, 1993 u32 total_bytes_to_map, 1994 int err) 1995 { 1996 /* At least one page did not map. Try zapping if we skipped earlier. */ 1997 if (err == -EBUSY && 1998 zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT) { 1999 u32 maybe_zap_len; 2000 2001 maybe_zap_len = total_bytes_to_map - /* All bytes to map */ 2002 *length + /* Mapped or pending */ 2003 (pages_remaining * PAGE_SIZE); /* Failed map. */ 2004 zap_page_range_single(vma, *address, maybe_zap_len, NULL); 2005 err = 0; 2006 } 2007 2008 if (!err) { 2009 unsigned long leftover_pages = pages_remaining; 2010 int bytes_mapped; 2011 2012 /* We called zap_page_range_single, try to reinsert. */ 2013 err = vm_insert_pages(vma, *address, 2014 pending_pages, 2015 &pages_remaining); 2016 bytes_mapped = PAGE_SIZE * (leftover_pages - pages_remaining); 2017 *seq += bytes_mapped; 2018 *address += bytes_mapped; 2019 } 2020 if (err) { 2021 /* Either we were unable to zap, OR we zapped, retried an 2022 * insert, and still had an issue. Either ways, pages_remaining 2023 * is the number of pages we were unable to map, and we unroll 2024 * some state we speculatively touched before. 2025 */ 2026 const int bytes_not_mapped = PAGE_SIZE * pages_remaining; 2027 2028 *length -= bytes_not_mapped; 2029 zc->recv_skip_hint += bytes_not_mapped; 2030 } 2031 return err; 2032 } 2033 2034 static int tcp_zerocopy_vm_insert_batch(struct vm_area_struct *vma, 2035 struct page **pages, 2036 unsigned int pages_to_map, 2037 unsigned long *address, 2038 u32 *length, 2039 u32 *seq, 2040 struct tcp_zerocopy_receive *zc, 2041 u32 total_bytes_to_map) 2042 { 2043 unsigned long pages_remaining = pages_to_map; 2044 unsigned int pages_mapped; 2045 unsigned int bytes_mapped; 2046 int err; 2047 2048 err = vm_insert_pages(vma, *address, pages, &pages_remaining); 2049 pages_mapped = pages_to_map - (unsigned int)pages_remaining; 2050 bytes_mapped = PAGE_SIZE * pages_mapped; 2051 /* Even if vm_insert_pages fails, it may have partially succeeded in 2052 * mapping (some but not all of the pages). 2053 */ 2054 *seq += bytes_mapped; 2055 *address += bytes_mapped; 2056 2057 if (likely(!err)) 2058 return 0; 2059 2060 /* Error: maybe zap and retry + rollback state for failed inserts. */ 2061 return tcp_zerocopy_vm_insert_batch_error(vma, pages + pages_mapped, 2062 pages_remaining, address, length, seq, zc, total_bytes_to_map, 2063 err); 2064 } 2065 2066 #define TCP_VALID_ZC_MSG_FLAGS (TCP_CMSG_TS) 2067 static void tcp_zc_finalize_rx_tstamp(struct sock *sk, 2068 struct tcp_zerocopy_receive *zc, 2069 struct scm_timestamping_internal *tss) 2070 { 2071 unsigned long msg_control_addr; 2072 struct msghdr cmsg_dummy; 2073 2074 msg_control_addr = (unsigned long)zc->msg_control; 2075 cmsg_dummy.msg_control_user = (void __user *)msg_control_addr; 2076 cmsg_dummy.msg_controllen = 2077 (__kernel_size_t)zc->msg_controllen; 2078 cmsg_dummy.msg_flags = in_compat_syscall() 2079 ? MSG_CMSG_COMPAT : 0; 2080 cmsg_dummy.msg_control_is_user = true; 2081 zc->msg_flags = 0; 2082 if (zc->msg_control == msg_control_addr && 2083 zc->msg_controllen == cmsg_dummy.msg_controllen) { 2084 tcp_recv_timestamp(&cmsg_dummy, sk, tss); 2085 zc->msg_control = (__u64) 2086 ((uintptr_t)cmsg_dummy.msg_control_user); 2087 zc->msg_controllen = 2088 (__u64)cmsg_dummy.msg_controllen; 2089 zc->msg_flags = (__u32)cmsg_dummy.msg_flags; 2090 } 2091 } 2092 2093 static struct vm_area_struct *find_tcp_vma(struct mm_struct *mm, 2094 unsigned long address, 2095 bool *mmap_locked) 2096 { 2097 struct vm_area_struct *vma = lock_vma_under_rcu(mm, address); 2098 2099 if (vma) { 2100 if (vma->vm_ops != &tcp_vm_ops) { 2101 vma_end_read(vma); 2102 return NULL; 2103 } 2104 *mmap_locked = false; 2105 return vma; 2106 } 2107 2108 mmap_read_lock(mm); 2109 vma = vma_lookup(mm, address); 2110 if (!vma || vma->vm_ops != &tcp_vm_ops) { 2111 mmap_read_unlock(mm); 2112 return NULL; 2113 } 2114 *mmap_locked = true; 2115 return vma; 2116 } 2117 2118 #define TCP_ZEROCOPY_PAGE_BATCH_SIZE 32 2119 static int tcp_zerocopy_receive(struct sock *sk, 2120 struct tcp_zerocopy_receive *zc, 2121 struct scm_timestamping_internal *tss) 2122 { 2123 u32 length = 0, offset, vma_len, avail_len, copylen = 0; 2124 unsigned long address = (unsigned long)zc->address; 2125 struct page *pages[TCP_ZEROCOPY_PAGE_BATCH_SIZE]; 2126 s32 copybuf_len = zc->copybuf_len; 2127 struct tcp_sock *tp = tcp_sk(sk); 2128 const skb_frag_t *frags = NULL; 2129 unsigned int pages_to_map = 0; 2130 struct vm_area_struct *vma; 2131 struct sk_buff *skb = NULL; 2132 u32 seq = tp->copied_seq; 2133 u32 total_bytes_to_map; 2134 int inq = tcp_inq(sk); 2135 bool mmap_locked; 2136 int ret; 2137 2138 zc->copybuf_len = 0; 2139 zc->msg_flags = 0; 2140 2141 if (address & (PAGE_SIZE - 1) || address != zc->address) 2142 return -EINVAL; 2143 2144 if (sk->sk_state == TCP_LISTEN) 2145 return -ENOTCONN; 2146 2147 sock_rps_record_flow(sk); 2148 2149 if (inq && inq <= copybuf_len) 2150 return receive_fallback_to_copy(sk, zc, inq, tss); 2151 2152 if (inq < PAGE_SIZE) { 2153 zc->length = 0; 2154 zc->recv_skip_hint = inq; 2155 if (!inq && sock_flag(sk, SOCK_DONE)) 2156 return -EIO; 2157 return 0; 2158 } 2159 2160 vma = find_tcp_vma(current->mm, address, &mmap_locked); 2161 if (!vma) 2162 return -EINVAL; 2163 2164 vma_len = min_t(unsigned long, zc->length, vma->vm_end - address); 2165 avail_len = min_t(u32, vma_len, inq); 2166 total_bytes_to_map = avail_len & ~(PAGE_SIZE - 1); 2167 if (total_bytes_to_map) { 2168 if (!(zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT)) 2169 zap_page_range_single(vma, address, total_bytes_to_map, 2170 NULL); 2171 zc->length = total_bytes_to_map; 2172 zc->recv_skip_hint = 0; 2173 } else { 2174 zc->length = avail_len; 2175 zc->recv_skip_hint = avail_len; 2176 } 2177 ret = 0; 2178 while (length + PAGE_SIZE <= zc->length) { 2179 int mappable_offset; 2180 struct page *page; 2181 2182 if (zc->recv_skip_hint < PAGE_SIZE) { 2183 u32 offset_frag; 2184 2185 if (skb) { 2186 if (zc->recv_skip_hint > 0) 2187 break; 2188 skb = skb->next; 2189 offset = seq - TCP_SKB_CB(skb)->seq; 2190 } else { 2191 skb = tcp_recv_skb(sk, seq, &offset); 2192 } 2193 2194 if (!skb_frags_readable(skb)) 2195 break; 2196 2197 if (TCP_SKB_CB(skb)->has_rxtstamp) { 2198 tcp_update_recv_tstamps(skb, tss); 2199 zc->msg_flags |= TCP_CMSG_TS; 2200 } 2201 zc->recv_skip_hint = skb->len - offset; 2202 frags = skb_advance_to_frag(skb, offset, &offset_frag); 2203 if (!frags || offset_frag) 2204 break; 2205 } 2206 2207 mappable_offset = find_next_mappable_frag(frags, 2208 zc->recv_skip_hint); 2209 if (mappable_offset) { 2210 zc->recv_skip_hint = mappable_offset; 2211 break; 2212 } 2213 page = skb_frag_page(frags); 2214 if (WARN_ON_ONCE(!page)) 2215 break; 2216 2217 prefetchw(page); 2218 pages[pages_to_map++] = page; 2219 length += PAGE_SIZE; 2220 zc->recv_skip_hint -= PAGE_SIZE; 2221 frags++; 2222 if (pages_to_map == TCP_ZEROCOPY_PAGE_BATCH_SIZE || 2223 zc->recv_skip_hint < PAGE_SIZE) { 2224 /* Either full batch, or we're about to go to next skb 2225 * (and we cannot unroll failed ops across skbs). 2226 */ 2227 ret = tcp_zerocopy_vm_insert_batch(vma, pages, 2228 pages_to_map, 2229 &address, &length, 2230 &seq, zc, 2231 total_bytes_to_map); 2232 if (ret) 2233 goto out; 2234 pages_to_map = 0; 2235 } 2236 } 2237 if (pages_to_map) { 2238 ret = tcp_zerocopy_vm_insert_batch(vma, pages, pages_to_map, 2239 &address, &length, &seq, 2240 zc, total_bytes_to_map); 2241 } 2242 out: 2243 if (mmap_locked) 2244 mmap_read_unlock(current->mm); 2245 else 2246 vma_end_read(vma); 2247 /* Try to copy straggler data. */ 2248 if (!ret) 2249 copylen = tcp_zc_handle_leftover(zc, sk, skb, &seq, copybuf_len, tss); 2250 2251 if (length + copylen) { 2252 WRITE_ONCE(tp->copied_seq, seq); 2253 tcp_rcv_space_adjust(sk); 2254 2255 /* Clean up data we have read: This will do ACK frames. */ 2256 tcp_recv_skb(sk, seq, &offset); 2257 tcp_cleanup_rbuf(sk, length + copylen); 2258 ret = 0; 2259 if (length == zc->length) 2260 zc->recv_skip_hint = 0; 2261 } else { 2262 if (!zc->recv_skip_hint && sock_flag(sk, SOCK_DONE)) 2263 ret = -EIO; 2264 } 2265 zc->length = length; 2266 return ret; 2267 } 2268 #endif 2269 2270 /* Similar to __sock_recv_timestamp, but does not require an skb */ 2271 void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk, 2272 struct scm_timestamping_internal *tss) 2273 { 2274 int new_tstamp = sock_flag(sk, SOCK_TSTAMP_NEW); 2275 u32 tsflags = READ_ONCE(sk->sk_tsflags); 2276 bool has_timestamping = false; 2277 2278 if (tss->ts[0].tv_sec || tss->ts[0].tv_nsec) { 2279 if (sock_flag(sk, SOCK_RCVTSTAMP)) { 2280 if (sock_flag(sk, SOCK_RCVTSTAMPNS)) { 2281 if (new_tstamp) { 2282 struct __kernel_timespec kts = { 2283 .tv_sec = tss->ts[0].tv_sec, 2284 .tv_nsec = tss->ts[0].tv_nsec, 2285 }; 2286 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_NEW, 2287 sizeof(kts), &kts); 2288 } else { 2289 struct __kernel_old_timespec ts_old = { 2290 .tv_sec = tss->ts[0].tv_sec, 2291 .tv_nsec = tss->ts[0].tv_nsec, 2292 }; 2293 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_OLD, 2294 sizeof(ts_old), &ts_old); 2295 } 2296 } else { 2297 if (new_tstamp) { 2298 struct __kernel_sock_timeval stv = { 2299 .tv_sec = tss->ts[0].tv_sec, 2300 .tv_usec = tss->ts[0].tv_nsec / 1000, 2301 }; 2302 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_NEW, 2303 sizeof(stv), &stv); 2304 } else { 2305 struct __kernel_old_timeval tv = { 2306 .tv_sec = tss->ts[0].tv_sec, 2307 .tv_usec = tss->ts[0].tv_nsec / 1000, 2308 }; 2309 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_OLD, 2310 sizeof(tv), &tv); 2311 } 2312 } 2313 } 2314 2315 if (tsflags & SOF_TIMESTAMPING_SOFTWARE && 2316 (tsflags & SOF_TIMESTAMPING_RX_SOFTWARE || 2317 !(tsflags & SOF_TIMESTAMPING_OPT_RX_FILTER))) 2318 has_timestamping = true; 2319 else 2320 tss->ts[0] = (struct timespec64) {0}; 2321 } 2322 2323 if (tss->ts[2].tv_sec || tss->ts[2].tv_nsec) { 2324 if (tsflags & SOF_TIMESTAMPING_RAW_HARDWARE && 2325 (tsflags & SOF_TIMESTAMPING_RX_HARDWARE || 2326 !(tsflags & SOF_TIMESTAMPING_OPT_RX_FILTER))) 2327 has_timestamping = true; 2328 else 2329 tss->ts[2] = (struct timespec64) {0}; 2330 } 2331 2332 if (has_timestamping) { 2333 tss->ts[1] = (struct timespec64) {0}; 2334 if (sock_flag(sk, SOCK_TSTAMP_NEW)) 2335 put_cmsg_scm_timestamping64(msg, tss); 2336 else 2337 put_cmsg_scm_timestamping(msg, tss); 2338 } 2339 } 2340 2341 static int tcp_inq_hint(struct sock *sk) 2342 { 2343 const struct tcp_sock *tp = tcp_sk(sk); 2344 u32 copied_seq = READ_ONCE(tp->copied_seq); 2345 u32 rcv_nxt = READ_ONCE(tp->rcv_nxt); 2346 int inq; 2347 2348 inq = rcv_nxt - copied_seq; 2349 if (unlikely(inq < 0 || copied_seq != READ_ONCE(tp->copied_seq))) { 2350 lock_sock(sk); 2351 inq = tp->rcv_nxt - tp->copied_seq; 2352 release_sock(sk); 2353 } 2354 /* After receiving a FIN, tell the user-space to continue reading 2355 * by returning a non-zero inq. 2356 */ 2357 if (inq == 0 && sock_flag(sk, SOCK_DONE)) 2358 inq = 1; 2359 return inq; 2360 } 2361 2362 /* batch __xa_alloc() calls and reduce xa_lock()/xa_unlock() overhead. */ 2363 struct tcp_xa_pool { 2364 u8 max; /* max <= MAX_SKB_FRAGS */ 2365 u8 idx; /* idx <= max */ 2366 __u32 tokens[MAX_SKB_FRAGS]; 2367 netmem_ref netmems[MAX_SKB_FRAGS]; 2368 }; 2369 2370 static void tcp_xa_pool_commit_locked(struct sock *sk, struct tcp_xa_pool *p) 2371 { 2372 int i; 2373 2374 /* Commit part that has been copied to user space. */ 2375 for (i = 0; i < p->idx; i++) 2376 __xa_cmpxchg(&sk->sk_user_frags, p->tokens[i], XA_ZERO_ENTRY, 2377 (__force void *)p->netmems[i], GFP_KERNEL); 2378 /* Rollback what has been pre-allocated and is no longer needed. */ 2379 for (; i < p->max; i++) 2380 __xa_erase(&sk->sk_user_frags, p->tokens[i]); 2381 2382 p->max = 0; 2383 p->idx = 0; 2384 } 2385 2386 static void tcp_xa_pool_commit(struct sock *sk, struct tcp_xa_pool *p) 2387 { 2388 if (!p->max) 2389 return; 2390 2391 xa_lock_bh(&sk->sk_user_frags); 2392 2393 tcp_xa_pool_commit_locked(sk, p); 2394 2395 xa_unlock_bh(&sk->sk_user_frags); 2396 } 2397 2398 static int tcp_xa_pool_refill(struct sock *sk, struct tcp_xa_pool *p, 2399 unsigned int max_frags) 2400 { 2401 int err, k; 2402 2403 if (p->idx < p->max) 2404 return 0; 2405 2406 xa_lock_bh(&sk->sk_user_frags); 2407 2408 tcp_xa_pool_commit_locked(sk, p); 2409 2410 for (k = 0; k < max_frags; k++) { 2411 err = __xa_alloc(&sk->sk_user_frags, &p->tokens[k], 2412 XA_ZERO_ENTRY, xa_limit_31b, GFP_KERNEL); 2413 if (err) 2414 break; 2415 } 2416 2417 xa_unlock_bh(&sk->sk_user_frags); 2418 2419 p->max = k; 2420 p->idx = 0; 2421 return k ? 0 : err; 2422 } 2423 2424 /* On error, returns the -errno. On success, returns number of bytes sent to the 2425 * user. May not consume all of @remaining_len. 2426 */ 2427 static int tcp_recvmsg_dmabuf(struct sock *sk, const struct sk_buff *skb, 2428 unsigned int offset, struct msghdr *msg, 2429 int remaining_len) 2430 { 2431 struct dmabuf_cmsg dmabuf_cmsg = { 0 }; 2432 struct tcp_xa_pool tcp_xa_pool; 2433 unsigned int start; 2434 int i, copy, n; 2435 int sent = 0; 2436 int err = 0; 2437 2438 tcp_xa_pool.max = 0; 2439 tcp_xa_pool.idx = 0; 2440 do { 2441 start = skb_headlen(skb); 2442 2443 if (skb_frags_readable(skb)) { 2444 err = -ENODEV; 2445 goto out; 2446 } 2447 2448 /* Copy header. */ 2449 copy = start - offset; 2450 if (copy > 0) { 2451 copy = min(copy, remaining_len); 2452 2453 n = copy_to_iter(skb->data + offset, copy, 2454 &msg->msg_iter); 2455 if (n != copy) { 2456 err = -EFAULT; 2457 goto out; 2458 } 2459 2460 offset += copy; 2461 remaining_len -= copy; 2462 2463 /* First a dmabuf_cmsg for # bytes copied to user 2464 * buffer. 2465 */ 2466 memset(&dmabuf_cmsg, 0, sizeof(dmabuf_cmsg)); 2467 dmabuf_cmsg.frag_size = copy; 2468 err = put_cmsg_notrunc(msg, SOL_SOCKET, 2469 SO_DEVMEM_LINEAR, 2470 sizeof(dmabuf_cmsg), 2471 &dmabuf_cmsg); 2472 if (err) 2473 goto out; 2474 2475 sent += copy; 2476 2477 if (remaining_len == 0) 2478 goto out; 2479 } 2480 2481 /* after that, send information of dmabuf pages through a 2482 * sequence of cmsg 2483 */ 2484 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2485 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2486 struct net_iov *niov; 2487 u64 frag_offset; 2488 int end; 2489 2490 /* !skb_frags_readable() should indicate that ALL the 2491 * frags in this skb are dmabuf net_iovs. We're checking 2492 * for that flag above, but also check individual frags 2493 * here. If the tcp stack is not setting 2494 * skb_frags_readable() correctly, we still don't want 2495 * to crash here. 2496 */ 2497 if (!skb_frag_net_iov(frag)) { 2498 net_err_ratelimited("Found non-dmabuf skb with net_iov"); 2499 err = -ENODEV; 2500 goto out; 2501 } 2502 2503 niov = skb_frag_net_iov(frag); 2504 if (!net_is_devmem_iov(niov)) { 2505 err = -ENODEV; 2506 goto out; 2507 } 2508 2509 end = start + skb_frag_size(frag); 2510 copy = end - offset; 2511 2512 if (copy > 0) { 2513 copy = min(copy, remaining_len); 2514 2515 frag_offset = net_iov_virtual_addr(niov) + 2516 skb_frag_off(frag) + offset - 2517 start; 2518 dmabuf_cmsg.frag_offset = frag_offset; 2519 dmabuf_cmsg.frag_size = copy; 2520 err = tcp_xa_pool_refill(sk, &tcp_xa_pool, 2521 skb_shinfo(skb)->nr_frags - i); 2522 if (err) 2523 goto out; 2524 2525 /* Will perform the exchange later */ 2526 dmabuf_cmsg.frag_token = tcp_xa_pool.tokens[tcp_xa_pool.idx]; 2527 dmabuf_cmsg.dmabuf_id = net_devmem_iov_binding_id(niov); 2528 2529 offset += copy; 2530 remaining_len -= copy; 2531 2532 err = put_cmsg_notrunc(msg, SOL_SOCKET, 2533 SO_DEVMEM_DMABUF, 2534 sizeof(dmabuf_cmsg), 2535 &dmabuf_cmsg); 2536 if (err) 2537 goto out; 2538 2539 atomic_long_inc(&niov->pp_ref_count); 2540 tcp_xa_pool.netmems[tcp_xa_pool.idx++] = skb_frag_netmem(frag); 2541 2542 sent += copy; 2543 2544 if (remaining_len == 0) 2545 goto out; 2546 } 2547 start = end; 2548 } 2549 2550 tcp_xa_pool_commit(sk, &tcp_xa_pool); 2551 if (!remaining_len) 2552 goto out; 2553 2554 /* if remaining_len is not satisfied yet, we need to go to the 2555 * next frag in the frag_list to satisfy remaining_len. 2556 */ 2557 skb = skb_shinfo(skb)->frag_list ?: skb->next; 2558 2559 offset = offset - start; 2560 } while (skb); 2561 2562 if (remaining_len) { 2563 err = -EFAULT; 2564 goto out; 2565 } 2566 2567 out: 2568 tcp_xa_pool_commit(sk, &tcp_xa_pool); 2569 if (!sent) 2570 sent = err; 2571 2572 return sent; 2573 } 2574 2575 /* 2576 * This routine copies from a sock struct into the user buffer. 2577 * 2578 * Technical note: in 2.3 we work on _locked_ socket, so that 2579 * tricks with *seq access order and skb->users are not required. 2580 * Probably, code can be easily improved even more. 2581 */ 2582 2583 static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len, 2584 int flags, struct scm_timestamping_internal *tss, 2585 int *cmsg_flags) 2586 { 2587 struct tcp_sock *tp = tcp_sk(sk); 2588 int last_copied_dmabuf = -1; /* uninitialized */ 2589 int copied = 0; 2590 u32 peek_seq; 2591 u32 *seq; 2592 unsigned long used; 2593 int err; 2594 int target; /* Read at least this many bytes */ 2595 long timeo; 2596 struct sk_buff *skb, *last; 2597 u32 peek_offset = 0; 2598 u32 urg_hole = 0; 2599 2600 err = -ENOTCONN; 2601 if (sk->sk_state == TCP_LISTEN) 2602 goto out; 2603 2604 if (tp->recvmsg_inq) { 2605 *cmsg_flags = TCP_CMSG_INQ; 2606 msg->msg_get_inq = 1; 2607 } 2608 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 2609 2610 /* Urgent data needs to be handled specially. */ 2611 if (flags & MSG_OOB) 2612 goto recv_urg; 2613 2614 if (unlikely(tp->repair)) { 2615 err = -EPERM; 2616 if (!(flags & MSG_PEEK)) 2617 goto out; 2618 2619 if (tp->repair_queue == TCP_SEND_QUEUE) 2620 goto recv_sndq; 2621 2622 err = -EINVAL; 2623 if (tp->repair_queue == TCP_NO_QUEUE) 2624 goto out; 2625 2626 /* 'common' recv queue MSG_PEEK-ing */ 2627 } 2628 2629 seq = &tp->copied_seq; 2630 if (flags & MSG_PEEK) { 2631 peek_offset = max(sk_peek_offset(sk, flags), 0); 2632 peek_seq = tp->copied_seq + peek_offset; 2633 seq = &peek_seq; 2634 } 2635 2636 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 2637 2638 do { 2639 u32 offset; 2640 2641 /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */ 2642 if (unlikely(tp->urg_data) && tp->urg_seq == *seq) { 2643 if (copied) 2644 break; 2645 if (signal_pending(current)) { 2646 copied = timeo ? sock_intr_errno(timeo) : -EAGAIN; 2647 break; 2648 } 2649 } 2650 2651 /* Next get a buffer. */ 2652 2653 last = skb_peek_tail(&sk->sk_receive_queue); 2654 skb_queue_walk(&sk->sk_receive_queue, skb) { 2655 last = skb; 2656 /* Now that we have two receive queues this 2657 * shouldn't happen. 2658 */ 2659 if (WARN(before(*seq, TCP_SKB_CB(skb)->seq), 2660 "TCP recvmsg seq # bug: copied %X, seq %X, rcvnxt %X, fl %X\n", 2661 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, 2662 flags)) 2663 break; 2664 2665 offset = *seq - TCP_SKB_CB(skb)->seq; 2666 if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) { 2667 pr_err_once("%s: found a SYN, please report !\n", __func__); 2668 offset--; 2669 } 2670 if (offset < skb->len) 2671 goto found_ok_skb; 2672 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 2673 goto found_fin_ok; 2674 WARN(!(flags & MSG_PEEK), 2675 "TCP recvmsg seq # bug 2: copied %X, seq %X, rcvnxt %X, fl %X\n", 2676 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags); 2677 } 2678 2679 /* Well, if we have backlog, try to process it now yet. */ 2680 2681 if (copied >= target && !READ_ONCE(sk->sk_backlog.tail)) 2682 break; 2683 2684 if (copied) { 2685 if (!timeo || 2686 sk->sk_err || 2687 sk->sk_state == TCP_CLOSE || 2688 (sk->sk_shutdown & RCV_SHUTDOWN) || 2689 signal_pending(current)) 2690 break; 2691 } else { 2692 if (sock_flag(sk, SOCK_DONE)) 2693 break; 2694 2695 if (sk->sk_err) { 2696 copied = sock_error(sk); 2697 break; 2698 } 2699 2700 if (sk->sk_shutdown & RCV_SHUTDOWN) 2701 break; 2702 2703 if (sk->sk_state == TCP_CLOSE) { 2704 /* This occurs when user tries to read 2705 * from never connected socket. 2706 */ 2707 copied = -ENOTCONN; 2708 break; 2709 } 2710 2711 if (!timeo) { 2712 copied = -EAGAIN; 2713 break; 2714 } 2715 2716 if (signal_pending(current)) { 2717 copied = sock_intr_errno(timeo); 2718 break; 2719 } 2720 } 2721 2722 if (copied >= target) { 2723 /* Do not sleep, just process backlog. */ 2724 __sk_flush_backlog(sk); 2725 } else { 2726 tcp_cleanup_rbuf(sk, copied); 2727 err = sk_wait_data(sk, &timeo, last); 2728 if (err < 0) { 2729 err = copied ? : err; 2730 goto out; 2731 } 2732 } 2733 2734 if ((flags & MSG_PEEK) && 2735 (peek_seq - peek_offset - copied - urg_hole != tp->copied_seq)) { 2736 net_dbg_ratelimited("TCP(%s:%d): Application bug, race in MSG_PEEK\n", 2737 current->comm, 2738 task_pid_nr(current)); 2739 peek_seq = tp->copied_seq + peek_offset; 2740 } 2741 continue; 2742 2743 found_ok_skb: 2744 /* Ok so how much can we use? */ 2745 used = skb->len - offset; 2746 if (len < used) 2747 used = len; 2748 2749 /* Do we have urgent data here? */ 2750 if (unlikely(tp->urg_data)) { 2751 u32 urg_offset = tp->urg_seq - *seq; 2752 if (urg_offset < used) { 2753 if (!urg_offset) { 2754 if (!sock_flag(sk, SOCK_URGINLINE)) { 2755 WRITE_ONCE(*seq, *seq + 1); 2756 urg_hole++; 2757 offset++; 2758 used--; 2759 if (!used) 2760 goto skip_copy; 2761 } 2762 } else 2763 used = urg_offset; 2764 } 2765 } 2766 2767 if (!(flags & MSG_TRUNC)) { 2768 if (last_copied_dmabuf != -1 && 2769 last_copied_dmabuf != !skb_frags_readable(skb)) 2770 break; 2771 2772 if (skb_frags_readable(skb)) { 2773 err = skb_copy_datagram_msg(skb, offset, msg, 2774 used); 2775 if (err) { 2776 /* Exception. Bailout! */ 2777 if (!copied) 2778 copied = -EFAULT; 2779 break; 2780 } 2781 } else { 2782 if (!(flags & MSG_SOCK_DEVMEM)) { 2783 /* dmabuf skbs can only be received 2784 * with the MSG_SOCK_DEVMEM flag. 2785 */ 2786 if (!copied) 2787 copied = -EFAULT; 2788 2789 break; 2790 } 2791 2792 err = tcp_recvmsg_dmabuf(sk, skb, offset, msg, 2793 used); 2794 if (err <= 0) { 2795 if (!copied) 2796 copied = -EFAULT; 2797 2798 break; 2799 } 2800 used = err; 2801 } 2802 } 2803 2804 last_copied_dmabuf = !skb_frags_readable(skb); 2805 2806 WRITE_ONCE(*seq, *seq + used); 2807 copied += used; 2808 len -= used; 2809 if (flags & MSG_PEEK) 2810 sk_peek_offset_fwd(sk, used); 2811 else 2812 sk_peek_offset_bwd(sk, used); 2813 tcp_rcv_space_adjust(sk); 2814 2815 skip_copy: 2816 if (unlikely(tp->urg_data) && after(tp->copied_seq, tp->urg_seq)) { 2817 WRITE_ONCE(tp->urg_data, 0); 2818 tcp_fast_path_check(sk); 2819 } 2820 2821 if (TCP_SKB_CB(skb)->has_rxtstamp) { 2822 tcp_update_recv_tstamps(skb, tss); 2823 *cmsg_flags |= TCP_CMSG_TS; 2824 } 2825 2826 if (used + offset < skb->len) 2827 continue; 2828 2829 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 2830 goto found_fin_ok; 2831 if (!(flags & MSG_PEEK)) 2832 tcp_eat_recv_skb(sk, skb); 2833 continue; 2834 2835 found_fin_ok: 2836 /* Process the FIN. */ 2837 WRITE_ONCE(*seq, *seq + 1); 2838 if (!(flags & MSG_PEEK)) 2839 tcp_eat_recv_skb(sk, skb); 2840 break; 2841 } while (len > 0); 2842 2843 /* According to UNIX98, msg_name/msg_namelen are ignored 2844 * on connected socket. I was just happy when found this 8) --ANK 2845 */ 2846 2847 /* Clean up data we have read: This will do ACK frames. */ 2848 tcp_cleanup_rbuf(sk, copied); 2849 return copied; 2850 2851 out: 2852 return err; 2853 2854 recv_urg: 2855 err = tcp_recv_urg(sk, msg, len, flags); 2856 goto out; 2857 2858 recv_sndq: 2859 err = tcp_peek_sndq(sk, msg, len); 2860 goto out; 2861 } 2862 2863 int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags, 2864 int *addr_len) 2865 { 2866 int cmsg_flags = 0, ret; 2867 struct scm_timestamping_internal tss; 2868 2869 if (unlikely(flags & MSG_ERRQUEUE)) 2870 return inet_recv_error(sk, msg, len, addr_len); 2871 2872 if (sk_can_busy_loop(sk) && 2873 skb_queue_empty_lockless(&sk->sk_receive_queue) && 2874 sk->sk_state == TCP_ESTABLISHED) 2875 sk_busy_loop(sk, flags & MSG_DONTWAIT); 2876 2877 lock_sock(sk); 2878 ret = tcp_recvmsg_locked(sk, msg, len, flags, &tss, &cmsg_flags); 2879 release_sock(sk); 2880 2881 if ((cmsg_flags || msg->msg_get_inq) && ret >= 0) { 2882 if (cmsg_flags & TCP_CMSG_TS) 2883 tcp_recv_timestamp(msg, sk, &tss); 2884 if (msg->msg_get_inq) { 2885 msg->msg_inq = tcp_inq_hint(sk); 2886 if (cmsg_flags & TCP_CMSG_INQ) 2887 put_cmsg(msg, SOL_TCP, TCP_CM_INQ, 2888 sizeof(msg->msg_inq), &msg->msg_inq); 2889 } 2890 } 2891 return ret; 2892 } 2893 EXPORT_IPV6_MOD(tcp_recvmsg); 2894 2895 void tcp_set_state(struct sock *sk, int state) 2896 { 2897 int oldstate = sk->sk_state; 2898 2899 /* We defined a new enum for TCP states that are exported in BPF 2900 * so as not force the internal TCP states to be frozen. The 2901 * following checks will detect if an internal state value ever 2902 * differs from the BPF value. If this ever happens, then we will 2903 * need to remap the internal value to the BPF value before calling 2904 * tcp_call_bpf_2arg. 2905 */ 2906 BUILD_BUG_ON((int)BPF_TCP_ESTABLISHED != (int)TCP_ESTABLISHED); 2907 BUILD_BUG_ON((int)BPF_TCP_SYN_SENT != (int)TCP_SYN_SENT); 2908 BUILD_BUG_ON((int)BPF_TCP_SYN_RECV != (int)TCP_SYN_RECV); 2909 BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT1 != (int)TCP_FIN_WAIT1); 2910 BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT2 != (int)TCP_FIN_WAIT2); 2911 BUILD_BUG_ON((int)BPF_TCP_TIME_WAIT != (int)TCP_TIME_WAIT); 2912 BUILD_BUG_ON((int)BPF_TCP_CLOSE != (int)TCP_CLOSE); 2913 BUILD_BUG_ON((int)BPF_TCP_CLOSE_WAIT != (int)TCP_CLOSE_WAIT); 2914 BUILD_BUG_ON((int)BPF_TCP_LAST_ACK != (int)TCP_LAST_ACK); 2915 BUILD_BUG_ON((int)BPF_TCP_LISTEN != (int)TCP_LISTEN); 2916 BUILD_BUG_ON((int)BPF_TCP_CLOSING != (int)TCP_CLOSING); 2917 BUILD_BUG_ON((int)BPF_TCP_NEW_SYN_RECV != (int)TCP_NEW_SYN_RECV); 2918 BUILD_BUG_ON((int)BPF_TCP_BOUND_INACTIVE != (int)TCP_BOUND_INACTIVE); 2919 BUILD_BUG_ON((int)BPF_TCP_MAX_STATES != (int)TCP_MAX_STATES); 2920 2921 /* bpf uapi header bpf.h defines an anonymous enum with values 2922 * BPF_TCP_* used by bpf programs. Currently gcc built vmlinux 2923 * is able to emit this enum in DWARF due to the above BUILD_BUG_ON. 2924 * But clang built vmlinux does not have this enum in DWARF 2925 * since clang removes the above code before generating IR/debuginfo. 2926 * Let us explicitly emit the type debuginfo to ensure the 2927 * above-mentioned anonymous enum in the vmlinux DWARF and hence BTF 2928 * regardless of which compiler is used. 2929 */ 2930 BTF_TYPE_EMIT_ENUM(BPF_TCP_ESTABLISHED); 2931 2932 if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_STATE_CB_FLAG)) 2933 tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state); 2934 2935 switch (state) { 2936 case TCP_ESTABLISHED: 2937 if (oldstate != TCP_ESTABLISHED) 2938 TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB); 2939 break; 2940 case TCP_CLOSE_WAIT: 2941 if (oldstate == TCP_SYN_RECV) 2942 TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB); 2943 break; 2944 2945 case TCP_CLOSE: 2946 if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED) 2947 TCP_INC_STATS(sock_net(sk), TCP_MIB_ESTABRESETS); 2948 2949 sk->sk_prot->unhash(sk); 2950 if (inet_csk(sk)->icsk_bind_hash && 2951 !(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) 2952 inet_put_port(sk); 2953 fallthrough; 2954 default: 2955 if (oldstate == TCP_ESTABLISHED || oldstate == TCP_CLOSE_WAIT) 2956 TCP_DEC_STATS(sock_net(sk), TCP_MIB_CURRESTAB); 2957 } 2958 2959 /* Change state AFTER socket is unhashed to avoid closed 2960 * socket sitting in hash tables. 2961 */ 2962 inet_sk_state_store(sk, state); 2963 } 2964 EXPORT_SYMBOL_GPL(tcp_set_state); 2965 2966 /* 2967 * State processing on a close. This implements the state shift for 2968 * sending our FIN frame. Note that we only send a FIN for some 2969 * states. A shutdown() may have already sent the FIN, or we may be 2970 * closed. 2971 */ 2972 2973 static const unsigned char new_state[16] = { 2974 /* current state: new state: action: */ 2975 [0 /* (Invalid) */] = TCP_CLOSE, 2976 [TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 2977 [TCP_SYN_SENT] = TCP_CLOSE, 2978 [TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 2979 [TCP_FIN_WAIT1] = TCP_FIN_WAIT1, 2980 [TCP_FIN_WAIT2] = TCP_FIN_WAIT2, 2981 [TCP_TIME_WAIT] = TCP_CLOSE, 2982 [TCP_CLOSE] = TCP_CLOSE, 2983 [TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN, 2984 [TCP_LAST_ACK] = TCP_LAST_ACK, 2985 [TCP_LISTEN] = TCP_CLOSE, 2986 [TCP_CLOSING] = TCP_CLOSING, 2987 [TCP_NEW_SYN_RECV] = TCP_CLOSE, /* should not happen ! */ 2988 }; 2989 2990 static int tcp_close_state(struct sock *sk) 2991 { 2992 int next = (int)new_state[sk->sk_state]; 2993 int ns = next & TCP_STATE_MASK; 2994 2995 tcp_set_state(sk, ns); 2996 2997 return next & TCP_ACTION_FIN; 2998 } 2999 3000 /* 3001 * Shutdown the sending side of a connection. Much like close except 3002 * that we don't receive shut down or sock_set_flag(sk, SOCK_DEAD). 3003 */ 3004 3005 void tcp_shutdown(struct sock *sk, int how) 3006 { 3007 /* We need to grab some memory, and put together a FIN, 3008 * and then put it into the queue to be sent. 3009 * Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92. 3010 */ 3011 if (!(how & SEND_SHUTDOWN)) 3012 return; 3013 3014 /* If we've already sent a FIN, or it's a closed state, skip this. */ 3015 if ((1 << sk->sk_state) & 3016 (TCPF_ESTABLISHED | TCPF_SYN_SENT | 3017 TCPF_CLOSE_WAIT)) { 3018 /* Clear out any half completed packets. FIN if needed. */ 3019 if (tcp_close_state(sk)) 3020 tcp_send_fin(sk); 3021 } 3022 } 3023 EXPORT_IPV6_MOD(tcp_shutdown); 3024 3025 int tcp_orphan_count_sum(void) 3026 { 3027 int i, total = 0; 3028 3029 for_each_possible_cpu(i) 3030 total += per_cpu(tcp_orphan_count, i); 3031 3032 return max(total, 0); 3033 } 3034 3035 static int tcp_orphan_cache; 3036 static struct timer_list tcp_orphan_timer; 3037 #define TCP_ORPHAN_TIMER_PERIOD msecs_to_jiffies(100) 3038 3039 static void tcp_orphan_update(struct timer_list *unused) 3040 { 3041 WRITE_ONCE(tcp_orphan_cache, tcp_orphan_count_sum()); 3042 mod_timer(&tcp_orphan_timer, jiffies + TCP_ORPHAN_TIMER_PERIOD); 3043 } 3044 3045 static bool tcp_too_many_orphans(int shift) 3046 { 3047 return READ_ONCE(tcp_orphan_cache) << shift > 3048 READ_ONCE(sysctl_tcp_max_orphans); 3049 } 3050 3051 static bool tcp_out_of_memory(const struct sock *sk) 3052 { 3053 if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF && 3054 sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2)) 3055 return true; 3056 return false; 3057 } 3058 3059 bool tcp_check_oom(const struct sock *sk, int shift) 3060 { 3061 bool too_many_orphans, out_of_socket_memory; 3062 3063 too_many_orphans = tcp_too_many_orphans(shift); 3064 out_of_socket_memory = tcp_out_of_memory(sk); 3065 3066 if (too_many_orphans) 3067 net_info_ratelimited("too many orphaned sockets\n"); 3068 if (out_of_socket_memory) 3069 net_info_ratelimited("out of memory -- consider tuning tcp_mem\n"); 3070 return too_many_orphans || out_of_socket_memory; 3071 } 3072 3073 void __tcp_close(struct sock *sk, long timeout) 3074 { 3075 struct sk_buff *skb; 3076 int data_was_unread = 0; 3077 int state; 3078 3079 WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK); 3080 3081 if (sk->sk_state == TCP_LISTEN) { 3082 tcp_set_state(sk, TCP_CLOSE); 3083 3084 /* Special case. */ 3085 inet_csk_listen_stop(sk); 3086 3087 goto adjudge_to_death; 3088 } 3089 3090 /* We need to flush the recv. buffs. We do this only on the 3091 * descriptor close, not protocol-sourced closes, because the 3092 * reader process may not have drained the data yet! 3093 */ 3094 while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) { 3095 u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq; 3096 3097 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 3098 len--; 3099 data_was_unread += len; 3100 __kfree_skb(skb); 3101 } 3102 3103 /* If socket has been already reset (e.g. in tcp_reset()) - kill it. */ 3104 if (sk->sk_state == TCP_CLOSE) 3105 goto adjudge_to_death; 3106 3107 /* As outlined in RFC 2525, section 2.17, we send a RST here because 3108 * data was lost. To witness the awful effects of the old behavior of 3109 * always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk 3110 * GET in an FTP client, suspend the process, wait for the client to 3111 * advertise a zero window, then kill -9 the FTP client, wheee... 3112 * Note: timeout is always zero in such a case. 3113 */ 3114 if (unlikely(tcp_sk(sk)->repair)) { 3115 sk->sk_prot->disconnect(sk, 0); 3116 } else if (data_was_unread) { 3117 /* Unread data was tossed, zap the connection. */ 3118 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE); 3119 tcp_set_state(sk, TCP_CLOSE); 3120 tcp_send_active_reset(sk, sk->sk_allocation, 3121 SK_RST_REASON_TCP_ABORT_ON_CLOSE); 3122 } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) { 3123 /* Check zero linger _after_ checking for unread data. */ 3124 sk->sk_prot->disconnect(sk, 0); 3125 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA); 3126 } else if (tcp_close_state(sk)) { 3127 /* We FIN if the application ate all the data before 3128 * zapping the connection. 3129 */ 3130 3131 /* RED-PEN. Formally speaking, we have broken TCP state 3132 * machine. State transitions: 3133 * 3134 * TCP_ESTABLISHED -> TCP_FIN_WAIT1 3135 * TCP_SYN_RECV -> TCP_FIN_WAIT1 (it is difficult) 3136 * TCP_CLOSE_WAIT -> TCP_LAST_ACK 3137 * 3138 * are legal only when FIN has been sent (i.e. in window), 3139 * rather than queued out of window. Purists blame. 3140 * 3141 * F.e. "RFC state" is ESTABLISHED, 3142 * if Linux state is FIN-WAIT-1, but FIN is still not sent. 3143 * 3144 * The visible declinations are that sometimes 3145 * we enter time-wait state, when it is not required really 3146 * (harmless), do not send active resets, when they are 3147 * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when 3148 * they look as CLOSING or LAST_ACK for Linux) 3149 * Probably, I missed some more holelets. 3150 * --ANK 3151 * XXX (TFO) - To start off we don't support SYN+ACK+FIN 3152 * in a single packet! (May consider it later but will 3153 * probably need API support or TCP_CORK SYN-ACK until 3154 * data is written and socket is closed.) 3155 */ 3156 tcp_send_fin(sk); 3157 } 3158 3159 sk_stream_wait_close(sk, timeout); 3160 3161 adjudge_to_death: 3162 state = sk->sk_state; 3163 sock_hold(sk); 3164 sock_orphan(sk); 3165 3166 local_bh_disable(); 3167 bh_lock_sock(sk); 3168 /* remove backlog if any, without releasing ownership. */ 3169 __release_sock(sk); 3170 3171 this_cpu_inc(tcp_orphan_count); 3172 3173 /* Have we already been destroyed by a softirq or backlog? */ 3174 if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE) 3175 goto out; 3176 3177 /* This is a (useful) BSD violating of the RFC. There is a 3178 * problem with TCP as specified in that the other end could 3179 * keep a socket open forever with no application left this end. 3180 * We use a 1 minute timeout (about the same as BSD) then kill 3181 * our end. If they send after that then tough - BUT: long enough 3182 * that we won't make the old 4*rto = almost no time - whoops 3183 * reset mistake. 3184 * 3185 * Nope, it was not mistake. It is really desired behaviour 3186 * f.e. on http servers, when such sockets are useless, but 3187 * consume significant resources. Let's do it with special 3188 * linger2 option. --ANK 3189 */ 3190 3191 if (sk->sk_state == TCP_FIN_WAIT2) { 3192 struct tcp_sock *tp = tcp_sk(sk); 3193 if (READ_ONCE(tp->linger2) < 0) { 3194 tcp_set_state(sk, TCP_CLOSE); 3195 tcp_send_active_reset(sk, GFP_ATOMIC, 3196 SK_RST_REASON_TCP_ABORT_ON_LINGER); 3197 __NET_INC_STATS(sock_net(sk), 3198 LINUX_MIB_TCPABORTONLINGER); 3199 } else { 3200 const int tmo = tcp_fin_time(sk); 3201 3202 if (tmo > TCP_TIMEWAIT_LEN) { 3203 tcp_reset_keepalive_timer(sk, 3204 tmo - TCP_TIMEWAIT_LEN); 3205 } else { 3206 tcp_time_wait(sk, TCP_FIN_WAIT2, tmo); 3207 goto out; 3208 } 3209 } 3210 } 3211 if (sk->sk_state != TCP_CLOSE) { 3212 if (tcp_check_oom(sk, 0)) { 3213 tcp_set_state(sk, TCP_CLOSE); 3214 tcp_send_active_reset(sk, GFP_ATOMIC, 3215 SK_RST_REASON_TCP_ABORT_ON_MEMORY); 3216 __NET_INC_STATS(sock_net(sk), 3217 LINUX_MIB_TCPABORTONMEMORY); 3218 } else if (!check_net(sock_net(sk))) { 3219 /* Not possible to send reset; just close */ 3220 tcp_set_state(sk, TCP_CLOSE); 3221 } 3222 } 3223 3224 if (sk->sk_state == TCP_CLOSE) { 3225 struct request_sock *req; 3226 3227 req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk, 3228 lockdep_sock_is_held(sk)); 3229 /* We could get here with a non-NULL req if the socket is 3230 * aborted (e.g., closed with unread data) before 3WHS 3231 * finishes. 3232 */ 3233 if (req) 3234 reqsk_fastopen_remove(sk, req, false); 3235 inet_csk_destroy_sock(sk); 3236 } 3237 /* Otherwise, socket is reprieved until protocol close. */ 3238 3239 out: 3240 bh_unlock_sock(sk); 3241 local_bh_enable(); 3242 } 3243 3244 void tcp_close(struct sock *sk, long timeout) 3245 { 3246 lock_sock(sk); 3247 __tcp_close(sk, timeout); 3248 release_sock(sk); 3249 if (!sk->sk_net_refcnt) 3250 inet_csk_clear_xmit_timers_sync(sk); 3251 sock_put(sk); 3252 } 3253 EXPORT_SYMBOL(tcp_close); 3254 3255 /* These states need RST on ABORT according to RFC793 */ 3256 3257 static inline bool tcp_need_reset(int state) 3258 { 3259 return (1 << state) & 3260 (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 | 3261 TCPF_FIN_WAIT2 | TCPF_SYN_RECV); 3262 } 3263 3264 static void tcp_rtx_queue_purge(struct sock *sk) 3265 { 3266 struct rb_node *p = rb_first(&sk->tcp_rtx_queue); 3267 3268 tcp_sk(sk)->highest_sack = NULL; 3269 while (p) { 3270 struct sk_buff *skb = rb_to_skb(p); 3271 3272 p = rb_next(p); 3273 /* Since we are deleting whole queue, no need to 3274 * list_del(&skb->tcp_tsorted_anchor) 3275 */ 3276 tcp_rtx_queue_unlink(skb, sk); 3277 tcp_wmem_free_skb(sk, skb); 3278 } 3279 } 3280 3281 void tcp_write_queue_purge(struct sock *sk) 3282 { 3283 struct sk_buff *skb; 3284 3285 tcp_chrono_stop(sk, TCP_CHRONO_BUSY); 3286 while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) { 3287 tcp_skb_tsorted_anchor_cleanup(skb); 3288 tcp_wmem_free_skb(sk, skb); 3289 } 3290 tcp_rtx_queue_purge(sk); 3291 INIT_LIST_HEAD(&tcp_sk(sk)->tsorted_sent_queue); 3292 tcp_clear_all_retrans_hints(tcp_sk(sk)); 3293 tcp_sk(sk)->packets_out = 0; 3294 inet_csk(sk)->icsk_backoff = 0; 3295 } 3296 3297 int tcp_disconnect(struct sock *sk, int flags) 3298 { 3299 struct inet_sock *inet = inet_sk(sk); 3300 struct inet_connection_sock *icsk = inet_csk(sk); 3301 struct tcp_sock *tp = tcp_sk(sk); 3302 int old_state = sk->sk_state; 3303 u32 seq; 3304 3305 if (old_state != TCP_CLOSE) 3306 tcp_set_state(sk, TCP_CLOSE); 3307 3308 /* ABORT function of RFC793 */ 3309 if (old_state == TCP_LISTEN) { 3310 inet_csk_listen_stop(sk); 3311 } else if (unlikely(tp->repair)) { 3312 WRITE_ONCE(sk->sk_err, ECONNABORTED); 3313 } else if (tcp_need_reset(old_state)) { 3314 tcp_send_active_reset(sk, gfp_any(), SK_RST_REASON_TCP_STATE); 3315 WRITE_ONCE(sk->sk_err, ECONNRESET); 3316 } else if (tp->snd_nxt != tp->write_seq && 3317 (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK)) { 3318 /* The last check adjusts for discrepancy of Linux wrt. RFC 3319 * states 3320 */ 3321 tcp_send_active_reset(sk, gfp_any(), 3322 SK_RST_REASON_TCP_DISCONNECT_WITH_DATA); 3323 WRITE_ONCE(sk->sk_err, ECONNRESET); 3324 } else if (old_state == TCP_SYN_SENT) 3325 WRITE_ONCE(sk->sk_err, ECONNRESET); 3326 3327 tcp_clear_xmit_timers(sk); 3328 __skb_queue_purge(&sk->sk_receive_queue); 3329 WRITE_ONCE(tp->copied_seq, tp->rcv_nxt); 3330 WRITE_ONCE(tp->urg_data, 0); 3331 sk_set_peek_off(sk, -1); 3332 tcp_write_queue_purge(sk); 3333 tcp_fastopen_active_disable_ofo_check(sk); 3334 skb_rbtree_purge(&tp->out_of_order_queue); 3335 3336 inet->inet_dport = 0; 3337 3338 inet_bhash2_reset_saddr(sk); 3339 3340 WRITE_ONCE(sk->sk_shutdown, 0); 3341 sock_reset_flag(sk, SOCK_DONE); 3342 tp->srtt_us = 0; 3343 tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT); 3344 tp->rcv_rtt_last_tsecr = 0; 3345 3346 seq = tp->write_seq + tp->max_window + 2; 3347 if (!seq) 3348 seq = 1; 3349 WRITE_ONCE(tp->write_seq, seq); 3350 3351 icsk->icsk_backoff = 0; 3352 icsk->icsk_probes_out = 0; 3353 icsk->icsk_probes_tstamp = 0; 3354 icsk->icsk_rto = TCP_TIMEOUT_INIT; 3355 WRITE_ONCE(icsk->icsk_rto_min, TCP_RTO_MIN); 3356 WRITE_ONCE(icsk->icsk_delack_max, TCP_DELACK_MAX); 3357 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; 3358 tcp_snd_cwnd_set(tp, TCP_INIT_CWND); 3359 tp->snd_cwnd_cnt = 0; 3360 tp->is_cwnd_limited = 0; 3361 tp->max_packets_out = 0; 3362 tp->window_clamp = 0; 3363 tp->delivered = 0; 3364 tp->delivered_ce = 0; 3365 if (icsk->icsk_ca_initialized && icsk->icsk_ca_ops->release) 3366 icsk->icsk_ca_ops->release(sk); 3367 memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv)); 3368 icsk->icsk_ca_initialized = 0; 3369 tcp_set_ca_state(sk, TCP_CA_Open); 3370 tp->is_sack_reneg = 0; 3371 tcp_clear_retrans(tp); 3372 tp->total_retrans = 0; 3373 inet_csk_delack_init(sk); 3374 /* Initialize rcv_mss to TCP_MIN_MSS to avoid division by 0 3375 * issue in __tcp_select_window() 3376 */ 3377 icsk->icsk_ack.rcv_mss = TCP_MIN_MSS; 3378 memset(&tp->rx_opt, 0, sizeof(tp->rx_opt)); 3379 __sk_dst_reset(sk); 3380 dst_release(unrcu_pointer(xchg(&sk->sk_rx_dst, NULL))); 3381 tcp_saved_syn_free(tp); 3382 tp->compressed_ack = 0; 3383 tp->segs_in = 0; 3384 tp->segs_out = 0; 3385 tp->bytes_sent = 0; 3386 tp->bytes_acked = 0; 3387 tp->bytes_received = 0; 3388 tp->bytes_retrans = 0; 3389 tp->data_segs_in = 0; 3390 tp->data_segs_out = 0; 3391 tp->duplicate_sack[0].start_seq = 0; 3392 tp->duplicate_sack[0].end_seq = 0; 3393 tp->dsack_dups = 0; 3394 tp->reord_seen = 0; 3395 tp->retrans_out = 0; 3396 tp->sacked_out = 0; 3397 tp->tlp_high_seq = 0; 3398 tp->last_oow_ack_time = 0; 3399 tp->plb_rehash = 0; 3400 /* There's a bubble in the pipe until at least the first ACK. */ 3401 tp->app_limited = ~0U; 3402 tp->rate_app_limited = 1; 3403 tp->rack.mstamp = 0; 3404 tp->rack.advanced = 0; 3405 tp->rack.reo_wnd_steps = 1; 3406 tp->rack.last_delivered = 0; 3407 tp->rack.reo_wnd_persist = 0; 3408 tp->rack.dsack_seen = 0; 3409 tp->syn_data_acked = 0; 3410 tp->rx_opt.saw_tstamp = 0; 3411 tp->rx_opt.dsack = 0; 3412 tp->rx_opt.num_sacks = 0; 3413 tp->rcv_ooopack = 0; 3414 3415 3416 /* Clean up fastopen related fields */ 3417 tcp_free_fastopen_req(tp); 3418 inet_clear_bit(DEFER_CONNECT, sk); 3419 tp->fastopen_client_fail = 0; 3420 3421 WARN_ON(inet->inet_num && !icsk->icsk_bind_hash); 3422 3423 if (sk->sk_frag.page) { 3424 put_page(sk->sk_frag.page); 3425 sk->sk_frag.page = NULL; 3426 sk->sk_frag.offset = 0; 3427 } 3428 sk_error_report(sk); 3429 return 0; 3430 } 3431 EXPORT_SYMBOL(tcp_disconnect); 3432 3433 static inline bool tcp_can_repair_sock(const struct sock *sk) 3434 { 3435 return sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) && 3436 (sk->sk_state != TCP_LISTEN); 3437 } 3438 3439 static int tcp_repair_set_window(struct tcp_sock *tp, sockptr_t optbuf, int len) 3440 { 3441 struct tcp_repair_window opt; 3442 3443 if (!tp->repair) 3444 return -EPERM; 3445 3446 if (len != sizeof(opt)) 3447 return -EINVAL; 3448 3449 if (copy_from_sockptr(&opt, optbuf, sizeof(opt))) 3450 return -EFAULT; 3451 3452 if (opt.max_window < opt.snd_wnd) 3453 return -EINVAL; 3454 3455 if (after(opt.snd_wl1, tp->rcv_nxt + opt.rcv_wnd)) 3456 return -EINVAL; 3457 3458 if (after(opt.rcv_wup, tp->rcv_nxt)) 3459 return -EINVAL; 3460 3461 tp->snd_wl1 = opt.snd_wl1; 3462 tp->snd_wnd = opt.snd_wnd; 3463 tp->max_window = opt.max_window; 3464 3465 tp->rcv_wnd = opt.rcv_wnd; 3466 tp->rcv_wup = opt.rcv_wup; 3467 3468 return 0; 3469 } 3470 3471 static int tcp_repair_options_est(struct sock *sk, sockptr_t optbuf, 3472 unsigned int len) 3473 { 3474 struct tcp_sock *tp = tcp_sk(sk); 3475 struct tcp_repair_opt opt; 3476 size_t offset = 0; 3477 3478 while (len >= sizeof(opt)) { 3479 if (copy_from_sockptr_offset(&opt, optbuf, offset, sizeof(opt))) 3480 return -EFAULT; 3481 3482 offset += sizeof(opt); 3483 len -= sizeof(opt); 3484 3485 switch (opt.opt_code) { 3486 case TCPOPT_MSS: 3487 tp->rx_opt.mss_clamp = opt.opt_val; 3488 tcp_mtup_init(sk); 3489 break; 3490 case TCPOPT_WINDOW: 3491 { 3492 u16 snd_wscale = opt.opt_val & 0xFFFF; 3493 u16 rcv_wscale = opt.opt_val >> 16; 3494 3495 if (snd_wscale > TCP_MAX_WSCALE || rcv_wscale > TCP_MAX_WSCALE) 3496 return -EFBIG; 3497 3498 tp->rx_opt.snd_wscale = snd_wscale; 3499 tp->rx_opt.rcv_wscale = rcv_wscale; 3500 tp->rx_opt.wscale_ok = 1; 3501 } 3502 break; 3503 case TCPOPT_SACK_PERM: 3504 if (opt.opt_val != 0) 3505 return -EINVAL; 3506 3507 tp->rx_opt.sack_ok |= TCP_SACK_SEEN; 3508 break; 3509 case TCPOPT_TIMESTAMP: 3510 if (opt.opt_val != 0) 3511 return -EINVAL; 3512 3513 tp->rx_opt.tstamp_ok = 1; 3514 break; 3515 } 3516 } 3517 3518 return 0; 3519 } 3520 3521 DEFINE_STATIC_KEY_FALSE(tcp_tx_delay_enabled); 3522 EXPORT_IPV6_MOD(tcp_tx_delay_enabled); 3523 3524 static void tcp_enable_tx_delay(void) 3525 { 3526 if (!static_branch_unlikely(&tcp_tx_delay_enabled)) { 3527 static int __tcp_tx_delay_enabled = 0; 3528 3529 if (cmpxchg(&__tcp_tx_delay_enabled, 0, 1) == 0) { 3530 static_branch_enable(&tcp_tx_delay_enabled); 3531 pr_info("TCP_TX_DELAY enabled\n"); 3532 } 3533 } 3534 } 3535 3536 /* When set indicates to always queue non-full frames. Later the user clears 3537 * this option and we transmit any pending partial frames in the queue. This is 3538 * meant to be used alongside sendfile() to get properly filled frames when the 3539 * user (for example) must write out headers with a write() call first and then 3540 * use sendfile to send out the data parts. 3541 * 3542 * TCP_CORK can be set together with TCP_NODELAY and it is stronger than 3543 * TCP_NODELAY. 3544 */ 3545 void __tcp_sock_set_cork(struct sock *sk, bool on) 3546 { 3547 struct tcp_sock *tp = tcp_sk(sk); 3548 3549 if (on) { 3550 tp->nonagle |= TCP_NAGLE_CORK; 3551 } else { 3552 tp->nonagle &= ~TCP_NAGLE_CORK; 3553 if (tp->nonagle & TCP_NAGLE_OFF) 3554 tp->nonagle |= TCP_NAGLE_PUSH; 3555 tcp_push_pending_frames(sk); 3556 } 3557 } 3558 3559 void tcp_sock_set_cork(struct sock *sk, bool on) 3560 { 3561 lock_sock(sk); 3562 __tcp_sock_set_cork(sk, on); 3563 release_sock(sk); 3564 } 3565 EXPORT_SYMBOL(tcp_sock_set_cork); 3566 3567 /* TCP_NODELAY is weaker than TCP_CORK, so that this option on corked socket is 3568 * remembered, but it is not activated until cork is cleared. 3569 * 3570 * However, when TCP_NODELAY is set we make an explicit push, which overrides 3571 * even TCP_CORK for currently queued segments. 3572 */ 3573 void __tcp_sock_set_nodelay(struct sock *sk, bool on) 3574 { 3575 if (on) { 3576 tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH; 3577 tcp_push_pending_frames(sk); 3578 } else { 3579 tcp_sk(sk)->nonagle &= ~TCP_NAGLE_OFF; 3580 } 3581 } 3582 3583 void tcp_sock_set_nodelay(struct sock *sk) 3584 { 3585 lock_sock(sk); 3586 __tcp_sock_set_nodelay(sk, true); 3587 release_sock(sk); 3588 } 3589 EXPORT_SYMBOL(tcp_sock_set_nodelay); 3590 3591 static void __tcp_sock_set_quickack(struct sock *sk, int val) 3592 { 3593 if (!val) { 3594 inet_csk_enter_pingpong_mode(sk); 3595 return; 3596 } 3597 3598 inet_csk_exit_pingpong_mode(sk); 3599 if ((1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) && 3600 inet_csk_ack_scheduled(sk)) { 3601 inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_PUSHED; 3602 tcp_cleanup_rbuf(sk, 1); 3603 if (!(val & 1)) 3604 inet_csk_enter_pingpong_mode(sk); 3605 } 3606 } 3607 3608 void tcp_sock_set_quickack(struct sock *sk, int val) 3609 { 3610 lock_sock(sk); 3611 __tcp_sock_set_quickack(sk, val); 3612 release_sock(sk); 3613 } 3614 EXPORT_SYMBOL(tcp_sock_set_quickack); 3615 3616 int tcp_sock_set_syncnt(struct sock *sk, int val) 3617 { 3618 if (val < 1 || val > MAX_TCP_SYNCNT) 3619 return -EINVAL; 3620 3621 WRITE_ONCE(inet_csk(sk)->icsk_syn_retries, val); 3622 return 0; 3623 } 3624 EXPORT_SYMBOL(tcp_sock_set_syncnt); 3625 3626 int tcp_sock_set_user_timeout(struct sock *sk, int val) 3627 { 3628 /* Cap the max time in ms TCP will retry or probe the window 3629 * before giving up and aborting (ETIMEDOUT) a connection. 3630 */ 3631 if (val < 0) 3632 return -EINVAL; 3633 3634 WRITE_ONCE(inet_csk(sk)->icsk_user_timeout, val); 3635 return 0; 3636 } 3637 EXPORT_SYMBOL(tcp_sock_set_user_timeout); 3638 3639 int tcp_sock_set_keepidle_locked(struct sock *sk, int val) 3640 { 3641 struct tcp_sock *tp = tcp_sk(sk); 3642 3643 if (val < 1 || val > MAX_TCP_KEEPIDLE) 3644 return -EINVAL; 3645 3646 /* Paired with WRITE_ONCE() in keepalive_time_when() */ 3647 WRITE_ONCE(tp->keepalive_time, val * HZ); 3648 if (sock_flag(sk, SOCK_KEEPOPEN) && 3649 !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) { 3650 u32 elapsed = keepalive_time_elapsed(tp); 3651 3652 if (tp->keepalive_time > elapsed) 3653 elapsed = tp->keepalive_time - elapsed; 3654 else 3655 elapsed = 0; 3656 tcp_reset_keepalive_timer(sk, elapsed); 3657 } 3658 3659 return 0; 3660 } 3661 3662 int tcp_sock_set_keepidle(struct sock *sk, int val) 3663 { 3664 int err; 3665 3666 lock_sock(sk); 3667 err = tcp_sock_set_keepidle_locked(sk, val); 3668 release_sock(sk); 3669 return err; 3670 } 3671 EXPORT_SYMBOL(tcp_sock_set_keepidle); 3672 3673 int tcp_sock_set_keepintvl(struct sock *sk, int val) 3674 { 3675 if (val < 1 || val > MAX_TCP_KEEPINTVL) 3676 return -EINVAL; 3677 3678 WRITE_ONCE(tcp_sk(sk)->keepalive_intvl, val * HZ); 3679 return 0; 3680 } 3681 EXPORT_SYMBOL(tcp_sock_set_keepintvl); 3682 3683 int tcp_sock_set_keepcnt(struct sock *sk, int val) 3684 { 3685 if (val < 1 || val > MAX_TCP_KEEPCNT) 3686 return -EINVAL; 3687 3688 /* Paired with READ_ONCE() in keepalive_probes() */ 3689 WRITE_ONCE(tcp_sk(sk)->keepalive_probes, val); 3690 return 0; 3691 } 3692 EXPORT_SYMBOL(tcp_sock_set_keepcnt); 3693 3694 int tcp_set_window_clamp(struct sock *sk, int val) 3695 { 3696 u32 old_window_clamp, new_window_clamp, new_rcv_ssthresh; 3697 struct tcp_sock *tp = tcp_sk(sk); 3698 3699 if (!val) { 3700 if (sk->sk_state != TCP_CLOSE) 3701 return -EINVAL; 3702 WRITE_ONCE(tp->window_clamp, 0); 3703 return 0; 3704 } 3705 3706 old_window_clamp = tp->window_clamp; 3707 new_window_clamp = max_t(int, SOCK_MIN_RCVBUF / 2, val); 3708 3709 if (new_window_clamp == old_window_clamp) 3710 return 0; 3711 3712 WRITE_ONCE(tp->window_clamp, new_window_clamp); 3713 3714 /* Need to apply the reserved mem provisioning only 3715 * when shrinking the window clamp. 3716 */ 3717 if (new_window_clamp < old_window_clamp) { 3718 __tcp_adjust_rcv_ssthresh(sk, new_window_clamp); 3719 } else { 3720 new_rcv_ssthresh = min(tp->rcv_wnd, new_window_clamp); 3721 tp->rcv_ssthresh = max(new_rcv_ssthresh, tp->rcv_ssthresh); 3722 } 3723 return 0; 3724 } 3725 3726 /* 3727 * Socket option code for TCP. 3728 */ 3729 int do_tcp_setsockopt(struct sock *sk, int level, int optname, 3730 sockptr_t optval, unsigned int optlen) 3731 { 3732 struct tcp_sock *tp = tcp_sk(sk); 3733 struct inet_connection_sock *icsk = inet_csk(sk); 3734 struct net *net = sock_net(sk); 3735 int val; 3736 int err = 0; 3737 3738 /* These are data/string values, all the others are ints */ 3739 switch (optname) { 3740 case TCP_CONGESTION: { 3741 char name[TCP_CA_NAME_MAX]; 3742 3743 if (optlen < 1) 3744 return -EINVAL; 3745 3746 val = strncpy_from_sockptr(name, optval, 3747 min_t(long, TCP_CA_NAME_MAX-1, optlen)); 3748 if (val < 0) 3749 return -EFAULT; 3750 name[val] = 0; 3751 3752 sockopt_lock_sock(sk); 3753 err = tcp_set_congestion_control(sk, name, !has_current_bpf_ctx(), 3754 sockopt_ns_capable(sock_net(sk)->user_ns, 3755 CAP_NET_ADMIN)); 3756 sockopt_release_sock(sk); 3757 return err; 3758 } 3759 case TCP_ULP: { 3760 char name[TCP_ULP_NAME_MAX]; 3761 3762 if (optlen < 1) 3763 return -EINVAL; 3764 3765 val = strncpy_from_sockptr(name, optval, 3766 min_t(long, TCP_ULP_NAME_MAX - 1, 3767 optlen)); 3768 if (val < 0) 3769 return -EFAULT; 3770 name[val] = 0; 3771 3772 sockopt_lock_sock(sk); 3773 err = tcp_set_ulp(sk, name); 3774 sockopt_release_sock(sk); 3775 return err; 3776 } 3777 case TCP_FASTOPEN_KEY: { 3778 __u8 key[TCP_FASTOPEN_KEY_BUF_LENGTH]; 3779 __u8 *backup_key = NULL; 3780 3781 /* Allow a backup key as well to facilitate key rotation 3782 * First key is the active one. 3783 */ 3784 if (optlen != TCP_FASTOPEN_KEY_LENGTH && 3785 optlen != TCP_FASTOPEN_KEY_BUF_LENGTH) 3786 return -EINVAL; 3787 3788 if (copy_from_sockptr(key, optval, optlen)) 3789 return -EFAULT; 3790 3791 if (optlen == TCP_FASTOPEN_KEY_BUF_LENGTH) 3792 backup_key = key + TCP_FASTOPEN_KEY_LENGTH; 3793 3794 return tcp_fastopen_reset_cipher(net, sk, key, backup_key); 3795 } 3796 default: 3797 /* fallthru */ 3798 break; 3799 } 3800 3801 if (optlen < sizeof(int)) 3802 return -EINVAL; 3803 3804 if (copy_from_sockptr(&val, optval, sizeof(val))) 3805 return -EFAULT; 3806 3807 /* Handle options that can be set without locking the socket. */ 3808 switch (optname) { 3809 case TCP_SYNCNT: 3810 return tcp_sock_set_syncnt(sk, val); 3811 case TCP_USER_TIMEOUT: 3812 return tcp_sock_set_user_timeout(sk, val); 3813 case TCP_KEEPINTVL: 3814 return tcp_sock_set_keepintvl(sk, val); 3815 case TCP_KEEPCNT: 3816 return tcp_sock_set_keepcnt(sk, val); 3817 case TCP_LINGER2: 3818 if (val < 0) 3819 WRITE_ONCE(tp->linger2, -1); 3820 else if (val > TCP_FIN_TIMEOUT_MAX / HZ) 3821 WRITE_ONCE(tp->linger2, TCP_FIN_TIMEOUT_MAX); 3822 else 3823 WRITE_ONCE(tp->linger2, val * HZ); 3824 return 0; 3825 case TCP_DEFER_ACCEPT: 3826 /* Translate value in seconds to number of retransmits */ 3827 WRITE_ONCE(icsk->icsk_accept_queue.rskq_defer_accept, 3828 secs_to_retrans(val, TCP_TIMEOUT_INIT / HZ, 3829 TCP_RTO_MAX / HZ)); 3830 return 0; 3831 case TCP_RTO_MAX_MS: 3832 if (val < MSEC_PER_SEC || val > TCP_RTO_MAX_SEC * MSEC_PER_SEC) 3833 return -EINVAL; 3834 WRITE_ONCE(inet_csk(sk)->icsk_rto_max, msecs_to_jiffies(val)); 3835 return 0; 3836 case TCP_RTO_MIN_US: { 3837 int rto_min = usecs_to_jiffies(val); 3838 3839 if (rto_min > TCP_RTO_MIN || rto_min < TCP_TIMEOUT_MIN) 3840 return -EINVAL; 3841 WRITE_ONCE(inet_csk(sk)->icsk_rto_min, rto_min); 3842 return 0; 3843 } 3844 case TCP_DELACK_MAX_US: { 3845 int delack_max = usecs_to_jiffies(val); 3846 3847 if (delack_max > TCP_DELACK_MAX || delack_max < TCP_TIMEOUT_MIN) 3848 return -EINVAL; 3849 WRITE_ONCE(inet_csk(sk)->icsk_delack_max, delack_max); 3850 return 0; 3851 } 3852 } 3853 3854 sockopt_lock_sock(sk); 3855 3856 switch (optname) { 3857 case TCP_MAXSEG: 3858 /* Values greater than interface MTU won't take effect. However 3859 * at the point when this call is done we typically don't yet 3860 * know which interface is going to be used 3861 */ 3862 if (val && (val < TCP_MIN_MSS || val > MAX_TCP_WINDOW)) { 3863 err = -EINVAL; 3864 break; 3865 } 3866 tp->rx_opt.user_mss = val; 3867 break; 3868 3869 case TCP_NODELAY: 3870 __tcp_sock_set_nodelay(sk, val); 3871 break; 3872 3873 case TCP_THIN_LINEAR_TIMEOUTS: 3874 if (val < 0 || val > 1) 3875 err = -EINVAL; 3876 else 3877 tp->thin_lto = val; 3878 break; 3879 3880 case TCP_THIN_DUPACK: 3881 if (val < 0 || val > 1) 3882 err = -EINVAL; 3883 break; 3884 3885 case TCP_REPAIR: 3886 if (!tcp_can_repair_sock(sk)) 3887 err = -EPERM; 3888 else if (val == TCP_REPAIR_ON) { 3889 tp->repair = 1; 3890 sk->sk_reuse = SK_FORCE_REUSE; 3891 tp->repair_queue = TCP_NO_QUEUE; 3892 } else if (val == TCP_REPAIR_OFF) { 3893 tp->repair = 0; 3894 sk->sk_reuse = SK_NO_REUSE; 3895 tcp_send_window_probe(sk); 3896 } else if (val == TCP_REPAIR_OFF_NO_WP) { 3897 tp->repair = 0; 3898 sk->sk_reuse = SK_NO_REUSE; 3899 } else 3900 err = -EINVAL; 3901 3902 break; 3903 3904 case TCP_REPAIR_QUEUE: 3905 if (!tp->repair) 3906 err = -EPERM; 3907 else if ((unsigned int)val < TCP_QUEUES_NR) 3908 tp->repair_queue = val; 3909 else 3910 err = -EINVAL; 3911 break; 3912 3913 case TCP_QUEUE_SEQ: 3914 if (sk->sk_state != TCP_CLOSE) { 3915 err = -EPERM; 3916 } else if (tp->repair_queue == TCP_SEND_QUEUE) { 3917 if (!tcp_rtx_queue_empty(sk)) 3918 err = -EPERM; 3919 else 3920 WRITE_ONCE(tp->write_seq, val); 3921 } else if (tp->repair_queue == TCP_RECV_QUEUE) { 3922 if (tp->rcv_nxt != tp->copied_seq) { 3923 err = -EPERM; 3924 } else { 3925 WRITE_ONCE(tp->rcv_nxt, val); 3926 WRITE_ONCE(tp->copied_seq, val); 3927 } 3928 } else { 3929 err = -EINVAL; 3930 } 3931 break; 3932 3933 case TCP_REPAIR_OPTIONS: 3934 if (!tp->repair) 3935 err = -EINVAL; 3936 else if (sk->sk_state == TCP_ESTABLISHED && !tp->bytes_sent) 3937 err = tcp_repair_options_est(sk, optval, optlen); 3938 else 3939 err = -EPERM; 3940 break; 3941 3942 case TCP_CORK: 3943 __tcp_sock_set_cork(sk, val); 3944 break; 3945 3946 case TCP_KEEPIDLE: 3947 err = tcp_sock_set_keepidle_locked(sk, val); 3948 break; 3949 case TCP_SAVE_SYN: 3950 /* 0: disable, 1: enable, 2: start from ether_header */ 3951 if (val < 0 || val > 2) 3952 err = -EINVAL; 3953 else 3954 tp->save_syn = val; 3955 break; 3956 3957 case TCP_WINDOW_CLAMP: 3958 err = tcp_set_window_clamp(sk, val); 3959 break; 3960 3961 case TCP_QUICKACK: 3962 __tcp_sock_set_quickack(sk, val); 3963 break; 3964 3965 case TCP_AO_REPAIR: 3966 if (!tcp_can_repair_sock(sk)) { 3967 err = -EPERM; 3968 break; 3969 } 3970 err = tcp_ao_set_repair(sk, optval, optlen); 3971 break; 3972 #ifdef CONFIG_TCP_AO 3973 case TCP_AO_ADD_KEY: 3974 case TCP_AO_DEL_KEY: 3975 case TCP_AO_INFO: { 3976 /* If this is the first TCP-AO setsockopt() on the socket, 3977 * sk_state has to be LISTEN or CLOSE. Allow TCP_REPAIR 3978 * in any state. 3979 */ 3980 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) 3981 goto ao_parse; 3982 if (rcu_dereference_protected(tcp_sk(sk)->ao_info, 3983 lockdep_sock_is_held(sk))) 3984 goto ao_parse; 3985 if (tp->repair) 3986 goto ao_parse; 3987 err = -EISCONN; 3988 break; 3989 ao_parse: 3990 err = tp->af_specific->ao_parse(sk, optname, optval, optlen); 3991 break; 3992 } 3993 #endif 3994 #ifdef CONFIG_TCP_MD5SIG 3995 case TCP_MD5SIG: 3996 case TCP_MD5SIG_EXT: 3997 err = tp->af_specific->md5_parse(sk, optname, optval, optlen); 3998 break; 3999 #endif 4000 case TCP_FASTOPEN: 4001 if (val >= 0 && ((1 << sk->sk_state) & (TCPF_CLOSE | 4002 TCPF_LISTEN))) { 4003 tcp_fastopen_init_key_once(net); 4004 4005 fastopen_queue_tune(sk, val); 4006 } else { 4007 err = -EINVAL; 4008 } 4009 break; 4010 case TCP_FASTOPEN_CONNECT: 4011 if (val > 1 || val < 0) { 4012 err = -EINVAL; 4013 } else if (READ_ONCE(net->ipv4.sysctl_tcp_fastopen) & 4014 TFO_CLIENT_ENABLE) { 4015 if (sk->sk_state == TCP_CLOSE) 4016 tp->fastopen_connect = val; 4017 else 4018 err = -EINVAL; 4019 } else { 4020 err = -EOPNOTSUPP; 4021 } 4022 break; 4023 case TCP_FASTOPEN_NO_COOKIE: 4024 if (val > 1 || val < 0) 4025 err = -EINVAL; 4026 else if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) 4027 err = -EINVAL; 4028 else 4029 tp->fastopen_no_cookie = val; 4030 break; 4031 case TCP_TIMESTAMP: 4032 if (!tp->repair) { 4033 err = -EPERM; 4034 break; 4035 } 4036 /* val is an opaque field, 4037 * and low order bit contains usec_ts enable bit. 4038 * Its a best effort, and we do not care if user makes an error. 4039 */ 4040 tp->tcp_usec_ts = val & 1; 4041 WRITE_ONCE(tp->tsoffset, val - tcp_clock_ts(tp->tcp_usec_ts)); 4042 break; 4043 case TCP_REPAIR_WINDOW: 4044 err = tcp_repair_set_window(tp, optval, optlen); 4045 break; 4046 case TCP_NOTSENT_LOWAT: 4047 WRITE_ONCE(tp->notsent_lowat, val); 4048 sk->sk_write_space(sk); 4049 break; 4050 case TCP_INQ: 4051 if (val > 1 || val < 0) 4052 err = -EINVAL; 4053 else 4054 tp->recvmsg_inq = val; 4055 break; 4056 case TCP_TX_DELAY: 4057 if (val) 4058 tcp_enable_tx_delay(); 4059 WRITE_ONCE(tp->tcp_tx_delay, val); 4060 break; 4061 default: 4062 err = -ENOPROTOOPT; 4063 break; 4064 } 4065 4066 sockopt_release_sock(sk); 4067 return err; 4068 } 4069 4070 int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, 4071 unsigned int optlen) 4072 { 4073 const struct inet_connection_sock *icsk = inet_csk(sk); 4074 4075 if (level != SOL_TCP) 4076 /* Paired with WRITE_ONCE() in do_ipv6_setsockopt() and tcp_v6_connect() */ 4077 return READ_ONCE(icsk->icsk_af_ops)->setsockopt(sk, level, optname, 4078 optval, optlen); 4079 return do_tcp_setsockopt(sk, level, optname, optval, optlen); 4080 } 4081 EXPORT_IPV6_MOD(tcp_setsockopt); 4082 4083 static void tcp_get_info_chrono_stats(const struct tcp_sock *tp, 4084 struct tcp_info *info) 4085 { 4086 u64 stats[__TCP_CHRONO_MAX], total = 0; 4087 enum tcp_chrono i; 4088 4089 for (i = TCP_CHRONO_BUSY; i < __TCP_CHRONO_MAX; ++i) { 4090 stats[i] = tp->chrono_stat[i - 1]; 4091 if (i == tp->chrono_type) 4092 stats[i] += tcp_jiffies32 - tp->chrono_start; 4093 stats[i] *= USEC_PER_SEC / HZ; 4094 total += stats[i]; 4095 } 4096 4097 info->tcpi_busy_time = total; 4098 info->tcpi_rwnd_limited = stats[TCP_CHRONO_RWND_LIMITED]; 4099 info->tcpi_sndbuf_limited = stats[TCP_CHRONO_SNDBUF_LIMITED]; 4100 } 4101 4102 /* Return information about state of tcp endpoint in API format. */ 4103 void tcp_get_info(struct sock *sk, struct tcp_info *info) 4104 { 4105 const struct tcp_sock *tp = tcp_sk(sk); /* iff sk_type == SOCK_STREAM */ 4106 const struct inet_connection_sock *icsk = inet_csk(sk); 4107 unsigned long rate; 4108 u32 now; 4109 u64 rate64; 4110 bool slow; 4111 4112 memset(info, 0, sizeof(*info)); 4113 if (sk->sk_type != SOCK_STREAM) 4114 return; 4115 4116 info->tcpi_state = inet_sk_state_load(sk); 4117 4118 /* Report meaningful fields for all TCP states, including listeners */ 4119 rate = READ_ONCE(sk->sk_pacing_rate); 4120 rate64 = (rate != ~0UL) ? rate : ~0ULL; 4121 info->tcpi_pacing_rate = rate64; 4122 4123 rate = READ_ONCE(sk->sk_max_pacing_rate); 4124 rate64 = (rate != ~0UL) ? rate : ~0ULL; 4125 info->tcpi_max_pacing_rate = rate64; 4126 4127 info->tcpi_reordering = tp->reordering; 4128 info->tcpi_snd_cwnd = tcp_snd_cwnd(tp); 4129 4130 if (info->tcpi_state == TCP_LISTEN) { 4131 /* listeners aliased fields : 4132 * tcpi_unacked -> Number of children ready for accept() 4133 * tcpi_sacked -> max backlog 4134 */ 4135 info->tcpi_unacked = READ_ONCE(sk->sk_ack_backlog); 4136 info->tcpi_sacked = READ_ONCE(sk->sk_max_ack_backlog); 4137 return; 4138 } 4139 4140 slow = lock_sock_fast(sk); 4141 4142 info->tcpi_ca_state = icsk->icsk_ca_state; 4143 info->tcpi_retransmits = icsk->icsk_retransmits; 4144 info->tcpi_probes = icsk->icsk_probes_out; 4145 info->tcpi_backoff = icsk->icsk_backoff; 4146 4147 if (tp->rx_opt.tstamp_ok) 4148 info->tcpi_options |= TCPI_OPT_TIMESTAMPS; 4149 if (tcp_is_sack(tp)) 4150 info->tcpi_options |= TCPI_OPT_SACK; 4151 if (tp->rx_opt.wscale_ok) { 4152 info->tcpi_options |= TCPI_OPT_WSCALE; 4153 info->tcpi_snd_wscale = tp->rx_opt.snd_wscale; 4154 info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale; 4155 } 4156 4157 if (tcp_ecn_mode_any(tp)) 4158 info->tcpi_options |= TCPI_OPT_ECN; 4159 if (tp->ecn_flags & TCP_ECN_SEEN) 4160 info->tcpi_options |= TCPI_OPT_ECN_SEEN; 4161 if (tp->syn_data_acked) 4162 info->tcpi_options |= TCPI_OPT_SYN_DATA; 4163 if (tp->tcp_usec_ts) 4164 info->tcpi_options |= TCPI_OPT_USEC_TS; 4165 4166 info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto); 4167 info->tcpi_ato = jiffies_to_usecs(min_t(u32, icsk->icsk_ack.ato, 4168 tcp_delack_max(sk))); 4169 info->tcpi_snd_mss = tp->mss_cache; 4170 info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss; 4171 4172 info->tcpi_unacked = tp->packets_out; 4173 info->tcpi_sacked = tp->sacked_out; 4174 4175 info->tcpi_lost = tp->lost_out; 4176 info->tcpi_retrans = tp->retrans_out; 4177 4178 now = tcp_jiffies32; 4179 info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime); 4180 info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime); 4181 info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp); 4182 4183 info->tcpi_pmtu = icsk->icsk_pmtu_cookie; 4184 info->tcpi_rcv_ssthresh = tp->rcv_ssthresh; 4185 info->tcpi_rtt = tp->srtt_us >> 3; 4186 info->tcpi_rttvar = tp->mdev_us >> 2; 4187 info->tcpi_snd_ssthresh = tp->snd_ssthresh; 4188 info->tcpi_advmss = tp->advmss; 4189 4190 info->tcpi_rcv_rtt = tp->rcv_rtt_est.rtt_us >> 3; 4191 info->tcpi_rcv_space = tp->rcvq_space.space; 4192 4193 info->tcpi_total_retrans = tp->total_retrans; 4194 4195 info->tcpi_bytes_acked = tp->bytes_acked; 4196 info->tcpi_bytes_received = tp->bytes_received; 4197 info->tcpi_notsent_bytes = max_t(int, 0, tp->write_seq - tp->snd_nxt); 4198 tcp_get_info_chrono_stats(tp, info); 4199 4200 info->tcpi_segs_out = tp->segs_out; 4201 4202 /* segs_in and data_segs_in can be updated from tcp_segs_in() from BH */ 4203 info->tcpi_segs_in = READ_ONCE(tp->segs_in); 4204 info->tcpi_data_segs_in = READ_ONCE(tp->data_segs_in); 4205 4206 info->tcpi_min_rtt = tcp_min_rtt(tp); 4207 info->tcpi_data_segs_out = tp->data_segs_out; 4208 4209 info->tcpi_delivery_rate_app_limited = tp->rate_app_limited ? 1 : 0; 4210 rate64 = tcp_compute_delivery_rate(tp); 4211 if (rate64) 4212 info->tcpi_delivery_rate = rate64; 4213 info->tcpi_delivered = tp->delivered; 4214 info->tcpi_delivered_ce = tp->delivered_ce; 4215 info->tcpi_bytes_sent = tp->bytes_sent; 4216 info->tcpi_bytes_retrans = tp->bytes_retrans; 4217 info->tcpi_dsack_dups = tp->dsack_dups; 4218 info->tcpi_reord_seen = tp->reord_seen; 4219 info->tcpi_rcv_ooopack = tp->rcv_ooopack; 4220 info->tcpi_snd_wnd = tp->snd_wnd; 4221 info->tcpi_rcv_wnd = tp->rcv_wnd; 4222 info->tcpi_rehash = tp->plb_rehash + tp->timeout_rehash; 4223 info->tcpi_fastopen_client_fail = tp->fastopen_client_fail; 4224 4225 info->tcpi_total_rto = tp->total_rto; 4226 info->tcpi_total_rto_recoveries = tp->total_rto_recoveries; 4227 info->tcpi_total_rto_time = tp->total_rto_time; 4228 if (tp->rto_stamp) 4229 info->tcpi_total_rto_time += tcp_clock_ms() - tp->rto_stamp; 4230 4231 unlock_sock_fast(sk, slow); 4232 } 4233 EXPORT_SYMBOL_GPL(tcp_get_info); 4234 4235 static size_t tcp_opt_stats_get_size(void) 4236 { 4237 return 4238 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BUSY */ 4239 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_RWND_LIMITED */ 4240 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_SNDBUF_LIMITED */ 4241 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_DATA_SEGS_OUT */ 4242 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_TOTAL_RETRANS */ 4243 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_PACING_RATE */ 4244 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_DELIVERY_RATE */ 4245 nla_total_size(sizeof(u32)) + /* TCP_NLA_SND_CWND */ 4246 nla_total_size(sizeof(u32)) + /* TCP_NLA_REORDERING */ 4247 nla_total_size(sizeof(u32)) + /* TCP_NLA_MIN_RTT */ 4248 nla_total_size(sizeof(u8)) + /* TCP_NLA_RECUR_RETRANS */ 4249 nla_total_size(sizeof(u8)) + /* TCP_NLA_DELIVERY_RATE_APP_LMT */ 4250 nla_total_size(sizeof(u32)) + /* TCP_NLA_SNDQ_SIZE */ 4251 nla_total_size(sizeof(u8)) + /* TCP_NLA_CA_STATE */ 4252 nla_total_size(sizeof(u32)) + /* TCP_NLA_SND_SSTHRESH */ 4253 nla_total_size(sizeof(u32)) + /* TCP_NLA_DELIVERED */ 4254 nla_total_size(sizeof(u32)) + /* TCP_NLA_DELIVERED_CE */ 4255 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BYTES_SENT */ 4256 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BYTES_RETRANS */ 4257 nla_total_size(sizeof(u32)) + /* TCP_NLA_DSACK_DUPS */ 4258 nla_total_size(sizeof(u32)) + /* TCP_NLA_REORD_SEEN */ 4259 nla_total_size(sizeof(u32)) + /* TCP_NLA_SRTT */ 4260 nla_total_size(sizeof(u16)) + /* TCP_NLA_TIMEOUT_REHASH */ 4261 nla_total_size(sizeof(u32)) + /* TCP_NLA_BYTES_NOTSENT */ 4262 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_EDT */ 4263 nla_total_size(sizeof(u8)) + /* TCP_NLA_TTL */ 4264 nla_total_size(sizeof(u32)) + /* TCP_NLA_REHASH */ 4265 0; 4266 } 4267 4268 /* Returns TTL or hop limit of an incoming packet from skb. */ 4269 static u8 tcp_skb_ttl_or_hop_limit(const struct sk_buff *skb) 4270 { 4271 if (skb->protocol == htons(ETH_P_IP)) 4272 return ip_hdr(skb)->ttl; 4273 else if (skb->protocol == htons(ETH_P_IPV6)) 4274 return ipv6_hdr(skb)->hop_limit; 4275 else 4276 return 0; 4277 } 4278 4279 struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk, 4280 const struct sk_buff *orig_skb, 4281 const struct sk_buff *ack_skb) 4282 { 4283 const struct tcp_sock *tp = tcp_sk(sk); 4284 struct sk_buff *stats; 4285 struct tcp_info info; 4286 unsigned long rate; 4287 u64 rate64; 4288 4289 stats = alloc_skb(tcp_opt_stats_get_size(), GFP_ATOMIC); 4290 if (!stats) 4291 return NULL; 4292 4293 tcp_get_info_chrono_stats(tp, &info); 4294 nla_put_u64_64bit(stats, TCP_NLA_BUSY, 4295 info.tcpi_busy_time, TCP_NLA_PAD); 4296 nla_put_u64_64bit(stats, TCP_NLA_RWND_LIMITED, 4297 info.tcpi_rwnd_limited, TCP_NLA_PAD); 4298 nla_put_u64_64bit(stats, TCP_NLA_SNDBUF_LIMITED, 4299 info.tcpi_sndbuf_limited, TCP_NLA_PAD); 4300 nla_put_u64_64bit(stats, TCP_NLA_DATA_SEGS_OUT, 4301 tp->data_segs_out, TCP_NLA_PAD); 4302 nla_put_u64_64bit(stats, TCP_NLA_TOTAL_RETRANS, 4303 tp->total_retrans, TCP_NLA_PAD); 4304 4305 rate = READ_ONCE(sk->sk_pacing_rate); 4306 rate64 = (rate != ~0UL) ? rate : ~0ULL; 4307 nla_put_u64_64bit(stats, TCP_NLA_PACING_RATE, rate64, TCP_NLA_PAD); 4308 4309 rate64 = tcp_compute_delivery_rate(tp); 4310 nla_put_u64_64bit(stats, TCP_NLA_DELIVERY_RATE, rate64, TCP_NLA_PAD); 4311 4312 nla_put_u32(stats, TCP_NLA_SND_CWND, tcp_snd_cwnd(tp)); 4313 nla_put_u32(stats, TCP_NLA_REORDERING, tp->reordering); 4314 nla_put_u32(stats, TCP_NLA_MIN_RTT, tcp_min_rtt(tp)); 4315 4316 nla_put_u8(stats, TCP_NLA_RECUR_RETRANS, inet_csk(sk)->icsk_retransmits); 4317 nla_put_u8(stats, TCP_NLA_DELIVERY_RATE_APP_LMT, !!tp->rate_app_limited); 4318 nla_put_u32(stats, TCP_NLA_SND_SSTHRESH, tp->snd_ssthresh); 4319 nla_put_u32(stats, TCP_NLA_DELIVERED, tp->delivered); 4320 nla_put_u32(stats, TCP_NLA_DELIVERED_CE, tp->delivered_ce); 4321 4322 nla_put_u32(stats, TCP_NLA_SNDQ_SIZE, tp->write_seq - tp->snd_una); 4323 nla_put_u8(stats, TCP_NLA_CA_STATE, inet_csk(sk)->icsk_ca_state); 4324 4325 nla_put_u64_64bit(stats, TCP_NLA_BYTES_SENT, tp->bytes_sent, 4326 TCP_NLA_PAD); 4327 nla_put_u64_64bit(stats, TCP_NLA_BYTES_RETRANS, tp->bytes_retrans, 4328 TCP_NLA_PAD); 4329 nla_put_u32(stats, TCP_NLA_DSACK_DUPS, tp->dsack_dups); 4330 nla_put_u32(stats, TCP_NLA_REORD_SEEN, tp->reord_seen); 4331 nla_put_u32(stats, TCP_NLA_SRTT, tp->srtt_us >> 3); 4332 nla_put_u16(stats, TCP_NLA_TIMEOUT_REHASH, tp->timeout_rehash); 4333 nla_put_u32(stats, TCP_NLA_BYTES_NOTSENT, 4334 max_t(int, 0, tp->write_seq - tp->snd_nxt)); 4335 nla_put_u64_64bit(stats, TCP_NLA_EDT, orig_skb->skb_mstamp_ns, 4336 TCP_NLA_PAD); 4337 if (ack_skb) 4338 nla_put_u8(stats, TCP_NLA_TTL, 4339 tcp_skb_ttl_or_hop_limit(ack_skb)); 4340 4341 nla_put_u32(stats, TCP_NLA_REHASH, tp->plb_rehash + tp->timeout_rehash); 4342 return stats; 4343 } 4344 4345 int do_tcp_getsockopt(struct sock *sk, int level, 4346 int optname, sockptr_t optval, sockptr_t optlen) 4347 { 4348 struct inet_connection_sock *icsk = inet_csk(sk); 4349 struct tcp_sock *tp = tcp_sk(sk); 4350 struct net *net = sock_net(sk); 4351 int val, len; 4352 4353 if (copy_from_sockptr(&len, optlen, sizeof(int))) 4354 return -EFAULT; 4355 4356 if (len < 0) 4357 return -EINVAL; 4358 4359 len = min_t(unsigned int, len, sizeof(int)); 4360 4361 switch (optname) { 4362 case TCP_MAXSEG: 4363 val = tp->mss_cache; 4364 if (tp->rx_opt.user_mss && 4365 ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) 4366 val = tp->rx_opt.user_mss; 4367 if (tp->repair) 4368 val = tp->rx_opt.mss_clamp; 4369 break; 4370 case TCP_NODELAY: 4371 val = !!(tp->nonagle&TCP_NAGLE_OFF); 4372 break; 4373 case TCP_CORK: 4374 val = !!(tp->nonagle&TCP_NAGLE_CORK); 4375 break; 4376 case TCP_KEEPIDLE: 4377 val = keepalive_time_when(tp) / HZ; 4378 break; 4379 case TCP_KEEPINTVL: 4380 val = keepalive_intvl_when(tp) / HZ; 4381 break; 4382 case TCP_KEEPCNT: 4383 val = keepalive_probes(tp); 4384 break; 4385 case TCP_SYNCNT: 4386 val = READ_ONCE(icsk->icsk_syn_retries) ? : 4387 READ_ONCE(net->ipv4.sysctl_tcp_syn_retries); 4388 break; 4389 case TCP_LINGER2: 4390 val = READ_ONCE(tp->linger2); 4391 if (val >= 0) 4392 val = (val ? : READ_ONCE(net->ipv4.sysctl_tcp_fin_timeout)) / HZ; 4393 break; 4394 case TCP_DEFER_ACCEPT: 4395 val = READ_ONCE(icsk->icsk_accept_queue.rskq_defer_accept); 4396 val = retrans_to_secs(val, TCP_TIMEOUT_INIT / HZ, 4397 TCP_RTO_MAX / HZ); 4398 break; 4399 case TCP_WINDOW_CLAMP: 4400 val = READ_ONCE(tp->window_clamp); 4401 break; 4402 case TCP_INFO: { 4403 struct tcp_info info; 4404 4405 if (copy_from_sockptr(&len, optlen, sizeof(int))) 4406 return -EFAULT; 4407 4408 tcp_get_info(sk, &info); 4409 4410 len = min_t(unsigned int, len, sizeof(info)); 4411 if (copy_to_sockptr(optlen, &len, sizeof(int))) 4412 return -EFAULT; 4413 if (copy_to_sockptr(optval, &info, len)) 4414 return -EFAULT; 4415 return 0; 4416 } 4417 case TCP_CC_INFO: { 4418 const struct tcp_congestion_ops *ca_ops; 4419 union tcp_cc_info info; 4420 size_t sz = 0; 4421 int attr; 4422 4423 if (copy_from_sockptr(&len, optlen, sizeof(int))) 4424 return -EFAULT; 4425 4426 ca_ops = icsk->icsk_ca_ops; 4427 if (ca_ops && ca_ops->get_info) 4428 sz = ca_ops->get_info(sk, ~0U, &attr, &info); 4429 4430 len = min_t(unsigned int, len, sz); 4431 if (copy_to_sockptr(optlen, &len, sizeof(int))) 4432 return -EFAULT; 4433 if (copy_to_sockptr(optval, &info, len)) 4434 return -EFAULT; 4435 return 0; 4436 } 4437 case TCP_QUICKACK: 4438 val = !inet_csk_in_pingpong_mode(sk); 4439 break; 4440 4441 case TCP_CONGESTION: 4442 if (copy_from_sockptr(&len, optlen, sizeof(int))) 4443 return -EFAULT; 4444 len = min_t(unsigned int, len, TCP_CA_NAME_MAX); 4445 if (copy_to_sockptr(optlen, &len, sizeof(int))) 4446 return -EFAULT; 4447 if (copy_to_sockptr(optval, icsk->icsk_ca_ops->name, len)) 4448 return -EFAULT; 4449 return 0; 4450 4451 case TCP_ULP: 4452 if (copy_from_sockptr(&len, optlen, sizeof(int))) 4453 return -EFAULT; 4454 len = min_t(unsigned int, len, TCP_ULP_NAME_MAX); 4455 if (!icsk->icsk_ulp_ops) { 4456 len = 0; 4457 if (copy_to_sockptr(optlen, &len, sizeof(int))) 4458 return -EFAULT; 4459 return 0; 4460 } 4461 if (copy_to_sockptr(optlen, &len, sizeof(int))) 4462 return -EFAULT; 4463 if (copy_to_sockptr(optval, icsk->icsk_ulp_ops->name, len)) 4464 return -EFAULT; 4465 return 0; 4466 4467 case TCP_FASTOPEN_KEY: { 4468 u64 key[TCP_FASTOPEN_KEY_BUF_LENGTH / sizeof(u64)]; 4469 unsigned int key_len; 4470 4471 if (copy_from_sockptr(&len, optlen, sizeof(int))) 4472 return -EFAULT; 4473 4474 key_len = tcp_fastopen_get_cipher(net, icsk, key) * 4475 TCP_FASTOPEN_KEY_LENGTH; 4476 len = min_t(unsigned int, len, key_len); 4477 if (copy_to_sockptr(optlen, &len, sizeof(int))) 4478 return -EFAULT; 4479 if (copy_to_sockptr(optval, key, len)) 4480 return -EFAULT; 4481 return 0; 4482 } 4483 case TCP_THIN_LINEAR_TIMEOUTS: 4484 val = tp->thin_lto; 4485 break; 4486 4487 case TCP_THIN_DUPACK: 4488 val = 0; 4489 break; 4490 4491 case TCP_REPAIR: 4492 val = tp->repair; 4493 break; 4494 4495 case TCP_REPAIR_QUEUE: 4496 if (tp->repair) 4497 val = tp->repair_queue; 4498 else 4499 return -EINVAL; 4500 break; 4501 4502 case TCP_REPAIR_WINDOW: { 4503 struct tcp_repair_window opt; 4504 4505 if (copy_from_sockptr(&len, optlen, sizeof(int))) 4506 return -EFAULT; 4507 4508 if (len != sizeof(opt)) 4509 return -EINVAL; 4510 4511 if (!tp->repair) 4512 return -EPERM; 4513 4514 opt.snd_wl1 = tp->snd_wl1; 4515 opt.snd_wnd = tp->snd_wnd; 4516 opt.max_window = tp->max_window; 4517 opt.rcv_wnd = tp->rcv_wnd; 4518 opt.rcv_wup = tp->rcv_wup; 4519 4520 if (copy_to_sockptr(optval, &opt, len)) 4521 return -EFAULT; 4522 return 0; 4523 } 4524 case TCP_QUEUE_SEQ: 4525 if (tp->repair_queue == TCP_SEND_QUEUE) 4526 val = tp->write_seq; 4527 else if (tp->repair_queue == TCP_RECV_QUEUE) 4528 val = tp->rcv_nxt; 4529 else 4530 return -EINVAL; 4531 break; 4532 4533 case TCP_USER_TIMEOUT: 4534 val = READ_ONCE(icsk->icsk_user_timeout); 4535 break; 4536 4537 case TCP_FASTOPEN: 4538 val = READ_ONCE(icsk->icsk_accept_queue.fastopenq.max_qlen); 4539 break; 4540 4541 case TCP_FASTOPEN_CONNECT: 4542 val = tp->fastopen_connect; 4543 break; 4544 4545 case TCP_FASTOPEN_NO_COOKIE: 4546 val = tp->fastopen_no_cookie; 4547 break; 4548 4549 case TCP_TX_DELAY: 4550 val = READ_ONCE(tp->tcp_tx_delay); 4551 break; 4552 4553 case TCP_TIMESTAMP: 4554 val = tcp_clock_ts(tp->tcp_usec_ts) + READ_ONCE(tp->tsoffset); 4555 if (tp->tcp_usec_ts) 4556 val |= 1; 4557 else 4558 val &= ~1; 4559 break; 4560 case TCP_NOTSENT_LOWAT: 4561 val = READ_ONCE(tp->notsent_lowat); 4562 break; 4563 case TCP_INQ: 4564 val = tp->recvmsg_inq; 4565 break; 4566 case TCP_SAVE_SYN: 4567 val = tp->save_syn; 4568 break; 4569 case TCP_SAVED_SYN: { 4570 if (copy_from_sockptr(&len, optlen, sizeof(int))) 4571 return -EFAULT; 4572 4573 sockopt_lock_sock(sk); 4574 if (tp->saved_syn) { 4575 if (len < tcp_saved_syn_len(tp->saved_syn)) { 4576 len = tcp_saved_syn_len(tp->saved_syn); 4577 if (copy_to_sockptr(optlen, &len, sizeof(int))) { 4578 sockopt_release_sock(sk); 4579 return -EFAULT; 4580 } 4581 sockopt_release_sock(sk); 4582 return -EINVAL; 4583 } 4584 len = tcp_saved_syn_len(tp->saved_syn); 4585 if (copy_to_sockptr(optlen, &len, sizeof(int))) { 4586 sockopt_release_sock(sk); 4587 return -EFAULT; 4588 } 4589 if (copy_to_sockptr(optval, tp->saved_syn->data, len)) { 4590 sockopt_release_sock(sk); 4591 return -EFAULT; 4592 } 4593 tcp_saved_syn_free(tp); 4594 sockopt_release_sock(sk); 4595 } else { 4596 sockopt_release_sock(sk); 4597 len = 0; 4598 if (copy_to_sockptr(optlen, &len, sizeof(int))) 4599 return -EFAULT; 4600 } 4601 return 0; 4602 } 4603 #ifdef CONFIG_MMU 4604 case TCP_ZEROCOPY_RECEIVE: { 4605 struct scm_timestamping_internal tss; 4606 struct tcp_zerocopy_receive zc = {}; 4607 int err; 4608 4609 if (copy_from_sockptr(&len, optlen, sizeof(int))) 4610 return -EFAULT; 4611 if (len < 0 || 4612 len < offsetofend(struct tcp_zerocopy_receive, length)) 4613 return -EINVAL; 4614 if (unlikely(len > sizeof(zc))) { 4615 err = check_zeroed_sockptr(optval, sizeof(zc), 4616 len - sizeof(zc)); 4617 if (err < 1) 4618 return err == 0 ? -EINVAL : err; 4619 len = sizeof(zc); 4620 if (copy_to_sockptr(optlen, &len, sizeof(int))) 4621 return -EFAULT; 4622 } 4623 if (copy_from_sockptr(&zc, optval, len)) 4624 return -EFAULT; 4625 if (zc.reserved) 4626 return -EINVAL; 4627 if (zc.msg_flags & ~(TCP_VALID_ZC_MSG_FLAGS)) 4628 return -EINVAL; 4629 sockopt_lock_sock(sk); 4630 err = tcp_zerocopy_receive(sk, &zc, &tss); 4631 err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname, 4632 &zc, &len, err); 4633 sockopt_release_sock(sk); 4634 if (len >= offsetofend(struct tcp_zerocopy_receive, msg_flags)) 4635 goto zerocopy_rcv_cmsg; 4636 switch (len) { 4637 case offsetofend(struct tcp_zerocopy_receive, msg_flags): 4638 goto zerocopy_rcv_cmsg; 4639 case offsetofend(struct tcp_zerocopy_receive, msg_controllen): 4640 case offsetofend(struct tcp_zerocopy_receive, msg_control): 4641 case offsetofend(struct tcp_zerocopy_receive, flags): 4642 case offsetofend(struct tcp_zerocopy_receive, copybuf_len): 4643 case offsetofend(struct tcp_zerocopy_receive, copybuf_address): 4644 case offsetofend(struct tcp_zerocopy_receive, err): 4645 goto zerocopy_rcv_sk_err; 4646 case offsetofend(struct tcp_zerocopy_receive, inq): 4647 goto zerocopy_rcv_inq; 4648 case offsetofend(struct tcp_zerocopy_receive, length): 4649 default: 4650 goto zerocopy_rcv_out; 4651 } 4652 zerocopy_rcv_cmsg: 4653 if (zc.msg_flags & TCP_CMSG_TS) 4654 tcp_zc_finalize_rx_tstamp(sk, &zc, &tss); 4655 else 4656 zc.msg_flags = 0; 4657 zerocopy_rcv_sk_err: 4658 if (!err) 4659 zc.err = sock_error(sk); 4660 zerocopy_rcv_inq: 4661 zc.inq = tcp_inq_hint(sk); 4662 zerocopy_rcv_out: 4663 if (!err && copy_to_sockptr(optval, &zc, len)) 4664 err = -EFAULT; 4665 return err; 4666 } 4667 #endif 4668 case TCP_AO_REPAIR: 4669 if (!tcp_can_repair_sock(sk)) 4670 return -EPERM; 4671 return tcp_ao_get_repair(sk, optval, optlen); 4672 case TCP_AO_GET_KEYS: 4673 case TCP_AO_INFO: { 4674 int err; 4675 4676 sockopt_lock_sock(sk); 4677 if (optname == TCP_AO_GET_KEYS) 4678 err = tcp_ao_get_mkts(sk, optval, optlen); 4679 else 4680 err = tcp_ao_get_sock_info(sk, optval, optlen); 4681 sockopt_release_sock(sk); 4682 4683 return err; 4684 } 4685 case TCP_IS_MPTCP: 4686 val = 0; 4687 break; 4688 case TCP_RTO_MAX_MS: 4689 val = jiffies_to_msecs(tcp_rto_max(sk)); 4690 break; 4691 case TCP_RTO_MIN_US: 4692 val = jiffies_to_usecs(READ_ONCE(inet_csk(sk)->icsk_rto_min)); 4693 break; 4694 case TCP_DELACK_MAX_US: 4695 val = jiffies_to_usecs(READ_ONCE(inet_csk(sk)->icsk_delack_max)); 4696 break; 4697 default: 4698 return -ENOPROTOOPT; 4699 } 4700 4701 if (copy_to_sockptr(optlen, &len, sizeof(int))) 4702 return -EFAULT; 4703 if (copy_to_sockptr(optval, &val, len)) 4704 return -EFAULT; 4705 return 0; 4706 } 4707 4708 bool tcp_bpf_bypass_getsockopt(int level, int optname) 4709 { 4710 /* TCP do_tcp_getsockopt has optimized getsockopt implementation 4711 * to avoid extra socket lock for TCP_ZEROCOPY_RECEIVE. 4712 */ 4713 if (level == SOL_TCP && optname == TCP_ZEROCOPY_RECEIVE) 4714 return true; 4715 4716 return false; 4717 } 4718 EXPORT_IPV6_MOD(tcp_bpf_bypass_getsockopt); 4719 4720 int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval, 4721 int __user *optlen) 4722 { 4723 struct inet_connection_sock *icsk = inet_csk(sk); 4724 4725 if (level != SOL_TCP) 4726 /* Paired with WRITE_ONCE() in do_ipv6_setsockopt() and tcp_v6_connect() */ 4727 return READ_ONCE(icsk->icsk_af_ops)->getsockopt(sk, level, optname, 4728 optval, optlen); 4729 return do_tcp_getsockopt(sk, level, optname, USER_SOCKPTR(optval), 4730 USER_SOCKPTR(optlen)); 4731 } 4732 EXPORT_IPV6_MOD(tcp_getsockopt); 4733 4734 #ifdef CONFIG_TCP_MD5SIG 4735 int tcp_md5_sigpool_id = -1; 4736 EXPORT_IPV6_MOD_GPL(tcp_md5_sigpool_id); 4737 4738 int tcp_md5_alloc_sigpool(void) 4739 { 4740 size_t scratch_size; 4741 int ret; 4742 4743 scratch_size = sizeof(union tcp_md5sum_block) + sizeof(struct tcphdr); 4744 ret = tcp_sigpool_alloc_ahash("md5", scratch_size); 4745 if (ret >= 0) { 4746 /* As long as any md5 sigpool was allocated, the return 4747 * id would stay the same. Re-write the id only for the case 4748 * when previously all MD5 keys were deleted and this call 4749 * allocates the first MD5 key, which may return a different 4750 * sigpool id than was used previously. 4751 */ 4752 WRITE_ONCE(tcp_md5_sigpool_id, ret); /* Avoids the compiler potentially being smart here */ 4753 return 0; 4754 } 4755 return ret; 4756 } 4757 4758 void tcp_md5_release_sigpool(void) 4759 { 4760 tcp_sigpool_release(READ_ONCE(tcp_md5_sigpool_id)); 4761 } 4762 4763 void tcp_md5_add_sigpool(void) 4764 { 4765 tcp_sigpool_get(READ_ONCE(tcp_md5_sigpool_id)); 4766 } 4767 4768 int tcp_md5_hash_key(struct tcp_sigpool *hp, 4769 const struct tcp_md5sig_key *key) 4770 { 4771 u8 keylen = READ_ONCE(key->keylen); /* paired with WRITE_ONCE() in tcp_md5_do_add */ 4772 struct scatterlist sg; 4773 4774 sg_init_one(&sg, key->key, keylen); 4775 ahash_request_set_crypt(hp->req, &sg, NULL, keylen); 4776 4777 /* We use data_race() because tcp_md5_do_add() might change 4778 * key->key under us 4779 */ 4780 return data_race(crypto_ahash_update(hp->req)); 4781 } 4782 EXPORT_IPV6_MOD(tcp_md5_hash_key); 4783 4784 /* Called with rcu_read_lock() */ 4785 static enum skb_drop_reason 4786 tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb, 4787 const void *saddr, const void *daddr, 4788 int family, int l3index, const __u8 *hash_location) 4789 { 4790 /* This gets called for each TCP segment that has TCP-MD5 option. 4791 * We have 3 drop cases: 4792 * o No MD5 hash and one expected. 4793 * o MD5 hash and we're not expecting one. 4794 * o MD5 hash and its wrong. 4795 */ 4796 const struct tcp_sock *tp = tcp_sk(sk); 4797 struct tcp_md5sig_key *key; 4798 u8 newhash[16]; 4799 int genhash; 4800 4801 key = tcp_md5_do_lookup(sk, l3index, saddr, family); 4802 4803 if (!key && hash_location) { 4804 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5UNEXPECTED); 4805 trace_tcp_hash_md5_unexpected(sk, skb); 4806 return SKB_DROP_REASON_TCP_MD5UNEXPECTED; 4807 } 4808 4809 /* Check the signature. 4810 * To support dual stack listeners, we need to handle 4811 * IPv4-mapped case. 4812 */ 4813 if (family == AF_INET) 4814 genhash = tcp_v4_md5_hash_skb(newhash, key, NULL, skb); 4815 else 4816 genhash = tp->af_specific->calc_md5_hash(newhash, key, 4817 NULL, skb); 4818 if (genhash || memcmp(hash_location, newhash, 16) != 0) { 4819 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5FAILURE); 4820 trace_tcp_hash_md5_mismatch(sk, skb); 4821 return SKB_DROP_REASON_TCP_MD5FAILURE; 4822 } 4823 return SKB_NOT_DROPPED_YET; 4824 } 4825 #else 4826 static inline enum skb_drop_reason 4827 tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb, 4828 const void *saddr, const void *daddr, 4829 int family, int l3index, const __u8 *hash_location) 4830 { 4831 return SKB_NOT_DROPPED_YET; 4832 } 4833 4834 #endif 4835 4836 /* Called with rcu_read_lock() */ 4837 enum skb_drop_reason 4838 tcp_inbound_hash(struct sock *sk, const struct request_sock *req, 4839 const struct sk_buff *skb, 4840 const void *saddr, const void *daddr, 4841 int family, int dif, int sdif) 4842 { 4843 const struct tcphdr *th = tcp_hdr(skb); 4844 const struct tcp_ao_hdr *aoh; 4845 const __u8 *md5_location; 4846 int l3index; 4847 4848 /* Invalid option or two times meet any of auth options */ 4849 if (tcp_parse_auth_options(th, &md5_location, &aoh)) { 4850 trace_tcp_hash_bad_header(sk, skb); 4851 return SKB_DROP_REASON_TCP_AUTH_HDR; 4852 } 4853 4854 if (req) { 4855 if (tcp_rsk_used_ao(req) != !!aoh) { 4856 u8 keyid, rnext, maclen; 4857 4858 if (aoh) { 4859 keyid = aoh->keyid; 4860 rnext = aoh->rnext_keyid; 4861 maclen = tcp_ao_hdr_maclen(aoh); 4862 } else { 4863 keyid = rnext = maclen = 0; 4864 } 4865 4866 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOBAD); 4867 trace_tcp_ao_handshake_failure(sk, skb, keyid, rnext, maclen); 4868 return SKB_DROP_REASON_TCP_AOFAILURE; 4869 } 4870 } 4871 4872 /* sdif set, means packet ingressed via a device 4873 * in an L3 domain and dif is set to the l3mdev 4874 */ 4875 l3index = sdif ? dif : 0; 4876 4877 /* Fast path: unsigned segments */ 4878 if (likely(!md5_location && !aoh)) { 4879 /* Drop if there's TCP-MD5 or TCP-AO key with any rcvid/sndid 4880 * for the remote peer. On TCP-AO established connection 4881 * the last key is impossible to remove, so there's 4882 * always at least one current_key. 4883 */ 4884 if (tcp_ao_required(sk, saddr, family, l3index, true)) { 4885 trace_tcp_hash_ao_required(sk, skb); 4886 return SKB_DROP_REASON_TCP_AONOTFOUND; 4887 } 4888 if (unlikely(tcp_md5_do_lookup(sk, l3index, saddr, family))) { 4889 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5NOTFOUND); 4890 trace_tcp_hash_md5_required(sk, skb); 4891 return SKB_DROP_REASON_TCP_MD5NOTFOUND; 4892 } 4893 return SKB_NOT_DROPPED_YET; 4894 } 4895 4896 if (aoh) 4897 return tcp_inbound_ao_hash(sk, skb, family, req, l3index, aoh); 4898 4899 return tcp_inbound_md5_hash(sk, skb, saddr, daddr, family, 4900 l3index, md5_location); 4901 } 4902 EXPORT_IPV6_MOD_GPL(tcp_inbound_hash); 4903 4904 void tcp_done(struct sock *sk) 4905 { 4906 struct request_sock *req; 4907 4908 /* We might be called with a new socket, after 4909 * inet_csk_prepare_forced_close() has been called 4910 * so we can not use lockdep_sock_is_held(sk) 4911 */ 4912 req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk, 1); 4913 4914 if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV) 4915 TCP_INC_STATS(sock_net(sk), TCP_MIB_ATTEMPTFAILS); 4916 4917 tcp_set_state(sk, TCP_CLOSE); 4918 tcp_clear_xmit_timers(sk); 4919 if (req) 4920 reqsk_fastopen_remove(sk, req, false); 4921 4922 WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK); 4923 4924 if (!sock_flag(sk, SOCK_DEAD)) 4925 sk->sk_state_change(sk); 4926 else 4927 inet_csk_destroy_sock(sk); 4928 } 4929 EXPORT_SYMBOL_GPL(tcp_done); 4930 4931 int tcp_abort(struct sock *sk, int err) 4932 { 4933 int state = inet_sk_state_load(sk); 4934 4935 if (state == TCP_NEW_SYN_RECV) { 4936 struct request_sock *req = inet_reqsk(sk); 4937 4938 local_bh_disable(); 4939 inet_csk_reqsk_queue_drop(req->rsk_listener, req); 4940 local_bh_enable(); 4941 return 0; 4942 } 4943 if (state == TCP_TIME_WAIT) { 4944 struct inet_timewait_sock *tw = inet_twsk(sk); 4945 4946 refcount_inc(&tw->tw_refcnt); 4947 local_bh_disable(); 4948 inet_twsk_deschedule_put(tw); 4949 local_bh_enable(); 4950 return 0; 4951 } 4952 4953 /* BPF context ensures sock locking. */ 4954 if (!has_current_bpf_ctx()) 4955 /* Don't race with userspace socket closes such as tcp_close. */ 4956 lock_sock(sk); 4957 4958 /* Avoid closing the same socket twice. */ 4959 if (sk->sk_state == TCP_CLOSE) { 4960 if (!has_current_bpf_ctx()) 4961 release_sock(sk); 4962 return -ENOENT; 4963 } 4964 4965 if (sk->sk_state == TCP_LISTEN) { 4966 tcp_set_state(sk, TCP_CLOSE); 4967 inet_csk_listen_stop(sk); 4968 } 4969 4970 /* Don't race with BH socket closes such as inet_csk_listen_stop. */ 4971 local_bh_disable(); 4972 bh_lock_sock(sk); 4973 4974 if (tcp_need_reset(sk->sk_state)) 4975 tcp_send_active_reset(sk, GFP_ATOMIC, 4976 SK_RST_REASON_TCP_STATE); 4977 tcp_done_with_error(sk, err); 4978 4979 bh_unlock_sock(sk); 4980 local_bh_enable(); 4981 if (!has_current_bpf_ctx()) 4982 release_sock(sk); 4983 return 0; 4984 } 4985 EXPORT_SYMBOL_GPL(tcp_abort); 4986 4987 extern struct tcp_congestion_ops tcp_reno; 4988 4989 static __initdata unsigned long thash_entries; 4990 static int __init set_thash_entries(char *str) 4991 { 4992 ssize_t ret; 4993 4994 if (!str) 4995 return 0; 4996 4997 ret = kstrtoul(str, 0, &thash_entries); 4998 if (ret) 4999 return 0; 5000 5001 return 1; 5002 } 5003 __setup("thash_entries=", set_thash_entries); 5004 5005 static void __init tcp_init_mem(void) 5006 { 5007 unsigned long limit = nr_free_buffer_pages() / 16; 5008 5009 limit = max(limit, 128UL); 5010 sysctl_tcp_mem[0] = limit / 4 * 3; /* 4.68 % */ 5011 sysctl_tcp_mem[1] = limit; /* 6.25 % */ 5012 sysctl_tcp_mem[2] = sysctl_tcp_mem[0] * 2; /* 9.37 % */ 5013 } 5014 5015 static void __init tcp_struct_check(void) 5016 { 5017 /* TX read-mostly hotpath cache lines */ 5018 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, max_window); 5019 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, rcv_ssthresh); 5020 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, reordering); 5021 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, notsent_lowat); 5022 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, gso_segs); 5023 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, lost_skb_hint); 5024 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, retransmit_skb_hint); 5025 CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_read_tx, 40); 5026 5027 /* TXRX read-mostly hotpath cache lines */ 5028 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, tsoffset); 5029 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, snd_wnd); 5030 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, mss_cache); 5031 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, snd_cwnd); 5032 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, prr_out); 5033 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, lost_out); 5034 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, sacked_out); 5035 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, scaling_ratio); 5036 CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_read_txrx, 32); 5037 5038 /* RX read-mostly hotpath cache lines */ 5039 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, copied_seq); 5040 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, rcv_tstamp); 5041 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, snd_wl1); 5042 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, tlp_high_seq); 5043 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, rttvar_us); 5044 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, retrans_out); 5045 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, advmss); 5046 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, urg_data); 5047 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, lost); 5048 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, rtt_min); 5049 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, out_of_order_queue); 5050 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, snd_ssthresh); 5051 #if IS_ENABLED(CONFIG_TLS_DEVICE) 5052 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, tcp_clean_acked); 5053 CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_read_rx, 77); 5054 #else 5055 CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_read_rx, 69); 5056 #endif 5057 5058 /* TX read-write hotpath cache lines */ 5059 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, segs_out); 5060 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, data_segs_out); 5061 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, bytes_sent); 5062 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, snd_sml); 5063 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_start); 5064 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_stat); 5065 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, write_seq); 5066 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, pushed_seq); 5067 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, lsndtime); 5068 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, mdev_us); 5069 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tcp_wstamp_ns); 5070 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, rtt_seq); 5071 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tsorted_sent_queue); 5072 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, highest_sack); 5073 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, ecn_flags); 5074 CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_write_tx, 89); 5075 5076 /* TXRX read-write hotpath cache lines */ 5077 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, pred_flags); 5078 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, tcp_clock_cache); 5079 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, tcp_mstamp); 5080 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_nxt); 5081 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_nxt); 5082 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_una); 5083 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, window_clamp); 5084 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, srtt_us); 5085 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, packets_out); 5086 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_up); 5087 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, delivered); 5088 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, delivered_ce); 5089 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, app_limited); 5090 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_wnd); 5091 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rx_opt); 5092 5093 /* 32bit arches with 8byte alignment on u64 fields might need padding 5094 * before tcp_clock_cache. 5095 */ 5096 CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_write_txrx, 92 + 4); 5097 5098 /* RX read-write hotpath cache lines */ 5099 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, bytes_received); 5100 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, segs_in); 5101 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, data_segs_in); 5102 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_wup); 5103 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, max_packets_out); 5104 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, cwnd_usage_seq); 5105 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rate_delivered); 5106 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rate_interval_us); 5107 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_rtt_last_tsecr); 5108 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, first_tx_mstamp); 5109 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, delivered_mstamp); 5110 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, bytes_acked); 5111 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_rtt_est); 5112 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcvq_space); 5113 CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_write_rx, 99); 5114 } 5115 5116 void __init tcp_init(void) 5117 { 5118 int max_rshare, max_wshare, cnt; 5119 unsigned long limit; 5120 unsigned int i; 5121 5122 BUILD_BUG_ON(TCP_MIN_SND_MSS <= MAX_TCP_OPTION_SPACE); 5123 BUILD_BUG_ON(sizeof(struct tcp_skb_cb) > 5124 sizeof_field(struct sk_buff, cb)); 5125 5126 tcp_struct_check(); 5127 5128 percpu_counter_init(&tcp_sockets_allocated, 0, GFP_KERNEL); 5129 5130 timer_setup(&tcp_orphan_timer, tcp_orphan_update, TIMER_DEFERRABLE); 5131 mod_timer(&tcp_orphan_timer, jiffies + TCP_ORPHAN_TIMER_PERIOD); 5132 5133 inet_hashinfo2_init(&tcp_hashinfo, "tcp_listen_portaddr_hash", 5134 thash_entries, 21, /* one slot per 2 MB*/ 5135 0, 64 * 1024); 5136 tcp_hashinfo.bind_bucket_cachep = 5137 kmem_cache_create("tcp_bind_bucket", 5138 sizeof(struct inet_bind_bucket), 0, 5139 SLAB_HWCACHE_ALIGN | SLAB_PANIC | 5140 SLAB_ACCOUNT, 5141 NULL); 5142 tcp_hashinfo.bind2_bucket_cachep = 5143 kmem_cache_create("tcp_bind2_bucket", 5144 sizeof(struct inet_bind2_bucket), 0, 5145 SLAB_HWCACHE_ALIGN | SLAB_PANIC | 5146 SLAB_ACCOUNT, 5147 NULL); 5148 5149 /* Size and allocate the main established and bind bucket 5150 * hash tables. 5151 * 5152 * The methodology is similar to that of the buffer cache. 5153 */ 5154 tcp_hashinfo.ehash = 5155 alloc_large_system_hash("TCP established", 5156 sizeof(struct inet_ehash_bucket), 5157 thash_entries, 5158 17, /* one slot per 128 KB of memory */ 5159 0, 5160 NULL, 5161 &tcp_hashinfo.ehash_mask, 5162 0, 5163 thash_entries ? 0 : 512 * 1024); 5164 for (i = 0; i <= tcp_hashinfo.ehash_mask; i++) 5165 INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].chain, i); 5166 5167 if (inet_ehash_locks_alloc(&tcp_hashinfo)) 5168 panic("TCP: failed to alloc ehash_locks"); 5169 tcp_hashinfo.bhash = 5170 alloc_large_system_hash("TCP bind", 5171 2 * sizeof(struct inet_bind_hashbucket), 5172 tcp_hashinfo.ehash_mask + 1, 5173 17, /* one slot per 128 KB of memory */ 5174 0, 5175 &tcp_hashinfo.bhash_size, 5176 NULL, 5177 0, 5178 64 * 1024); 5179 tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size; 5180 tcp_hashinfo.bhash2 = tcp_hashinfo.bhash + tcp_hashinfo.bhash_size; 5181 for (i = 0; i < tcp_hashinfo.bhash_size; i++) { 5182 spin_lock_init(&tcp_hashinfo.bhash[i].lock); 5183 INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain); 5184 spin_lock_init(&tcp_hashinfo.bhash2[i].lock); 5185 INIT_HLIST_HEAD(&tcp_hashinfo.bhash2[i].chain); 5186 } 5187 5188 tcp_hashinfo.pernet = false; 5189 5190 cnt = tcp_hashinfo.ehash_mask + 1; 5191 sysctl_tcp_max_orphans = cnt / 2; 5192 5193 tcp_init_mem(); 5194 /* Set per-socket limits to no more than 1/128 the pressure threshold */ 5195 limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7); 5196 max_wshare = min(4UL*1024*1024, limit); 5197 max_rshare = min(6UL*1024*1024, limit); 5198 5199 init_net.ipv4.sysctl_tcp_wmem[0] = PAGE_SIZE; 5200 init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024; 5201 init_net.ipv4.sysctl_tcp_wmem[2] = max(64*1024, max_wshare); 5202 5203 init_net.ipv4.sysctl_tcp_rmem[0] = PAGE_SIZE; 5204 init_net.ipv4.sysctl_tcp_rmem[1] = 131072; 5205 init_net.ipv4.sysctl_tcp_rmem[2] = max(131072, max_rshare); 5206 5207 pr_info("Hash tables configured (established %u bind %u)\n", 5208 tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size); 5209 5210 tcp_v4_init(); 5211 tcp_metrics_init(); 5212 BUG_ON(tcp_register_congestion_control(&tcp_reno) != 0); 5213 tcp_tasklet_init(); 5214 mptcp_init(); 5215 } 5216