1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * Implementation of the Transmission Control Protocol(TCP). 8 * 9 * Authors: Ross Biro 10 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 11 * Mark Evans, <evansmp@uhura.aston.ac.uk> 12 * Corey Minyard <wf-rch!minyard@relay.EU.net> 13 * Florian La Roche, <flla@stud.uni-sb.de> 14 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu> 15 * Linus Torvalds, <torvalds@cs.helsinki.fi> 16 * Alan Cox, <gw4pts@gw4pts.ampr.org> 17 * Matthew Dillon, <dillon@apollo.west.oic.com> 18 * Arnt Gulbrandsen, <agulbra@nvg.unit.no> 19 * Jorge Cwik, <jorge@laser.satlink.net> 20 */ 21 22 /* 23 * Changes: Pedro Roque : Retransmit queue handled by TCP. 24 * : Fragmentation on mtu decrease 25 * : Segment collapse on retransmit 26 * : AF independence 27 * 28 * Linus Torvalds : send_delayed_ack 29 * David S. Miller : Charge memory using the right skb 30 * during syn/ack processing. 31 * David S. Miller : Output engine completely rewritten. 32 * Andrea Arcangeli: SYNACK carry ts_recent in tsecr. 33 * Cacophonix Gaul : draft-minshall-nagle-01 34 * J Hadi Salim : ECN support 35 * 36 */ 37 38 #define pr_fmt(fmt) "TCP: " fmt 39 40 #include <net/tcp.h> 41 #include <net/mptcp.h> 42 43 #include <linux/compiler.h> 44 #include <linux/gfp.h> 45 #include <linux/module.h> 46 #include <linux/static_key.h> 47 48 #include <trace/events/tcp.h> 49 50 /* Refresh clocks of a TCP socket, 51 * ensuring monotically increasing values. 52 */ 53 void tcp_mstamp_refresh(struct tcp_sock *tp) 54 { 55 u64 val = tcp_clock_ns(); 56 57 tp->tcp_clock_cache = val; 58 tp->tcp_mstamp = div_u64(val, NSEC_PER_USEC); 59 } 60 61 static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, 62 int push_one, gfp_t gfp); 63 64 /* Account for new data that has been sent to the network. */ 65 static void tcp_event_new_data_sent(struct sock *sk, struct sk_buff *skb) 66 { 67 struct inet_connection_sock *icsk = inet_csk(sk); 68 struct tcp_sock *tp = tcp_sk(sk); 69 unsigned int prior_packets = tp->packets_out; 70 71 WRITE_ONCE(tp->snd_nxt, TCP_SKB_CB(skb)->end_seq); 72 73 __skb_unlink(skb, &sk->sk_write_queue); 74 tcp_rbtree_insert(&sk->tcp_rtx_queue, skb); 75 76 if (tp->highest_sack == NULL) 77 tp->highest_sack = skb; 78 79 tp->packets_out += tcp_skb_pcount(skb); 80 if (!prior_packets || icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) 81 tcp_rearm_rto(sk); 82 83 NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT, 84 tcp_skb_pcount(skb)); 85 tcp_check_space(sk); 86 } 87 88 /* SND.NXT, if window was not shrunk or the amount of shrunk was less than one 89 * window scaling factor due to loss of precision. 90 * If window has been shrunk, what should we make? It is not clear at all. 91 * Using SND.UNA we will fail to open window, SND.NXT is out of window. :-( 92 * Anything in between SND.UNA...SND.UNA+SND.WND also can be already 93 * invalid. OK, let's make this for now: 94 */ 95 static inline __u32 tcp_acceptable_seq(const struct sock *sk) 96 { 97 const struct tcp_sock *tp = tcp_sk(sk); 98 99 if (!before(tcp_wnd_end(tp), tp->snd_nxt) || 100 (tp->rx_opt.wscale_ok && 101 ((tp->snd_nxt - tcp_wnd_end(tp)) < (1 << tp->rx_opt.rcv_wscale)))) 102 return tp->snd_nxt; 103 else 104 return tcp_wnd_end(tp); 105 } 106 107 /* Calculate mss to advertise in SYN segment. 108 * RFC1122, RFC1063, draft-ietf-tcpimpl-pmtud-01 state that: 109 * 110 * 1. It is independent of path mtu. 111 * 2. Ideally, it is maximal possible segment size i.e. 65535-40. 112 * 3. For IPv4 it is reasonable to calculate it from maximal MTU of 113 * attached devices, because some buggy hosts are confused by 114 * large MSS. 115 * 4. We do not make 3, we advertise MSS, calculated from first 116 * hop device mtu, but allow to raise it to ip_rt_min_advmss. 117 * This may be overridden via information stored in routing table. 118 * 5. Value 65535 for MSS is valid in IPv6 and means "as large as possible, 119 * probably even Jumbo". 120 */ 121 static __u16 tcp_advertise_mss(struct sock *sk) 122 { 123 struct tcp_sock *tp = tcp_sk(sk); 124 const struct dst_entry *dst = __sk_dst_get(sk); 125 int mss = tp->advmss; 126 127 if (dst) { 128 unsigned int metric = dst_metric_advmss(dst); 129 130 if (metric < mss) { 131 mss = metric; 132 tp->advmss = mss; 133 } 134 } 135 136 return (__u16)mss; 137 } 138 139 /* RFC2861. Reset CWND after idle period longer RTO to "restart window". 140 * This is the first part of cwnd validation mechanism. 141 */ 142 void tcp_cwnd_restart(struct sock *sk, s32 delta) 143 { 144 struct tcp_sock *tp = tcp_sk(sk); 145 u32 restart_cwnd = tcp_init_cwnd(tp, __sk_dst_get(sk)); 146 u32 cwnd = tcp_snd_cwnd(tp); 147 148 tcp_ca_event(sk, CA_EVENT_CWND_RESTART); 149 150 tp->snd_ssthresh = tcp_current_ssthresh(sk); 151 restart_cwnd = min(restart_cwnd, cwnd); 152 153 while ((delta -= inet_csk(sk)->icsk_rto) > 0 && cwnd > restart_cwnd) 154 cwnd >>= 1; 155 tcp_snd_cwnd_set(tp, max(cwnd, restart_cwnd)); 156 tp->snd_cwnd_stamp = tcp_jiffies32; 157 tp->snd_cwnd_used = 0; 158 } 159 160 /* Congestion state accounting after a packet has been sent. */ 161 static void tcp_event_data_sent(struct tcp_sock *tp, 162 struct sock *sk) 163 { 164 struct inet_connection_sock *icsk = inet_csk(sk); 165 const u32 now = tcp_jiffies32; 166 167 if (tcp_packets_in_flight(tp) == 0) 168 tcp_ca_event(sk, CA_EVENT_TX_START); 169 170 tp->lsndtime = now; 171 172 /* If it is a reply for ato after last received 173 * packet, increase pingpong count. 174 */ 175 if ((u32)(now - icsk->icsk_ack.lrcvtime) < icsk->icsk_ack.ato) 176 inet_csk_inc_pingpong_cnt(sk); 177 } 178 179 /* Account for an ACK we sent. */ 180 static inline void tcp_event_ack_sent(struct sock *sk, u32 rcv_nxt) 181 { 182 struct tcp_sock *tp = tcp_sk(sk); 183 184 if (unlikely(tp->compressed_ack)) { 185 NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPACKCOMPRESSED, 186 tp->compressed_ack); 187 tp->compressed_ack = 0; 188 if (hrtimer_try_to_cancel(&tp->compressed_ack_timer) == 1) 189 __sock_put(sk); 190 } 191 192 if (unlikely(rcv_nxt != tp->rcv_nxt)) 193 return; /* Special ACK sent by DCTCP to reflect ECN */ 194 tcp_dec_quickack_mode(sk); 195 inet_csk_clear_xmit_timer(sk, ICSK_TIME_DACK); 196 } 197 198 /* Determine a window scaling and initial window to offer. 199 * Based on the assumption that the given amount of space 200 * will be offered. Store the results in the tp structure. 201 * NOTE: for smooth operation initial space offering should 202 * be a multiple of mss if possible. We assume here that mss >= 1. 203 * This MUST be enforced by all callers. 204 */ 205 void tcp_select_initial_window(const struct sock *sk, int __space, __u32 mss, 206 __u32 *rcv_wnd, __u32 *window_clamp, 207 int wscale_ok, __u8 *rcv_wscale, 208 __u32 init_rcv_wnd) 209 { 210 unsigned int space = (__space < 0 ? 0 : __space); 211 212 /* If no clamp set the clamp to the max possible scaled window */ 213 if (*window_clamp == 0) 214 (*window_clamp) = (U16_MAX << TCP_MAX_WSCALE); 215 space = min(*window_clamp, space); 216 217 /* Quantize space offering to a multiple of mss if possible. */ 218 if (space > mss) 219 space = rounddown(space, mss); 220 221 /* NOTE: offering an initial window larger than 32767 222 * will break some buggy TCP stacks. If the admin tells us 223 * it is likely we could be speaking with such a buggy stack 224 * we will truncate our initial window offering to 32K-1 225 * unless the remote has sent us a window scaling option, 226 * which we interpret as a sign the remote TCP is not 227 * misinterpreting the window field as a signed quantity. 228 */ 229 if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_workaround_signed_windows)) 230 (*rcv_wnd) = min(space, MAX_TCP_WINDOW); 231 else 232 (*rcv_wnd) = min_t(u32, space, U16_MAX); 233 234 if (init_rcv_wnd) 235 *rcv_wnd = min(*rcv_wnd, init_rcv_wnd * mss); 236 237 *rcv_wscale = 0; 238 if (wscale_ok) { 239 /* Set window scaling on max possible window */ 240 space = max_t(u32, space, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2])); 241 space = max_t(u32, space, READ_ONCE(sysctl_rmem_max)); 242 space = min_t(u32, space, *window_clamp); 243 *rcv_wscale = clamp_t(int, ilog2(space) - 15, 244 0, TCP_MAX_WSCALE); 245 } 246 /* Set the clamp no higher than max representable value */ 247 (*window_clamp) = min_t(__u32, U16_MAX << (*rcv_wscale), *window_clamp); 248 } 249 EXPORT_SYMBOL(tcp_select_initial_window); 250 251 /* Chose a new window to advertise, update state in tcp_sock for the 252 * socket, and return result with RFC1323 scaling applied. The return 253 * value can be stuffed directly into th->window for an outgoing 254 * frame. 255 */ 256 static u16 tcp_select_window(struct sock *sk) 257 { 258 struct tcp_sock *tp = tcp_sk(sk); 259 struct net *net = sock_net(sk); 260 u32 old_win = tp->rcv_wnd; 261 u32 cur_win, new_win; 262 263 /* Make the window 0 if we failed to queue the data because we 264 * are out of memory. The window is temporary, so we don't store 265 * it on the socket. 266 */ 267 if (unlikely(inet_csk(sk)->icsk_ack.pending & ICSK_ACK_NOMEM)) 268 return 0; 269 270 cur_win = tcp_receive_window(tp); 271 new_win = __tcp_select_window(sk); 272 if (new_win < cur_win) { 273 /* Danger Will Robinson! 274 * Don't update rcv_wup/rcv_wnd here or else 275 * we will not be able to advertise a zero 276 * window in time. --DaveM 277 * 278 * Relax Will Robinson. 279 */ 280 if (!READ_ONCE(net->ipv4.sysctl_tcp_shrink_window) || !tp->rx_opt.rcv_wscale) { 281 /* Never shrink the offered window */ 282 if (new_win == 0) 283 NET_INC_STATS(net, LINUX_MIB_TCPWANTZEROWINDOWADV); 284 new_win = ALIGN(cur_win, 1 << tp->rx_opt.rcv_wscale); 285 } 286 } 287 288 tp->rcv_wnd = new_win; 289 tp->rcv_wup = tp->rcv_nxt; 290 291 /* Make sure we do not exceed the maximum possible 292 * scaled window. 293 */ 294 if (!tp->rx_opt.rcv_wscale && 295 READ_ONCE(net->ipv4.sysctl_tcp_workaround_signed_windows)) 296 new_win = min(new_win, MAX_TCP_WINDOW); 297 else 298 new_win = min(new_win, (65535U << tp->rx_opt.rcv_wscale)); 299 300 /* RFC1323 scaling applied */ 301 new_win >>= tp->rx_opt.rcv_wscale; 302 303 /* If we advertise zero window, disable fast path. */ 304 if (new_win == 0) { 305 tp->pred_flags = 0; 306 if (old_win) 307 NET_INC_STATS(net, LINUX_MIB_TCPTOZEROWINDOWADV); 308 } else if (old_win == 0) { 309 NET_INC_STATS(net, LINUX_MIB_TCPFROMZEROWINDOWADV); 310 } 311 312 return new_win; 313 } 314 315 /* Packet ECN state for a SYN-ACK */ 316 static void tcp_ecn_send_synack(struct sock *sk, struct sk_buff *skb) 317 { 318 const struct tcp_sock *tp = tcp_sk(sk); 319 320 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_CWR; 321 if (!(tp->ecn_flags & TCP_ECN_OK)) 322 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ECE; 323 else if (tcp_ca_needs_ecn(sk) || 324 tcp_bpf_ca_needs_ecn(sk)) 325 INET_ECN_xmit(sk); 326 } 327 328 /* Packet ECN state for a SYN. */ 329 static void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb) 330 { 331 struct tcp_sock *tp = tcp_sk(sk); 332 bool bpf_needs_ecn = tcp_bpf_ca_needs_ecn(sk); 333 bool use_ecn = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn) == 1 || 334 tcp_ca_needs_ecn(sk) || bpf_needs_ecn; 335 336 if (!use_ecn) { 337 const struct dst_entry *dst = __sk_dst_get(sk); 338 339 if (dst && dst_feature(dst, RTAX_FEATURE_ECN)) 340 use_ecn = true; 341 } 342 343 tp->ecn_flags = 0; 344 345 if (use_ecn) { 346 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ECE | TCPHDR_CWR; 347 tp->ecn_flags = TCP_ECN_OK; 348 if (tcp_ca_needs_ecn(sk) || bpf_needs_ecn) 349 INET_ECN_xmit(sk); 350 } 351 } 352 353 static void tcp_ecn_clear_syn(struct sock *sk, struct sk_buff *skb) 354 { 355 if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn_fallback)) 356 /* tp->ecn_flags are cleared at a later point in time when 357 * SYN ACK is ultimatively being received. 358 */ 359 TCP_SKB_CB(skb)->tcp_flags &= ~(TCPHDR_ECE | TCPHDR_CWR); 360 } 361 362 static void 363 tcp_ecn_make_synack(const struct request_sock *req, struct tcphdr *th) 364 { 365 if (inet_rsk(req)->ecn_ok) 366 th->ece = 1; 367 } 368 369 /* Set up ECN state for a packet on a ESTABLISHED socket that is about to 370 * be sent. 371 */ 372 static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb, 373 struct tcphdr *th, int tcp_header_len) 374 { 375 struct tcp_sock *tp = tcp_sk(sk); 376 377 if (tp->ecn_flags & TCP_ECN_OK) { 378 /* Not-retransmitted data segment: set ECT and inject CWR. */ 379 if (skb->len != tcp_header_len && 380 !before(TCP_SKB_CB(skb)->seq, tp->snd_nxt)) { 381 INET_ECN_xmit(sk); 382 if (tp->ecn_flags & TCP_ECN_QUEUE_CWR) { 383 tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR; 384 th->cwr = 1; 385 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN; 386 } 387 } else if (!tcp_ca_needs_ecn(sk)) { 388 /* ACK or retransmitted segment: clear ECT|CE */ 389 INET_ECN_dontxmit(sk); 390 } 391 if (tp->ecn_flags & TCP_ECN_DEMAND_CWR) 392 th->ece = 1; 393 } 394 } 395 396 /* Constructs common control bits of non-data skb. If SYN/FIN is present, 397 * auto increment end seqno. 398 */ 399 static void tcp_init_nondata_skb(struct sk_buff *skb, u32 seq, u8 flags) 400 { 401 skb->ip_summed = CHECKSUM_PARTIAL; 402 403 TCP_SKB_CB(skb)->tcp_flags = flags; 404 405 tcp_skb_pcount_set(skb, 1); 406 407 TCP_SKB_CB(skb)->seq = seq; 408 if (flags & (TCPHDR_SYN | TCPHDR_FIN)) 409 seq++; 410 TCP_SKB_CB(skb)->end_seq = seq; 411 } 412 413 static inline bool tcp_urg_mode(const struct tcp_sock *tp) 414 { 415 return tp->snd_una != tp->snd_up; 416 } 417 418 #define OPTION_SACK_ADVERTISE BIT(0) 419 #define OPTION_TS BIT(1) 420 #define OPTION_MD5 BIT(2) 421 #define OPTION_WSCALE BIT(3) 422 #define OPTION_FAST_OPEN_COOKIE BIT(8) 423 #define OPTION_SMC BIT(9) 424 #define OPTION_MPTCP BIT(10) 425 #define OPTION_AO BIT(11) 426 427 static void smc_options_write(__be32 *ptr, u16 *options) 428 { 429 #if IS_ENABLED(CONFIG_SMC) 430 if (static_branch_unlikely(&tcp_have_smc)) { 431 if (unlikely(OPTION_SMC & *options)) { 432 *ptr++ = htonl((TCPOPT_NOP << 24) | 433 (TCPOPT_NOP << 16) | 434 (TCPOPT_EXP << 8) | 435 (TCPOLEN_EXP_SMC_BASE)); 436 *ptr++ = htonl(TCPOPT_SMC_MAGIC); 437 } 438 } 439 #endif 440 } 441 442 struct tcp_out_options { 443 u16 options; /* bit field of OPTION_* */ 444 u16 mss; /* 0 to disable */ 445 u8 ws; /* window scale, 0 to disable */ 446 u8 num_sack_blocks; /* number of SACK blocks to include */ 447 u8 hash_size; /* bytes in hash_location */ 448 u8 bpf_opt_len; /* length of BPF hdr option */ 449 __u8 *hash_location; /* temporary pointer, overloaded */ 450 __u32 tsval, tsecr; /* need to include OPTION_TS */ 451 struct tcp_fastopen_cookie *fastopen_cookie; /* Fast open cookie */ 452 struct mptcp_out_options mptcp; 453 }; 454 455 static void mptcp_options_write(struct tcphdr *th, __be32 *ptr, 456 struct tcp_sock *tp, 457 struct tcp_out_options *opts) 458 { 459 #if IS_ENABLED(CONFIG_MPTCP) 460 if (unlikely(OPTION_MPTCP & opts->options)) 461 mptcp_write_options(th, ptr, tp, &opts->mptcp); 462 #endif 463 } 464 465 #ifdef CONFIG_CGROUP_BPF 466 static int bpf_skops_write_hdr_opt_arg0(struct sk_buff *skb, 467 enum tcp_synack_type synack_type) 468 { 469 if (unlikely(!skb)) 470 return BPF_WRITE_HDR_TCP_CURRENT_MSS; 471 472 if (unlikely(synack_type == TCP_SYNACK_COOKIE)) 473 return BPF_WRITE_HDR_TCP_SYNACK_COOKIE; 474 475 return 0; 476 } 477 478 /* req, syn_skb and synack_type are used when writing synack */ 479 static void bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb, 480 struct request_sock *req, 481 struct sk_buff *syn_skb, 482 enum tcp_synack_type synack_type, 483 struct tcp_out_options *opts, 484 unsigned int *remaining) 485 { 486 struct bpf_sock_ops_kern sock_ops; 487 int err; 488 489 if (likely(!BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), 490 BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG)) || 491 !*remaining) 492 return; 493 494 /* *remaining has already been aligned to 4 bytes, so *remaining >= 4 */ 495 496 /* init sock_ops */ 497 memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); 498 499 sock_ops.op = BPF_SOCK_OPS_HDR_OPT_LEN_CB; 500 501 if (req) { 502 /* The listen "sk" cannot be passed here because 503 * it is not locked. It would not make too much 504 * sense to do bpf_setsockopt(listen_sk) based 505 * on individual connection request also. 506 * 507 * Thus, "req" is passed here and the cgroup-bpf-progs 508 * of the listen "sk" will be run. 509 * 510 * "req" is also used here for fastopen even the "sk" here is 511 * a fullsock "child" sk. It is to keep the behavior 512 * consistent between fastopen and non-fastopen on 513 * the bpf programming side. 514 */ 515 sock_ops.sk = (struct sock *)req; 516 sock_ops.syn_skb = syn_skb; 517 } else { 518 sock_owned_by_me(sk); 519 520 sock_ops.is_fullsock = 1; 521 sock_ops.sk = sk; 522 } 523 524 sock_ops.args[0] = bpf_skops_write_hdr_opt_arg0(skb, synack_type); 525 sock_ops.remaining_opt_len = *remaining; 526 /* tcp_current_mss() does not pass a skb */ 527 if (skb) 528 bpf_skops_init_skb(&sock_ops, skb, 0); 529 530 err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk); 531 532 if (err || sock_ops.remaining_opt_len == *remaining) 533 return; 534 535 opts->bpf_opt_len = *remaining - sock_ops.remaining_opt_len; 536 /* round up to 4 bytes */ 537 opts->bpf_opt_len = (opts->bpf_opt_len + 3) & ~3; 538 539 *remaining -= opts->bpf_opt_len; 540 } 541 542 static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb, 543 struct request_sock *req, 544 struct sk_buff *syn_skb, 545 enum tcp_synack_type synack_type, 546 struct tcp_out_options *opts) 547 { 548 u8 first_opt_off, nr_written, max_opt_len = opts->bpf_opt_len; 549 struct bpf_sock_ops_kern sock_ops; 550 int err; 551 552 if (likely(!max_opt_len)) 553 return; 554 555 memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); 556 557 sock_ops.op = BPF_SOCK_OPS_WRITE_HDR_OPT_CB; 558 559 if (req) { 560 sock_ops.sk = (struct sock *)req; 561 sock_ops.syn_skb = syn_skb; 562 } else { 563 sock_owned_by_me(sk); 564 565 sock_ops.is_fullsock = 1; 566 sock_ops.sk = sk; 567 } 568 569 sock_ops.args[0] = bpf_skops_write_hdr_opt_arg0(skb, synack_type); 570 sock_ops.remaining_opt_len = max_opt_len; 571 first_opt_off = tcp_hdrlen(skb) - max_opt_len; 572 bpf_skops_init_skb(&sock_ops, skb, first_opt_off); 573 574 err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk); 575 576 if (err) 577 nr_written = 0; 578 else 579 nr_written = max_opt_len - sock_ops.remaining_opt_len; 580 581 if (nr_written < max_opt_len) 582 memset(skb->data + first_opt_off + nr_written, TCPOPT_NOP, 583 max_opt_len - nr_written); 584 } 585 #else 586 static void bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb, 587 struct request_sock *req, 588 struct sk_buff *syn_skb, 589 enum tcp_synack_type synack_type, 590 struct tcp_out_options *opts, 591 unsigned int *remaining) 592 { 593 } 594 595 static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb, 596 struct request_sock *req, 597 struct sk_buff *syn_skb, 598 enum tcp_synack_type synack_type, 599 struct tcp_out_options *opts) 600 { 601 } 602 #endif 603 604 static __be32 *process_tcp_ao_options(struct tcp_sock *tp, 605 const struct tcp_request_sock *tcprsk, 606 struct tcp_out_options *opts, 607 struct tcp_key *key, __be32 *ptr) 608 { 609 #ifdef CONFIG_TCP_AO 610 u8 maclen = tcp_ao_maclen(key->ao_key); 611 612 if (tcprsk) { 613 u8 aolen = maclen + sizeof(struct tcp_ao_hdr); 614 615 *ptr++ = htonl((TCPOPT_AO << 24) | (aolen << 16) | 616 (tcprsk->ao_keyid << 8) | 617 (tcprsk->ao_rcv_next)); 618 } else { 619 struct tcp_ao_key *rnext_key; 620 struct tcp_ao_info *ao_info; 621 622 ao_info = rcu_dereference_check(tp->ao_info, 623 lockdep_sock_is_held(&tp->inet_conn.icsk_inet.sk)); 624 rnext_key = READ_ONCE(ao_info->rnext_key); 625 if (WARN_ON_ONCE(!rnext_key)) 626 return ptr; 627 *ptr++ = htonl((TCPOPT_AO << 24) | 628 (tcp_ao_len(key->ao_key) << 16) | 629 (key->ao_key->sndid << 8) | 630 (rnext_key->rcvid)); 631 } 632 opts->hash_location = (__u8 *)ptr; 633 ptr += maclen / sizeof(*ptr); 634 if (unlikely(maclen % sizeof(*ptr))) { 635 memset(ptr, TCPOPT_NOP, sizeof(*ptr)); 636 ptr++; 637 } 638 #endif 639 return ptr; 640 } 641 642 /* Write previously computed TCP options to the packet. 643 * 644 * Beware: Something in the Internet is very sensitive to the ordering of 645 * TCP options, we learned this through the hard way, so be careful here. 646 * Luckily we can at least blame others for their non-compliance but from 647 * inter-operability perspective it seems that we're somewhat stuck with 648 * the ordering which we have been using if we want to keep working with 649 * those broken things (not that it currently hurts anybody as there isn't 650 * particular reason why the ordering would need to be changed). 651 * 652 * At least SACK_PERM as the first option is known to lead to a disaster 653 * (but it may well be that other scenarios fail similarly). 654 */ 655 static void tcp_options_write(struct tcphdr *th, struct tcp_sock *tp, 656 const struct tcp_request_sock *tcprsk, 657 struct tcp_out_options *opts, 658 struct tcp_key *key) 659 { 660 __be32 *ptr = (__be32 *)(th + 1); 661 u16 options = opts->options; /* mungable copy */ 662 663 if (tcp_key_is_md5(key)) { 664 *ptr++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | 665 (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG); 666 /* overload cookie hash location */ 667 opts->hash_location = (__u8 *)ptr; 668 ptr += 4; 669 } else if (tcp_key_is_ao(key)) { 670 ptr = process_tcp_ao_options(tp, tcprsk, opts, key, ptr); 671 } 672 if (unlikely(opts->mss)) { 673 *ptr++ = htonl((TCPOPT_MSS << 24) | 674 (TCPOLEN_MSS << 16) | 675 opts->mss); 676 } 677 678 if (likely(OPTION_TS & options)) { 679 if (unlikely(OPTION_SACK_ADVERTISE & options)) { 680 *ptr++ = htonl((TCPOPT_SACK_PERM << 24) | 681 (TCPOLEN_SACK_PERM << 16) | 682 (TCPOPT_TIMESTAMP << 8) | 683 TCPOLEN_TIMESTAMP); 684 options &= ~OPTION_SACK_ADVERTISE; 685 } else { 686 *ptr++ = htonl((TCPOPT_NOP << 24) | 687 (TCPOPT_NOP << 16) | 688 (TCPOPT_TIMESTAMP << 8) | 689 TCPOLEN_TIMESTAMP); 690 } 691 *ptr++ = htonl(opts->tsval); 692 *ptr++ = htonl(opts->tsecr); 693 } 694 695 if (unlikely(OPTION_SACK_ADVERTISE & options)) { 696 *ptr++ = htonl((TCPOPT_NOP << 24) | 697 (TCPOPT_NOP << 16) | 698 (TCPOPT_SACK_PERM << 8) | 699 TCPOLEN_SACK_PERM); 700 } 701 702 if (unlikely(OPTION_WSCALE & options)) { 703 *ptr++ = htonl((TCPOPT_NOP << 24) | 704 (TCPOPT_WINDOW << 16) | 705 (TCPOLEN_WINDOW << 8) | 706 opts->ws); 707 } 708 709 if (unlikely(opts->num_sack_blocks)) { 710 struct tcp_sack_block *sp = tp->rx_opt.dsack ? 711 tp->duplicate_sack : tp->selective_acks; 712 int this_sack; 713 714 *ptr++ = htonl((TCPOPT_NOP << 24) | 715 (TCPOPT_NOP << 16) | 716 (TCPOPT_SACK << 8) | 717 (TCPOLEN_SACK_BASE + (opts->num_sack_blocks * 718 TCPOLEN_SACK_PERBLOCK))); 719 720 for (this_sack = 0; this_sack < opts->num_sack_blocks; 721 ++this_sack) { 722 *ptr++ = htonl(sp[this_sack].start_seq); 723 *ptr++ = htonl(sp[this_sack].end_seq); 724 } 725 726 tp->rx_opt.dsack = 0; 727 } 728 729 if (unlikely(OPTION_FAST_OPEN_COOKIE & options)) { 730 struct tcp_fastopen_cookie *foc = opts->fastopen_cookie; 731 u8 *p = (u8 *)ptr; 732 u32 len; /* Fast Open option length */ 733 734 if (foc->exp) { 735 len = TCPOLEN_EXP_FASTOPEN_BASE + foc->len; 736 *ptr = htonl((TCPOPT_EXP << 24) | (len << 16) | 737 TCPOPT_FASTOPEN_MAGIC); 738 p += TCPOLEN_EXP_FASTOPEN_BASE; 739 } else { 740 len = TCPOLEN_FASTOPEN_BASE + foc->len; 741 *p++ = TCPOPT_FASTOPEN; 742 *p++ = len; 743 } 744 745 memcpy(p, foc->val, foc->len); 746 if ((len & 3) == 2) { 747 p[foc->len] = TCPOPT_NOP; 748 p[foc->len + 1] = TCPOPT_NOP; 749 } 750 ptr += (len + 3) >> 2; 751 } 752 753 smc_options_write(ptr, &options); 754 755 mptcp_options_write(th, ptr, tp, opts); 756 } 757 758 static void smc_set_option(const struct tcp_sock *tp, 759 struct tcp_out_options *opts, 760 unsigned int *remaining) 761 { 762 #if IS_ENABLED(CONFIG_SMC) 763 if (static_branch_unlikely(&tcp_have_smc)) { 764 if (tp->syn_smc) { 765 if (*remaining >= TCPOLEN_EXP_SMC_BASE_ALIGNED) { 766 opts->options |= OPTION_SMC; 767 *remaining -= TCPOLEN_EXP_SMC_BASE_ALIGNED; 768 } 769 } 770 } 771 #endif 772 } 773 774 static void smc_set_option_cond(const struct tcp_sock *tp, 775 const struct inet_request_sock *ireq, 776 struct tcp_out_options *opts, 777 unsigned int *remaining) 778 { 779 #if IS_ENABLED(CONFIG_SMC) 780 if (static_branch_unlikely(&tcp_have_smc)) { 781 if (tp->syn_smc && ireq->smc_ok) { 782 if (*remaining >= TCPOLEN_EXP_SMC_BASE_ALIGNED) { 783 opts->options |= OPTION_SMC; 784 *remaining -= TCPOLEN_EXP_SMC_BASE_ALIGNED; 785 } 786 } 787 } 788 #endif 789 } 790 791 static void mptcp_set_option_cond(const struct request_sock *req, 792 struct tcp_out_options *opts, 793 unsigned int *remaining) 794 { 795 if (rsk_is_mptcp(req)) { 796 unsigned int size; 797 798 if (mptcp_synack_options(req, &size, &opts->mptcp)) { 799 if (*remaining >= size) { 800 opts->options |= OPTION_MPTCP; 801 *remaining -= size; 802 } 803 } 804 } 805 } 806 807 /* Compute TCP options for SYN packets. This is not the final 808 * network wire format yet. 809 */ 810 static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb, 811 struct tcp_out_options *opts, 812 struct tcp_key *key) 813 { 814 struct tcp_sock *tp = tcp_sk(sk); 815 unsigned int remaining = MAX_TCP_OPTION_SPACE; 816 struct tcp_fastopen_request *fastopen = tp->fastopen_req; 817 bool timestamps; 818 819 /* Better than switch (key.type) as it has static branches */ 820 if (tcp_key_is_md5(key)) { 821 timestamps = false; 822 opts->options |= OPTION_MD5; 823 remaining -= TCPOLEN_MD5SIG_ALIGNED; 824 } else { 825 timestamps = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_timestamps); 826 if (tcp_key_is_ao(key)) { 827 opts->options |= OPTION_AO; 828 remaining -= tcp_ao_len_aligned(key->ao_key); 829 } 830 } 831 832 /* We always get an MSS option. The option bytes which will be seen in 833 * normal data packets should timestamps be used, must be in the MSS 834 * advertised. But we subtract them from tp->mss_cache so that 835 * calculations in tcp_sendmsg are simpler etc. So account for this 836 * fact here if necessary. If we don't do this correctly, as a 837 * receiver we won't recognize data packets as being full sized when we 838 * should, and thus we won't abide by the delayed ACK rules correctly. 839 * SACKs don't matter, we never delay an ACK when we have any of those 840 * going out. */ 841 opts->mss = tcp_advertise_mss(sk); 842 remaining -= TCPOLEN_MSS_ALIGNED; 843 844 if (likely(timestamps)) { 845 opts->options |= OPTION_TS; 846 opts->tsval = tcp_skb_timestamp_ts(tp->tcp_usec_ts, skb) + tp->tsoffset; 847 opts->tsecr = tp->rx_opt.ts_recent; 848 remaining -= TCPOLEN_TSTAMP_ALIGNED; 849 } 850 if (likely(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_window_scaling))) { 851 opts->ws = tp->rx_opt.rcv_wscale; 852 opts->options |= OPTION_WSCALE; 853 remaining -= TCPOLEN_WSCALE_ALIGNED; 854 } 855 if (likely(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_sack))) { 856 opts->options |= OPTION_SACK_ADVERTISE; 857 if (unlikely(!(OPTION_TS & opts->options))) 858 remaining -= TCPOLEN_SACKPERM_ALIGNED; 859 } 860 861 if (fastopen && fastopen->cookie.len >= 0) { 862 u32 need = fastopen->cookie.len; 863 864 need += fastopen->cookie.exp ? TCPOLEN_EXP_FASTOPEN_BASE : 865 TCPOLEN_FASTOPEN_BASE; 866 need = (need + 3) & ~3U; /* Align to 32 bits */ 867 if (remaining >= need) { 868 opts->options |= OPTION_FAST_OPEN_COOKIE; 869 opts->fastopen_cookie = &fastopen->cookie; 870 remaining -= need; 871 tp->syn_fastopen = 1; 872 tp->syn_fastopen_exp = fastopen->cookie.exp ? 1 : 0; 873 } 874 } 875 876 smc_set_option(tp, opts, &remaining); 877 878 if (sk_is_mptcp(sk)) { 879 unsigned int size; 880 881 if (mptcp_syn_options(sk, skb, &size, &opts->mptcp)) { 882 opts->options |= OPTION_MPTCP; 883 remaining -= size; 884 } 885 } 886 887 bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, &remaining); 888 889 return MAX_TCP_OPTION_SPACE - remaining; 890 } 891 892 /* Set up TCP options for SYN-ACKs. */ 893 static unsigned int tcp_synack_options(const struct sock *sk, 894 struct request_sock *req, 895 unsigned int mss, struct sk_buff *skb, 896 struct tcp_out_options *opts, 897 const struct tcp_key *key, 898 struct tcp_fastopen_cookie *foc, 899 enum tcp_synack_type synack_type, 900 struct sk_buff *syn_skb) 901 { 902 struct inet_request_sock *ireq = inet_rsk(req); 903 unsigned int remaining = MAX_TCP_OPTION_SPACE; 904 905 if (tcp_key_is_md5(key)) { 906 opts->options |= OPTION_MD5; 907 remaining -= TCPOLEN_MD5SIG_ALIGNED; 908 909 /* We can't fit any SACK blocks in a packet with MD5 + TS 910 * options. There was discussion about disabling SACK 911 * rather than TS in order to fit in better with old, 912 * buggy kernels, but that was deemed to be unnecessary. 913 */ 914 if (synack_type != TCP_SYNACK_COOKIE) 915 ireq->tstamp_ok &= !ireq->sack_ok; 916 } else if (tcp_key_is_ao(key)) { 917 opts->options |= OPTION_AO; 918 remaining -= tcp_ao_len_aligned(key->ao_key); 919 ireq->tstamp_ok &= !ireq->sack_ok; 920 } 921 922 /* We always send an MSS option. */ 923 opts->mss = mss; 924 remaining -= TCPOLEN_MSS_ALIGNED; 925 926 if (likely(ireq->wscale_ok)) { 927 opts->ws = ireq->rcv_wscale; 928 opts->options |= OPTION_WSCALE; 929 remaining -= TCPOLEN_WSCALE_ALIGNED; 930 } 931 if (likely(ireq->tstamp_ok)) { 932 opts->options |= OPTION_TS; 933 opts->tsval = tcp_skb_timestamp_ts(tcp_rsk(req)->req_usec_ts, skb) + 934 tcp_rsk(req)->ts_off; 935 opts->tsecr = READ_ONCE(req->ts_recent); 936 remaining -= TCPOLEN_TSTAMP_ALIGNED; 937 } 938 if (likely(ireq->sack_ok)) { 939 opts->options |= OPTION_SACK_ADVERTISE; 940 if (unlikely(!ireq->tstamp_ok)) 941 remaining -= TCPOLEN_SACKPERM_ALIGNED; 942 } 943 if (foc != NULL && foc->len >= 0) { 944 u32 need = foc->len; 945 946 need += foc->exp ? TCPOLEN_EXP_FASTOPEN_BASE : 947 TCPOLEN_FASTOPEN_BASE; 948 need = (need + 3) & ~3U; /* Align to 32 bits */ 949 if (remaining >= need) { 950 opts->options |= OPTION_FAST_OPEN_COOKIE; 951 opts->fastopen_cookie = foc; 952 remaining -= need; 953 } 954 } 955 956 mptcp_set_option_cond(req, opts, &remaining); 957 958 smc_set_option_cond(tcp_sk(sk), ireq, opts, &remaining); 959 960 bpf_skops_hdr_opt_len((struct sock *)sk, skb, req, syn_skb, 961 synack_type, opts, &remaining); 962 963 return MAX_TCP_OPTION_SPACE - remaining; 964 } 965 966 /* Compute TCP options for ESTABLISHED sockets. This is not the 967 * final wire format yet. 968 */ 969 static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb, 970 struct tcp_out_options *opts, 971 struct tcp_key *key) 972 { 973 struct tcp_sock *tp = tcp_sk(sk); 974 unsigned int size = 0; 975 unsigned int eff_sacks; 976 977 opts->options = 0; 978 979 /* Better than switch (key.type) as it has static branches */ 980 if (tcp_key_is_md5(key)) { 981 opts->options |= OPTION_MD5; 982 size += TCPOLEN_MD5SIG_ALIGNED; 983 } else if (tcp_key_is_ao(key)) { 984 opts->options |= OPTION_AO; 985 size += tcp_ao_len_aligned(key->ao_key); 986 } 987 988 if (likely(tp->rx_opt.tstamp_ok)) { 989 opts->options |= OPTION_TS; 990 opts->tsval = skb ? tcp_skb_timestamp_ts(tp->tcp_usec_ts, skb) + 991 tp->tsoffset : 0; 992 opts->tsecr = tp->rx_opt.ts_recent; 993 size += TCPOLEN_TSTAMP_ALIGNED; 994 } 995 996 /* MPTCP options have precedence over SACK for the limited TCP 997 * option space because a MPTCP connection would be forced to 998 * fall back to regular TCP if a required multipath option is 999 * missing. SACK still gets a chance to use whatever space is 1000 * left. 1001 */ 1002 if (sk_is_mptcp(sk)) { 1003 unsigned int remaining = MAX_TCP_OPTION_SPACE - size; 1004 unsigned int opt_size = 0; 1005 1006 if (mptcp_established_options(sk, skb, &opt_size, remaining, 1007 &opts->mptcp)) { 1008 opts->options |= OPTION_MPTCP; 1009 size += opt_size; 1010 } 1011 } 1012 1013 eff_sacks = tp->rx_opt.num_sacks + tp->rx_opt.dsack; 1014 if (unlikely(eff_sacks)) { 1015 const unsigned int remaining = MAX_TCP_OPTION_SPACE - size; 1016 if (unlikely(remaining < TCPOLEN_SACK_BASE_ALIGNED + 1017 TCPOLEN_SACK_PERBLOCK)) 1018 return size; 1019 1020 opts->num_sack_blocks = 1021 min_t(unsigned int, eff_sacks, 1022 (remaining - TCPOLEN_SACK_BASE_ALIGNED) / 1023 TCPOLEN_SACK_PERBLOCK); 1024 1025 size += TCPOLEN_SACK_BASE_ALIGNED + 1026 opts->num_sack_blocks * TCPOLEN_SACK_PERBLOCK; 1027 } 1028 1029 if (unlikely(BPF_SOCK_OPS_TEST_FLAG(tp, 1030 BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG))) { 1031 unsigned int remaining = MAX_TCP_OPTION_SPACE - size; 1032 1033 bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, &remaining); 1034 1035 size = MAX_TCP_OPTION_SPACE - remaining; 1036 } 1037 1038 return size; 1039 } 1040 1041 1042 /* TCP SMALL QUEUES (TSQ) 1043 * 1044 * TSQ goal is to keep small amount of skbs per tcp flow in tx queues (qdisc+dev) 1045 * to reduce RTT and bufferbloat. 1046 * We do this using a special skb destructor (tcp_wfree). 1047 * 1048 * Its important tcp_wfree() can be replaced by sock_wfree() in the event skb 1049 * needs to be reallocated in a driver. 1050 * The invariant being skb->truesize subtracted from sk->sk_wmem_alloc 1051 * 1052 * Since transmit from skb destructor is forbidden, we use a tasklet 1053 * to process all sockets that eventually need to send more skbs. 1054 * We use one tasklet per cpu, with its own queue of sockets. 1055 */ 1056 struct tsq_tasklet { 1057 struct tasklet_struct tasklet; 1058 struct list_head head; /* queue of tcp sockets */ 1059 }; 1060 static DEFINE_PER_CPU(struct tsq_tasklet, tsq_tasklet); 1061 1062 static void tcp_tsq_write(struct sock *sk) 1063 { 1064 if ((1 << sk->sk_state) & 1065 (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_CLOSING | 1066 TCPF_CLOSE_WAIT | TCPF_LAST_ACK)) { 1067 struct tcp_sock *tp = tcp_sk(sk); 1068 1069 if (tp->lost_out > tp->retrans_out && 1070 tcp_snd_cwnd(tp) > tcp_packets_in_flight(tp)) { 1071 tcp_mstamp_refresh(tp); 1072 tcp_xmit_retransmit_queue(sk); 1073 } 1074 1075 tcp_write_xmit(sk, tcp_current_mss(sk), tp->nonagle, 1076 0, GFP_ATOMIC); 1077 } 1078 } 1079 1080 static void tcp_tsq_handler(struct sock *sk) 1081 { 1082 bh_lock_sock(sk); 1083 if (!sock_owned_by_user(sk)) 1084 tcp_tsq_write(sk); 1085 else if (!test_and_set_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags)) 1086 sock_hold(sk); 1087 bh_unlock_sock(sk); 1088 } 1089 /* 1090 * One tasklet per cpu tries to send more skbs. 1091 * We run in tasklet context but need to disable irqs when 1092 * transferring tsq->head because tcp_wfree() might 1093 * interrupt us (non NAPI drivers) 1094 */ 1095 static void tcp_tasklet_func(struct tasklet_struct *t) 1096 { 1097 struct tsq_tasklet *tsq = from_tasklet(tsq, t, tasklet); 1098 LIST_HEAD(list); 1099 unsigned long flags; 1100 struct list_head *q, *n; 1101 struct tcp_sock *tp; 1102 struct sock *sk; 1103 1104 local_irq_save(flags); 1105 list_splice_init(&tsq->head, &list); 1106 local_irq_restore(flags); 1107 1108 list_for_each_safe(q, n, &list) { 1109 tp = list_entry(q, struct tcp_sock, tsq_node); 1110 list_del(&tp->tsq_node); 1111 1112 sk = (struct sock *)tp; 1113 smp_mb__before_atomic(); 1114 clear_bit(TSQ_QUEUED, &sk->sk_tsq_flags); 1115 1116 tcp_tsq_handler(sk); 1117 sk_free(sk); 1118 } 1119 } 1120 1121 #define TCP_DEFERRED_ALL (TCPF_TSQ_DEFERRED | \ 1122 TCPF_WRITE_TIMER_DEFERRED | \ 1123 TCPF_DELACK_TIMER_DEFERRED | \ 1124 TCPF_MTU_REDUCED_DEFERRED | \ 1125 TCPF_ACK_DEFERRED) 1126 /** 1127 * tcp_release_cb - tcp release_sock() callback 1128 * @sk: socket 1129 * 1130 * called from release_sock() to perform protocol dependent 1131 * actions before socket release. 1132 */ 1133 void tcp_release_cb(struct sock *sk) 1134 { 1135 unsigned long flags = smp_load_acquire(&sk->sk_tsq_flags); 1136 unsigned long nflags; 1137 1138 /* perform an atomic operation only if at least one flag is set */ 1139 do { 1140 if (!(flags & TCP_DEFERRED_ALL)) 1141 return; 1142 nflags = flags & ~TCP_DEFERRED_ALL; 1143 } while (!try_cmpxchg(&sk->sk_tsq_flags, &flags, nflags)); 1144 1145 if (flags & TCPF_TSQ_DEFERRED) { 1146 tcp_tsq_write(sk); 1147 __sock_put(sk); 1148 } 1149 1150 if (flags & TCPF_WRITE_TIMER_DEFERRED) { 1151 tcp_write_timer_handler(sk); 1152 __sock_put(sk); 1153 } 1154 if (flags & TCPF_DELACK_TIMER_DEFERRED) { 1155 tcp_delack_timer_handler(sk); 1156 __sock_put(sk); 1157 } 1158 if (flags & TCPF_MTU_REDUCED_DEFERRED) { 1159 inet_csk(sk)->icsk_af_ops->mtu_reduced(sk); 1160 __sock_put(sk); 1161 } 1162 if ((flags & TCPF_ACK_DEFERRED) && inet_csk_ack_scheduled(sk)) 1163 tcp_send_ack(sk); 1164 } 1165 EXPORT_SYMBOL(tcp_release_cb); 1166 1167 void __init tcp_tasklet_init(void) 1168 { 1169 int i; 1170 1171 for_each_possible_cpu(i) { 1172 struct tsq_tasklet *tsq = &per_cpu(tsq_tasklet, i); 1173 1174 INIT_LIST_HEAD(&tsq->head); 1175 tasklet_setup(&tsq->tasklet, tcp_tasklet_func); 1176 } 1177 } 1178 1179 /* 1180 * Write buffer destructor automatically called from kfree_skb. 1181 * We can't xmit new skbs from this context, as we might already 1182 * hold qdisc lock. 1183 */ 1184 void tcp_wfree(struct sk_buff *skb) 1185 { 1186 struct sock *sk = skb->sk; 1187 struct tcp_sock *tp = tcp_sk(sk); 1188 unsigned long flags, nval, oval; 1189 struct tsq_tasklet *tsq; 1190 bool empty; 1191 1192 /* Keep one reference on sk_wmem_alloc. 1193 * Will be released by sk_free() from here or tcp_tasklet_func() 1194 */ 1195 WARN_ON(refcount_sub_and_test(skb->truesize - 1, &sk->sk_wmem_alloc)); 1196 1197 /* If this softirq is serviced by ksoftirqd, we are likely under stress. 1198 * Wait until our queues (qdisc + devices) are drained. 1199 * This gives : 1200 * - less callbacks to tcp_write_xmit(), reducing stress (batches) 1201 * - chance for incoming ACK (processed by another cpu maybe) 1202 * to migrate this flow (skb->ooo_okay will be eventually set) 1203 */ 1204 if (refcount_read(&sk->sk_wmem_alloc) >= SKB_TRUESIZE(1) && this_cpu_ksoftirqd() == current) 1205 goto out; 1206 1207 oval = smp_load_acquire(&sk->sk_tsq_flags); 1208 do { 1209 if (!(oval & TSQF_THROTTLED) || (oval & TSQF_QUEUED)) 1210 goto out; 1211 1212 nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED; 1213 } while (!try_cmpxchg(&sk->sk_tsq_flags, &oval, nval)); 1214 1215 /* queue this socket to tasklet queue */ 1216 local_irq_save(flags); 1217 tsq = this_cpu_ptr(&tsq_tasklet); 1218 empty = list_empty(&tsq->head); 1219 list_add(&tp->tsq_node, &tsq->head); 1220 if (empty) 1221 tasklet_schedule(&tsq->tasklet); 1222 local_irq_restore(flags); 1223 return; 1224 out: 1225 sk_free(sk); 1226 } 1227 1228 /* Note: Called under soft irq. 1229 * We can call TCP stack right away, unless socket is owned by user. 1230 */ 1231 enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer) 1232 { 1233 struct tcp_sock *tp = container_of(timer, struct tcp_sock, pacing_timer); 1234 struct sock *sk = (struct sock *)tp; 1235 1236 tcp_tsq_handler(sk); 1237 sock_put(sk); 1238 1239 return HRTIMER_NORESTART; 1240 } 1241 1242 static void tcp_update_skb_after_send(struct sock *sk, struct sk_buff *skb, 1243 u64 prior_wstamp) 1244 { 1245 struct tcp_sock *tp = tcp_sk(sk); 1246 1247 if (sk->sk_pacing_status != SK_PACING_NONE) { 1248 unsigned long rate = READ_ONCE(sk->sk_pacing_rate); 1249 1250 /* Original sch_fq does not pace first 10 MSS 1251 * Note that tp->data_segs_out overflows after 2^32 packets, 1252 * this is a minor annoyance. 1253 */ 1254 if (rate != ~0UL && rate && tp->data_segs_out >= 10) { 1255 u64 len_ns = div64_ul((u64)skb->len * NSEC_PER_SEC, rate); 1256 u64 credit = tp->tcp_wstamp_ns - prior_wstamp; 1257 1258 /* take into account OS jitter */ 1259 len_ns -= min_t(u64, len_ns / 2, credit); 1260 tp->tcp_wstamp_ns += len_ns; 1261 } 1262 } 1263 list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue); 1264 } 1265 1266 INDIRECT_CALLABLE_DECLARE(int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl)); 1267 INDIRECT_CALLABLE_DECLARE(int inet6_csk_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl)); 1268 INDIRECT_CALLABLE_DECLARE(void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb)); 1269 1270 /* This routine actually transmits TCP packets queued in by 1271 * tcp_do_sendmsg(). This is used by both the initial 1272 * transmission and possible later retransmissions. 1273 * All SKB's seen here are completely headerless. It is our 1274 * job to build the TCP header, and pass the packet down to 1275 * IP so it can do the same plus pass the packet off to the 1276 * device. 1277 * 1278 * We are working here with either a clone of the original 1279 * SKB, or a fresh unique copy made by the retransmit engine. 1280 */ 1281 static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, 1282 int clone_it, gfp_t gfp_mask, u32 rcv_nxt) 1283 { 1284 const struct inet_connection_sock *icsk = inet_csk(sk); 1285 struct inet_sock *inet; 1286 struct tcp_sock *tp; 1287 struct tcp_skb_cb *tcb; 1288 struct tcp_out_options opts; 1289 unsigned int tcp_options_size, tcp_header_size; 1290 struct sk_buff *oskb = NULL; 1291 struct tcp_key key; 1292 struct tcphdr *th; 1293 u64 prior_wstamp; 1294 int err; 1295 1296 BUG_ON(!skb || !tcp_skb_pcount(skb)); 1297 tp = tcp_sk(sk); 1298 prior_wstamp = tp->tcp_wstamp_ns; 1299 tp->tcp_wstamp_ns = max(tp->tcp_wstamp_ns, tp->tcp_clock_cache); 1300 skb_set_delivery_time(skb, tp->tcp_wstamp_ns, true); 1301 if (clone_it) { 1302 oskb = skb; 1303 1304 tcp_skb_tsorted_save(oskb) { 1305 if (unlikely(skb_cloned(oskb))) 1306 skb = pskb_copy(oskb, gfp_mask); 1307 else 1308 skb = skb_clone(oskb, gfp_mask); 1309 } tcp_skb_tsorted_restore(oskb); 1310 1311 if (unlikely(!skb)) 1312 return -ENOBUFS; 1313 /* retransmit skbs might have a non zero value in skb->dev 1314 * because skb->dev is aliased with skb->rbnode.rb_left 1315 */ 1316 skb->dev = NULL; 1317 } 1318 1319 inet = inet_sk(sk); 1320 tcb = TCP_SKB_CB(skb); 1321 memset(&opts, 0, sizeof(opts)); 1322 1323 tcp_get_current_key(sk, &key); 1324 if (unlikely(tcb->tcp_flags & TCPHDR_SYN)) { 1325 tcp_options_size = tcp_syn_options(sk, skb, &opts, &key); 1326 } else { 1327 tcp_options_size = tcp_established_options(sk, skb, &opts, &key); 1328 /* Force a PSH flag on all (GSO) packets to expedite GRO flush 1329 * at receiver : This slightly improve GRO performance. 1330 * Note that we do not force the PSH flag for non GSO packets, 1331 * because they might be sent under high congestion events, 1332 * and in this case it is better to delay the delivery of 1-MSS 1333 * packets and thus the corresponding ACK packet that would 1334 * release the following packet. 1335 */ 1336 if (tcp_skb_pcount(skb) > 1) 1337 tcb->tcp_flags |= TCPHDR_PSH; 1338 } 1339 tcp_header_size = tcp_options_size + sizeof(struct tcphdr); 1340 1341 /* We set skb->ooo_okay to one if this packet can select 1342 * a different TX queue than prior packets of this flow, 1343 * to avoid self inflicted reorders. 1344 * The 'other' queue decision is based on current cpu number 1345 * if XPS is enabled, or sk->sk_txhash otherwise. 1346 * We can switch to another (and better) queue if: 1347 * 1) No packet with payload is in qdisc/device queues. 1348 * Delays in TX completion can defeat the test 1349 * even if packets were already sent. 1350 * 2) Or rtx queue is empty. 1351 * This mitigates above case if ACK packets for 1352 * all prior packets were already processed. 1353 */ 1354 skb->ooo_okay = sk_wmem_alloc_get(sk) < SKB_TRUESIZE(1) || 1355 tcp_rtx_queue_empty(sk); 1356 1357 /* If we had to use memory reserve to allocate this skb, 1358 * this might cause drops if packet is looped back : 1359 * Other socket might not have SOCK_MEMALLOC. 1360 * Packets not looped back do not care about pfmemalloc. 1361 */ 1362 skb->pfmemalloc = 0; 1363 1364 skb_push(skb, tcp_header_size); 1365 skb_reset_transport_header(skb); 1366 1367 skb_orphan(skb); 1368 skb->sk = sk; 1369 skb->destructor = skb_is_tcp_pure_ack(skb) ? __sock_wfree : tcp_wfree; 1370 refcount_add(skb->truesize, &sk->sk_wmem_alloc); 1371 1372 skb_set_dst_pending_confirm(skb, READ_ONCE(sk->sk_dst_pending_confirm)); 1373 1374 /* Build TCP header and checksum it. */ 1375 th = (struct tcphdr *)skb->data; 1376 th->source = inet->inet_sport; 1377 th->dest = inet->inet_dport; 1378 th->seq = htonl(tcb->seq); 1379 th->ack_seq = htonl(rcv_nxt); 1380 *(((__be16 *)th) + 6) = htons(((tcp_header_size >> 2) << 12) | 1381 tcb->tcp_flags); 1382 1383 th->check = 0; 1384 th->urg_ptr = 0; 1385 1386 /* The urg_mode check is necessary during a below snd_una win probe */ 1387 if (unlikely(tcp_urg_mode(tp) && before(tcb->seq, tp->snd_up))) { 1388 if (before(tp->snd_up, tcb->seq + 0x10000)) { 1389 th->urg_ptr = htons(tp->snd_up - tcb->seq); 1390 th->urg = 1; 1391 } else if (after(tcb->seq + 0xFFFF, tp->snd_nxt)) { 1392 th->urg_ptr = htons(0xFFFF); 1393 th->urg = 1; 1394 } 1395 } 1396 1397 skb_shinfo(skb)->gso_type = sk->sk_gso_type; 1398 if (likely(!(tcb->tcp_flags & TCPHDR_SYN))) { 1399 th->window = htons(tcp_select_window(sk)); 1400 tcp_ecn_send(sk, skb, th, tcp_header_size); 1401 } else { 1402 /* RFC1323: The window in SYN & SYN/ACK segments 1403 * is never scaled. 1404 */ 1405 th->window = htons(min(tp->rcv_wnd, 65535U)); 1406 } 1407 1408 tcp_options_write(th, tp, NULL, &opts, &key); 1409 1410 if (tcp_key_is_md5(&key)) { 1411 #ifdef CONFIG_TCP_MD5SIG 1412 /* Calculate the MD5 hash, as we have all we need now */ 1413 sk_gso_disable(sk); 1414 tp->af_specific->calc_md5_hash(opts.hash_location, 1415 key.md5_key, sk, skb); 1416 #endif 1417 } else if (tcp_key_is_ao(&key)) { 1418 int err; 1419 1420 err = tcp_ao_transmit_skb(sk, skb, key.ao_key, th, 1421 opts.hash_location); 1422 if (err) { 1423 kfree_skb_reason(skb, SKB_DROP_REASON_NOT_SPECIFIED); 1424 return -ENOMEM; 1425 } 1426 } 1427 1428 /* BPF prog is the last one writing header option */ 1429 bpf_skops_write_hdr_opt(sk, skb, NULL, NULL, 0, &opts); 1430 1431 INDIRECT_CALL_INET(icsk->icsk_af_ops->send_check, 1432 tcp_v6_send_check, tcp_v4_send_check, 1433 sk, skb); 1434 1435 if (likely(tcb->tcp_flags & TCPHDR_ACK)) 1436 tcp_event_ack_sent(sk, rcv_nxt); 1437 1438 if (skb->len != tcp_header_size) { 1439 tcp_event_data_sent(tp, sk); 1440 tp->data_segs_out += tcp_skb_pcount(skb); 1441 tp->bytes_sent += skb->len - tcp_header_size; 1442 } 1443 1444 if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq) 1445 TCP_ADD_STATS(sock_net(sk), TCP_MIB_OUTSEGS, 1446 tcp_skb_pcount(skb)); 1447 1448 tp->segs_out += tcp_skb_pcount(skb); 1449 skb_set_hash_from_sk(skb, sk); 1450 /* OK, its time to fill skb_shinfo(skb)->gso_{segs|size} */ 1451 skb_shinfo(skb)->gso_segs = tcp_skb_pcount(skb); 1452 skb_shinfo(skb)->gso_size = tcp_skb_mss(skb); 1453 1454 /* Leave earliest departure time in skb->tstamp (skb->skb_mstamp_ns) */ 1455 1456 /* Cleanup our debris for IP stacks */ 1457 memset(skb->cb, 0, max(sizeof(struct inet_skb_parm), 1458 sizeof(struct inet6_skb_parm))); 1459 1460 tcp_add_tx_delay(skb, tp); 1461 1462 err = INDIRECT_CALL_INET(icsk->icsk_af_ops->queue_xmit, 1463 inet6_csk_xmit, ip_queue_xmit, 1464 sk, skb, &inet->cork.fl); 1465 1466 if (unlikely(err > 0)) { 1467 tcp_enter_cwr(sk); 1468 err = net_xmit_eval(err); 1469 } 1470 if (!err && oskb) { 1471 tcp_update_skb_after_send(sk, oskb, prior_wstamp); 1472 tcp_rate_skb_sent(sk, oskb); 1473 } 1474 return err; 1475 } 1476 1477 static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it, 1478 gfp_t gfp_mask) 1479 { 1480 return __tcp_transmit_skb(sk, skb, clone_it, gfp_mask, 1481 tcp_sk(sk)->rcv_nxt); 1482 } 1483 1484 /* This routine just queues the buffer for sending. 1485 * 1486 * NOTE: probe0 timer is not checked, do not forget tcp_push_pending_frames, 1487 * otherwise socket can stall. 1488 */ 1489 static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb) 1490 { 1491 struct tcp_sock *tp = tcp_sk(sk); 1492 1493 /* Advance write_seq and place onto the write_queue. */ 1494 WRITE_ONCE(tp->write_seq, TCP_SKB_CB(skb)->end_seq); 1495 __skb_header_release(skb); 1496 tcp_add_write_queue_tail(sk, skb); 1497 sk_wmem_queued_add(sk, skb->truesize); 1498 sk_mem_charge(sk, skb->truesize); 1499 } 1500 1501 /* Initialize TSO segments for a packet. */ 1502 static void tcp_set_skb_tso_segs(struct sk_buff *skb, unsigned int mss_now) 1503 { 1504 if (skb->len <= mss_now) { 1505 /* Avoid the costly divide in the normal 1506 * non-TSO case. 1507 */ 1508 tcp_skb_pcount_set(skb, 1); 1509 TCP_SKB_CB(skb)->tcp_gso_size = 0; 1510 } else { 1511 tcp_skb_pcount_set(skb, DIV_ROUND_UP(skb->len, mss_now)); 1512 TCP_SKB_CB(skb)->tcp_gso_size = mss_now; 1513 } 1514 } 1515 1516 /* Pcount in the middle of the write queue got changed, we need to do various 1517 * tweaks to fix counters 1518 */ 1519 static void tcp_adjust_pcount(struct sock *sk, const struct sk_buff *skb, int decr) 1520 { 1521 struct tcp_sock *tp = tcp_sk(sk); 1522 1523 tp->packets_out -= decr; 1524 1525 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED) 1526 tp->sacked_out -= decr; 1527 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) 1528 tp->retrans_out -= decr; 1529 if (TCP_SKB_CB(skb)->sacked & TCPCB_LOST) 1530 tp->lost_out -= decr; 1531 1532 /* Reno case is special. Sigh... */ 1533 if (tcp_is_reno(tp) && decr > 0) 1534 tp->sacked_out -= min_t(u32, tp->sacked_out, decr); 1535 1536 if (tp->lost_skb_hint && 1537 before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(tp->lost_skb_hint)->seq) && 1538 (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)) 1539 tp->lost_cnt_hint -= decr; 1540 1541 tcp_verify_left_out(tp); 1542 } 1543 1544 static bool tcp_has_tx_tstamp(const struct sk_buff *skb) 1545 { 1546 return TCP_SKB_CB(skb)->txstamp_ack || 1547 (skb_shinfo(skb)->tx_flags & SKBTX_ANY_TSTAMP); 1548 } 1549 1550 static void tcp_fragment_tstamp(struct sk_buff *skb, struct sk_buff *skb2) 1551 { 1552 struct skb_shared_info *shinfo = skb_shinfo(skb); 1553 1554 if (unlikely(tcp_has_tx_tstamp(skb)) && 1555 !before(shinfo->tskey, TCP_SKB_CB(skb2)->seq)) { 1556 struct skb_shared_info *shinfo2 = skb_shinfo(skb2); 1557 u8 tsflags = shinfo->tx_flags & SKBTX_ANY_TSTAMP; 1558 1559 shinfo->tx_flags &= ~tsflags; 1560 shinfo2->tx_flags |= tsflags; 1561 swap(shinfo->tskey, shinfo2->tskey); 1562 TCP_SKB_CB(skb2)->txstamp_ack = TCP_SKB_CB(skb)->txstamp_ack; 1563 TCP_SKB_CB(skb)->txstamp_ack = 0; 1564 } 1565 } 1566 1567 static void tcp_skb_fragment_eor(struct sk_buff *skb, struct sk_buff *skb2) 1568 { 1569 TCP_SKB_CB(skb2)->eor = TCP_SKB_CB(skb)->eor; 1570 TCP_SKB_CB(skb)->eor = 0; 1571 } 1572 1573 /* Insert buff after skb on the write or rtx queue of sk. */ 1574 static void tcp_insert_write_queue_after(struct sk_buff *skb, 1575 struct sk_buff *buff, 1576 struct sock *sk, 1577 enum tcp_queue tcp_queue) 1578 { 1579 if (tcp_queue == TCP_FRAG_IN_WRITE_QUEUE) 1580 __skb_queue_after(&sk->sk_write_queue, skb, buff); 1581 else 1582 tcp_rbtree_insert(&sk->tcp_rtx_queue, buff); 1583 } 1584 1585 /* Function to create two new TCP segments. Shrinks the given segment 1586 * to the specified size and appends a new segment with the rest of the 1587 * packet to the list. This won't be called frequently, I hope. 1588 * Remember, these are still headerless SKBs at this point. 1589 */ 1590 int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, 1591 struct sk_buff *skb, u32 len, 1592 unsigned int mss_now, gfp_t gfp) 1593 { 1594 struct tcp_sock *tp = tcp_sk(sk); 1595 struct sk_buff *buff; 1596 int old_factor; 1597 long limit; 1598 int nlen; 1599 u8 flags; 1600 1601 if (WARN_ON(len > skb->len)) 1602 return -EINVAL; 1603 1604 DEBUG_NET_WARN_ON_ONCE(skb_headlen(skb)); 1605 1606 /* tcp_sendmsg() can overshoot sk_wmem_queued by one full size skb. 1607 * We need some allowance to not penalize applications setting small 1608 * SO_SNDBUF values. 1609 * Also allow first and last skb in retransmit queue to be split. 1610 */ 1611 limit = sk->sk_sndbuf + 2 * SKB_TRUESIZE(GSO_LEGACY_MAX_SIZE); 1612 if (unlikely((sk->sk_wmem_queued >> 1) > limit && 1613 tcp_queue != TCP_FRAG_IN_WRITE_QUEUE && 1614 skb != tcp_rtx_queue_head(sk) && 1615 skb != tcp_rtx_queue_tail(sk))) { 1616 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPWQUEUETOOBIG); 1617 return -ENOMEM; 1618 } 1619 1620 if (skb_unclone_keeptruesize(skb, gfp)) 1621 return -ENOMEM; 1622 1623 /* Get a new skb... force flag on. */ 1624 buff = tcp_stream_alloc_skb(sk, gfp, true); 1625 if (!buff) 1626 return -ENOMEM; /* We'll just try again later. */ 1627 skb_copy_decrypted(buff, skb); 1628 mptcp_skb_ext_copy(buff, skb); 1629 1630 sk_wmem_queued_add(sk, buff->truesize); 1631 sk_mem_charge(sk, buff->truesize); 1632 nlen = skb->len - len; 1633 buff->truesize += nlen; 1634 skb->truesize -= nlen; 1635 1636 /* Correct the sequence numbers. */ 1637 TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len; 1638 TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq; 1639 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq; 1640 1641 /* PSH and FIN should only be set in the second packet. */ 1642 flags = TCP_SKB_CB(skb)->tcp_flags; 1643 TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH); 1644 TCP_SKB_CB(buff)->tcp_flags = flags; 1645 TCP_SKB_CB(buff)->sacked = TCP_SKB_CB(skb)->sacked; 1646 tcp_skb_fragment_eor(skb, buff); 1647 1648 skb_split(skb, buff, len); 1649 1650 skb_set_delivery_time(buff, skb->tstamp, true); 1651 tcp_fragment_tstamp(skb, buff); 1652 1653 old_factor = tcp_skb_pcount(skb); 1654 1655 /* Fix up tso_factor for both original and new SKB. */ 1656 tcp_set_skb_tso_segs(skb, mss_now); 1657 tcp_set_skb_tso_segs(buff, mss_now); 1658 1659 /* Update delivered info for the new segment */ 1660 TCP_SKB_CB(buff)->tx = TCP_SKB_CB(skb)->tx; 1661 1662 /* If this packet has been sent out already, we must 1663 * adjust the various packet counters. 1664 */ 1665 if (!before(tp->snd_nxt, TCP_SKB_CB(buff)->end_seq)) { 1666 int diff = old_factor - tcp_skb_pcount(skb) - 1667 tcp_skb_pcount(buff); 1668 1669 if (diff) 1670 tcp_adjust_pcount(sk, skb, diff); 1671 } 1672 1673 /* Link BUFF into the send queue. */ 1674 __skb_header_release(buff); 1675 tcp_insert_write_queue_after(skb, buff, sk, tcp_queue); 1676 if (tcp_queue == TCP_FRAG_IN_RTX_QUEUE) 1677 list_add(&buff->tcp_tsorted_anchor, &skb->tcp_tsorted_anchor); 1678 1679 return 0; 1680 } 1681 1682 /* This is similar to __pskb_pull_tail(). The difference is that pulled 1683 * data is not copied, but immediately discarded. 1684 */ 1685 static int __pskb_trim_head(struct sk_buff *skb, int len) 1686 { 1687 struct skb_shared_info *shinfo; 1688 int i, k, eat; 1689 1690 DEBUG_NET_WARN_ON_ONCE(skb_headlen(skb)); 1691 eat = len; 1692 k = 0; 1693 shinfo = skb_shinfo(skb); 1694 for (i = 0; i < shinfo->nr_frags; i++) { 1695 int size = skb_frag_size(&shinfo->frags[i]); 1696 1697 if (size <= eat) { 1698 skb_frag_unref(skb, i); 1699 eat -= size; 1700 } else { 1701 shinfo->frags[k] = shinfo->frags[i]; 1702 if (eat) { 1703 skb_frag_off_add(&shinfo->frags[k], eat); 1704 skb_frag_size_sub(&shinfo->frags[k], eat); 1705 eat = 0; 1706 } 1707 k++; 1708 } 1709 } 1710 shinfo->nr_frags = k; 1711 1712 skb->data_len -= len; 1713 skb->len = skb->data_len; 1714 return len; 1715 } 1716 1717 /* Remove acked data from a packet in the transmit queue. */ 1718 int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len) 1719 { 1720 u32 delta_truesize; 1721 1722 if (skb_unclone_keeptruesize(skb, GFP_ATOMIC)) 1723 return -ENOMEM; 1724 1725 delta_truesize = __pskb_trim_head(skb, len); 1726 1727 TCP_SKB_CB(skb)->seq += len; 1728 1729 skb->truesize -= delta_truesize; 1730 sk_wmem_queued_add(sk, -delta_truesize); 1731 if (!skb_zcopy_pure(skb)) 1732 sk_mem_uncharge(sk, delta_truesize); 1733 1734 /* Any change of skb->len requires recalculation of tso factor. */ 1735 if (tcp_skb_pcount(skb) > 1) 1736 tcp_set_skb_tso_segs(skb, tcp_skb_mss(skb)); 1737 1738 return 0; 1739 } 1740 1741 /* Calculate MSS not accounting any TCP options. */ 1742 static inline int __tcp_mtu_to_mss(struct sock *sk, int pmtu) 1743 { 1744 const struct tcp_sock *tp = tcp_sk(sk); 1745 const struct inet_connection_sock *icsk = inet_csk(sk); 1746 int mss_now; 1747 1748 /* Calculate base mss without TCP options: 1749 It is MMS_S - sizeof(tcphdr) of rfc1122 1750 */ 1751 mss_now = pmtu - icsk->icsk_af_ops->net_header_len - sizeof(struct tcphdr); 1752 1753 /* Clamp it (mss_clamp does not include tcp options) */ 1754 if (mss_now > tp->rx_opt.mss_clamp) 1755 mss_now = tp->rx_opt.mss_clamp; 1756 1757 /* Now subtract optional transport overhead */ 1758 mss_now -= icsk->icsk_ext_hdr_len; 1759 1760 /* Then reserve room for full set of TCP options and 8 bytes of data */ 1761 mss_now = max(mss_now, 1762 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_snd_mss)); 1763 return mss_now; 1764 } 1765 1766 /* Calculate MSS. Not accounting for SACKs here. */ 1767 int tcp_mtu_to_mss(struct sock *sk, int pmtu) 1768 { 1769 /* Subtract TCP options size, not including SACKs */ 1770 return __tcp_mtu_to_mss(sk, pmtu) - 1771 (tcp_sk(sk)->tcp_header_len - sizeof(struct tcphdr)); 1772 } 1773 EXPORT_SYMBOL(tcp_mtu_to_mss); 1774 1775 /* Inverse of above */ 1776 int tcp_mss_to_mtu(struct sock *sk, int mss) 1777 { 1778 const struct tcp_sock *tp = tcp_sk(sk); 1779 const struct inet_connection_sock *icsk = inet_csk(sk); 1780 1781 return mss + 1782 tp->tcp_header_len + 1783 icsk->icsk_ext_hdr_len + 1784 icsk->icsk_af_ops->net_header_len; 1785 } 1786 EXPORT_SYMBOL(tcp_mss_to_mtu); 1787 1788 /* MTU probing init per socket */ 1789 void tcp_mtup_init(struct sock *sk) 1790 { 1791 struct tcp_sock *tp = tcp_sk(sk); 1792 struct inet_connection_sock *icsk = inet_csk(sk); 1793 struct net *net = sock_net(sk); 1794 1795 icsk->icsk_mtup.enabled = READ_ONCE(net->ipv4.sysctl_tcp_mtu_probing) > 1; 1796 icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp + sizeof(struct tcphdr) + 1797 icsk->icsk_af_ops->net_header_len; 1798 icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, READ_ONCE(net->ipv4.sysctl_tcp_base_mss)); 1799 icsk->icsk_mtup.probe_size = 0; 1800 if (icsk->icsk_mtup.enabled) 1801 icsk->icsk_mtup.probe_timestamp = tcp_jiffies32; 1802 } 1803 EXPORT_SYMBOL(tcp_mtup_init); 1804 1805 /* This function synchronize snd mss to current pmtu/exthdr set. 1806 1807 tp->rx_opt.user_mss is mss set by user by TCP_MAXSEG. It does NOT counts 1808 for TCP options, but includes only bare TCP header. 1809 1810 tp->rx_opt.mss_clamp is mss negotiated at connection setup. 1811 It is minimum of user_mss and mss received with SYN. 1812 It also does not include TCP options. 1813 1814 inet_csk(sk)->icsk_pmtu_cookie is last pmtu, seen by this function. 1815 1816 tp->mss_cache is current effective sending mss, including 1817 all tcp options except for SACKs. It is evaluated, 1818 taking into account current pmtu, but never exceeds 1819 tp->rx_opt.mss_clamp. 1820 1821 NOTE1. rfc1122 clearly states that advertised MSS 1822 DOES NOT include either tcp or ip options. 1823 1824 NOTE2. inet_csk(sk)->icsk_pmtu_cookie and tp->mss_cache 1825 are READ ONLY outside this function. --ANK (980731) 1826 */ 1827 unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu) 1828 { 1829 struct tcp_sock *tp = tcp_sk(sk); 1830 struct inet_connection_sock *icsk = inet_csk(sk); 1831 int mss_now; 1832 1833 if (icsk->icsk_mtup.search_high > pmtu) 1834 icsk->icsk_mtup.search_high = pmtu; 1835 1836 mss_now = tcp_mtu_to_mss(sk, pmtu); 1837 mss_now = tcp_bound_to_half_wnd(tp, mss_now); 1838 1839 /* And store cached results */ 1840 icsk->icsk_pmtu_cookie = pmtu; 1841 if (icsk->icsk_mtup.enabled) 1842 mss_now = min(mss_now, tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_low)); 1843 tp->mss_cache = mss_now; 1844 1845 return mss_now; 1846 } 1847 EXPORT_SYMBOL(tcp_sync_mss); 1848 1849 /* Compute the current effective MSS, taking SACKs and IP options, 1850 * and even PMTU discovery events into account. 1851 */ 1852 unsigned int tcp_current_mss(struct sock *sk) 1853 { 1854 const struct tcp_sock *tp = tcp_sk(sk); 1855 const struct dst_entry *dst = __sk_dst_get(sk); 1856 u32 mss_now; 1857 unsigned int header_len; 1858 struct tcp_out_options opts; 1859 struct tcp_key key; 1860 1861 mss_now = tp->mss_cache; 1862 1863 if (dst) { 1864 u32 mtu = dst_mtu(dst); 1865 if (mtu != inet_csk(sk)->icsk_pmtu_cookie) 1866 mss_now = tcp_sync_mss(sk, mtu); 1867 } 1868 tcp_get_current_key(sk, &key); 1869 header_len = tcp_established_options(sk, NULL, &opts, &key) + 1870 sizeof(struct tcphdr); 1871 /* The mss_cache is sized based on tp->tcp_header_len, which assumes 1872 * some common options. If this is an odd packet (because we have SACK 1873 * blocks etc) then our calculated header_len will be different, and 1874 * we have to adjust mss_now correspondingly */ 1875 if (header_len != tp->tcp_header_len) { 1876 int delta = (int) header_len - tp->tcp_header_len; 1877 mss_now -= delta; 1878 } 1879 1880 return mss_now; 1881 } 1882 1883 /* RFC2861, slow part. Adjust cwnd, after it was not full during one rto. 1884 * As additional protections, we do not touch cwnd in retransmission phases, 1885 * and if application hit its sndbuf limit recently. 1886 */ 1887 static void tcp_cwnd_application_limited(struct sock *sk) 1888 { 1889 struct tcp_sock *tp = tcp_sk(sk); 1890 1891 if (inet_csk(sk)->icsk_ca_state == TCP_CA_Open && 1892 sk->sk_socket && !test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) { 1893 /* Limited by application or receiver window. */ 1894 u32 init_win = tcp_init_cwnd(tp, __sk_dst_get(sk)); 1895 u32 win_used = max(tp->snd_cwnd_used, init_win); 1896 if (win_used < tcp_snd_cwnd(tp)) { 1897 tp->snd_ssthresh = tcp_current_ssthresh(sk); 1898 tcp_snd_cwnd_set(tp, (tcp_snd_cwnd(tp) + win_used) >> 1); 1899 } 1900 tp->snd_cwnd_used = 0; 1901 } 1902 tp->snd_cwnd_stamp = tcp_jiffies32; 1903 } 1904 1905 static void tcp_cwnd_validate(struct sock *sk, bool is_cwnd_limited) 1906 { 1907 const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; 1908 struct tcp_sock *tp = tcp_sk(sk); 1909 1910 /* Track the strongest available signal of the degree to which the cwnd 1911 * is fully utilized. If cwnd-limited then remember that fact for the 1912 * current window. If not cwnd-limited then track the maximum number of 1913 * outstanding packets in the current window. (If cwnd-limited then we 1914 * chose to not update tp->max_packets_out to avoid an extra else 1915 * clause with no functional impact.) 1916 */ 1917 if (!before(tp->snd_una, tp->cwnd_usage_seq) || 1918 is_cwnd_limited || 1919 (!tp->is_cwnd_limited && 1920 tp->packets_out > tp->max_packets_out)) { 1921 tp->is_cwnd_limited = is_cwnd_limited; 1922 tp->max_packets_out = tp->packets_out; 1923 tp->cwnd_usage_seq = tp->snd_nxt; 1924 } 1925 1926 if (tcp_is_cwnd_limited(sk)) { 1927 /* Network is feed fully. */ 1928 tp->snd_cwnd_used = 0; 1929 tp->snd_cwnd_stamp = tcp_jiffies32; 1930 } else { 1931 /* Network starves. */ 1932 if (tp->packets_out > tp->snd_cwnd_used) 1933 tp->snd_cwnd_used = tp->packets_out; 1934 1935 if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) && 1936 (s32)(tcp_jiffies32 - tp->snd_cwnd_stamp) >= inet_csk(sk)->icsk_rto && 1937 !ca_ops->cong_control) 1938 tcp_cwnd_application_limited(sk); 1939 1940 /* The following conditions together indicate the starvation 1941 * is caused by insufficient sender buffer: 1942 * 1) just sent some data (see tcp_write_xmit) 1943 * 2) not cwnd limited (this else condition) 1944 * 3) no more data to send (tcp_write_queue_empty()) 1945 * 4) application is hitting buffer limit (SOCK_NOSPACE) 1946 */ 1947 if (tcp_write_queue_empty(sk) && sk->sk_socket && 1948 test_bit(SOCK_NOSPACE, &sk->sk_socket->flags) && 1949 (1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) 1950 tcp_chrono_start(sk, TCP_CHRONO_SNDBUF_LIMITED); 1951 } 1952 } 1953 1954 /* Minshall's variant of the Nagle send check. */ 1955 static bool tcp_minshall_check(const struct tcp_sock *tp) 1956 { 1957 return after(tp->snd_sml, tp->snd_una) && 1958 !after(tp->snd_sml, tp->snd_nxt); 1959 } 1960 1961 /* Update snd_sml if this skb is under mss 1962 * Note that a TSO packet might end with a sub-mss segment 1963 * The test is really : 1964 * if ((skb->len % mss) != 0) 1965 * tp->snd_sml = TCP_SKB_CB(skb)->end_seq; 1966 * But we can avoid doing the divide again given we already have 1967 * skb_pcount = skb->len / mss_now 1968 */ 1969 static void tcp_minshall_update(struct tcp_sock *tp, unsigned int mss_now, 1970 const struct sk_buff *skb) 1971 { 1972 if (skb->len < tcp_skb_pcount(skb) * mss_now) 1973 tp->snd_sml = TCP_SKB_CB(skb)->end_seq; 1974 } 1975 1976 /* Return false, if packet can be sent now without violation Nagle's rules: 1977 * 1. It is full sized. (provided by caller in %partial bool) 1978 * 2. Or it contains FIN. (already checked by caller) 1979 * 3. Or TCP_CORK is not set, and TCP_NODELAY is set. 1980 * 4. Or TCP_CORK is not set, and all sent packets are ACKed. 1981 * With Minshall's modification: all sent small packets are ACKed. 1982 */ 1983 static bool tcp_nagle_check(bool partial, const struct tcp_sock *tp, 1984 int nonagle) 1985 { 1986 return partial && 1987 ((nonagle & TCP_NAGLE_CORK) || 1988 (!nonagle && tp->packets_out && tcp_minshall_check(tp))); 1989 } 1990 1991 /* Return how many segs we'd like on a TSO packet, 1992 * depending on current pacing rate, and how close the peer is. 1993 * 1994 * Rationale is: 1995 * - For close peers, we rather send bigger packets to reduce 1996 * cpu costs, because occasional losses will be repaired fast. 1997 * - For long distance/rtt flows, we would like to get ACK clocking 1998 * with 1 ACK per ms. 1999 * 2000 * Use min_rtt to help adapt TSO burst size, with smaller min_rtt resulting 2001 * in bigger TSO bursts. We we cut the RTT-based allowance in half 2002 * for every 2^9 usec (aka 512 us) of RTT, so that the RTT-based allowance 2003 * is below 1500 bytes after 6 * ~500 usec = 3ms. 2004 */ 2005 static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, 2006 int min_tso_segs) 2007 { 2008 unsigned long bytes; 2009 u32 r; 2010 2011 bytes = READ_ONCE(sk->sk_pacing_rate) >> READ_ONCE(sk->sk_pacing_shift); 2012 2013 r = tcp_min_rtt(tcp_sk(sk)) >> READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tso_rtt_log); 2014 if (r < BITS_PER_TYPE(sk->sk_gso_max_size)) 2015 bytes += sk->sk_gso_max_size >> r; 2016 2017 bytes = min_t(unsigned long, bytes, sk->sk_gso_max_size); 2018 2019 return max_t(u32, bytes / mss_now, min_tso_segs); 2020 } 2021 2022 /* Return the number of segments we want in the skb we are transmitting. 2023 * See if congestion control module wants to decide; otherwise, autosize. 2024 */ 2025 static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now) 2026 { 2027 const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; 2028 u32 min_tso, tso_segs; 2029 2030 min_tso = ca_ops->min_tso_segs ? 2031 ca_ops->min_tso_segs(sk) : 2032 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs); 2033 2034 tso_segs = tcp_tso_autosize(sk, mss_now, min_tso); 2035 return min_t(u32, tso_segs, sk->sk_gso_max_segs); 2036 } 2037 2038 /* Returns the portion of skb which can be sent right away */ 2039 static unsigned int tcp_mss_split_point(const struct sock *sk, 2040 const struct sk_buff *skb, 2041 unsigned int mss_now, 2042 unsigned int max_segs, 2043 int nonagle) 2044 { 2045 const struct tcp_sock *tp = tcp_sk(sk); 2046 u32 partial, needed, window, max_len; 2047 2048 window = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq; 2049 max_len = mss_now * max_segs; 2050 2051 if (likely(max_len <= window && skb != tcp_write_queue_tail(sk))) 2052 return max_len; 2053 2054 needed = min(skb->len, window); 2055 2056 if (max_len <= needed) 2057 return max_len; 2058 2059 partial = needed % mss_now; 2060 /* If last segment is not a full MSS, check if Nagle rules allow us 2061 * to include this last segment in this skb. 2062 * Otherwise, we'll split the skb at last MSS boundary 2063 */ 2064 if (tcp_nagle_check(partial != 0, tp, nonagle)) 2065 return needed - partial; 2066 2067 return needed; 2068 } 2069 2070 /* Can at least one segment of SKB be sent right now, according to the 2071 * congestion window rules? If so, return how many segments are allowed. 2072 */ 2073 static inline unsigned int tcp_cwnd_test(const struct tcp_sock *tp, 2074 const struct sk_buff *skb) 2075 { 2076 u32 in_flight, cwnd, halfcwnd; 2077 2078 /* Don't be strict about the congestion window for the final FIN. */ 2079 if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) && 2080 tcp_skb_pcount(skb) == 1) 2081 return 1; 2082 2083 in_flight = tcp_packets_in_flight(tp); 2084 cwnd = tcp_snd_cwnd(tp); 2085 if (in_flight >= cwnd) 2086 return 0; 2087 2088 /* For better scheduling, ensure we have at least 2089 * 2 GSO packets in flight. 2090 */ 2091 halfcwnd = max(cwnd >> 1, 1U); 2092 return min(halfcwnd, cwnd - in_flight); 2093 } 2094 2095 /* Initialize TSO state of a skb. 2096 * This must be invoked the first time we consider transmitting 2097 * SKB onto the wire. 2098 */ 2099 static int tcp_init_tso_segs(struct sk_buff *skb, unsigned int mss_now) 2100 { 2101 int tso_segs = tcp_skb_pcount(skb); 2102 2103 if (!tso_segs || (tso_segs > 1 && tcp_skb_mss(skb) != mss_now)) { 2104 tcp_set_skb_tso_segs(skb, mss_now); 2105 tso_segs = tcp_skb_pcount(skb); 2106 } 2107 return tso_segs; 2108 } 2109 2110 2111 /* Return true if the Nagle test allows this packet to be 2112 * sent now. 2113 */ 2114 static inline bool tcp_nagle_test(const struct tcp_sock *tp, const struct sk_buff *skb, 2115 unsigned int cur_mss, int nonagle) 2116 { 2117 /* Nagle rule does not apply to frames, which sit in the middle of the 2118 * write_queue (they have no chances to get new data). 2119 * 2120 * This is implemented in the callers, where they modify the 'nonagle' 2121 * argument based upon the location of SKB in the send queue. 2122 */ 2123 if (nonagle & TCP_NAGLE_PUSH) 2124 return true; 2125 2126 /* Don't use the nagle rule for urgent data (or for the final FIN). */ 2127 if (tcp_urg_mode(tp) || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) 2128 return true; 2129 2130 if (!tcp_nagle_check(skb->len < cur_mss, tp, nonagle)) 2131 return true; 2132 2133 return false; 2134 } 2135 2136 /* Does at least the first segment of SKB fit into the send window? */ 2137 static bool tcp_snd_wnd_test(const struct tcp_sock *tp, 2138 const struct sk_buff *skb, 2139 unsigned int cur_mss) 2140 { 2141 u32 end_seq = TCP_SKB_CB(skb)->end_seq; 2142 2143 if (skb->len > cur_mss) 2144 end_seq = TCP_SKB_CB(skb)->seq + cur_mss; 2145 2146 return !after(end_seq, tcp_wnd_end(tp)); 2147 } 2148 2149 /* Trim TSO SKB to LEN bytes, put the remaining data into a new packet 2150 * which is put after SKB on the list. It is very much like 2151 * tcp_fragment() except that it may make several kinds of assumptions 2152 * in order to speed up the splitting operation. In particular, we 2153 * know that all the data is in scatter-gather pages, and that the 2154 * packet has never been sent out before (and thus is not cloned). 2155 */ 2156 static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len, 2157 unsigned int mss_now, gfp_t gfp) 2158 { 2159 int nlen = skb->len - len; 2160 struct sk_buff *buff; 2161 u8 flags; 2162 2163 /* All of a TSO frame must be composed of paged data. */ 2164 DEBUG_NET_WARN_ON_ONCE(skb->len != skb->data_len); 2165 2166 buff = tcp_stream_alloc_skb(sk, gfp, true); 2167 if (unlikely(!buff)) 2168 return -ENOMEM; 2169 skb_copy_decrypted(buff, skb); 2170 mptcp_skb_ext_copy(buff, skb); 2171 2172 sk_wmem_queued_add(sk, buff->truesize); 2173 sk_mem_charge(sk, buff->truesize); 2174 buff->truesize += nlen; 2175 skb->truesize -= nlen; 2176 2177 /* Correct the sequence numbers. */ 2178 TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len; 2179 TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq; 2180 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq; 2181 2182 /* PSH and FIN should only be set in the second packet. */ 2183 flags = TCP_SKB_CB(skb)->tcp_flags; 2184 TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH); 2185 TCP_SKB_CB(buff)->tcp_flags = flags; 2186 2187 tcp_skb_fragment_eor(skb, buff); 2188 2189 skb_split(skb, buff, len); 2190 tcp_fragment_tstamp(skb, buff); 2191 2192 /* Fix up tso_factor for both original and new SKB. */ 2193 tcp_set_skb_tso_segs(skb, mss_now); 2194 tcp_set_skb_tso_segs(buff, mss_now); 2195 2196 /* Link BUFF into the send queue. */ 2197 __skb_header_release(buff); 2198 tcp_insert_write_queue_after(skb, buff, sk, TCP_FRAG_IN_WRITE_QUEUE); 2199 2200 return 0; 2201 } 2202 2203 /* Try to defer sending, if possible, in order to minimize the amount 2204 * of TSO splitting we do. View it as a kind of TSO Nagle test. 2205 * 2206 * This algorithm is from John Heffner. 2207 */ 2208 static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb, 2209 bool *is_cwnd_limited, 2210 bool *is_rwnd_limited, 2211 u32 max_segs) 2212 { 2213 const struct inet_connection_sock *icsk = inet_csk(sk); 2214 u32 send_win, cong_win, limit, in_flight; 2215 struct tcp_sock *tp = tcp_sk(sk); 2216 struct sk_buff *head; 2217 int win_divisor; 2218 s64 delta; 2219 2220 if (icsk->icsk_ca_state >= TCP_CA_Recovery) 2221 goto send_now; 2222 2223 /* Avoid bursty behavior by allowing defer 2224 * only if the last write was recent (1 ms). 2225 * Note that tp->tcp_wstamp_ns can be in the future if we have 2226 * packets waiting in a qdisc or device for EDT delivery. 2227 */ 2228 delta = tp->tcp_clock_cache - tp->tcp_wstamp_ns - NSEC_PER_MSEC; 2229 if (delta > 0) 2230 goto send_now; 2231 2232 in_flight = tcp_packets_in_flight(tp); 2233 2234 BUG_ON(tcp_skb_pcount(skb) <= 1); 2235 BUG_ON(tcp_snd_cwnd(tp) <= in_flight); 2236 2237 send_win = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq; 2238 2239 /* From in_flight test above, we know that cwnd > in_flight. */ 2240 cong_win = (tcp_snd_cwnd(tp) - in_flight) * tp->mss_cache; 2241 2242 limit = min(send_win, cong_win); 2243 2244 /* If a full-sized TSO skb can be sent, do it. */ 2245 if (limit >= max_segs * tp->mss_cache) 2246 goto send_now; 2247 2248 /* Middle in queue won't get any more data, full sendable already? */ 2249 if ((skb != tcp_write_queue_tail(sk)) && (limit >= skb->len)) 2250 goto send_now; 2251 2252 win_divisor = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tso_win_divisor); 2253 if (win_divisor) { 2254 u32 chunk = min(tp->snd_wnd, tcp_snd_cwnd(tp) * tp->mss_cache); 2255 2256 /* If at least some fraction of a window is available, 2257 * just use it. 2258 */ 2259 chunk /= win_divisor; 2260 if (limit >= chunk) 2261 goto send_now; 2262 } else { 2263 /* Different approach, try not to defer past a single 2264 * ACK. Receiver should ACK every other full sized 2265 * frame, so if we have space for more than 3 frames 2266 * then send now. 2267 */ 2268 if (limit > tcp_max_tso_deferred_mss(tp) * tp->mss_cache) 2269 goto send_now; 2270 } 2271 2272 /* TODO : use tsorted_sent_queue ? */ 2273 head = tcp_rtx_queue_head(sk); 2274 if (!head) 2275 goto send_now; 2276 delta = tp->tcp_clock_cache - head->tstamp; 2277 /* If next ACK is likely to come too late (half srtt), do not defer */ 2278 if ((s64)(delta - (u64)NSEC_PER_USEC * (tp->srtt_us >> 4)) < 0) 2279 goto send_now; 2280 2281 /* Ok, it looks like it is advisable to defer. 2282 * Three cases are tracked : 2283 * 1) We are cwnd-limited 2284 * 2) We are rwnd-limited 2285 * 3) We are application limited. 2286 */ 2287 if (cong_win < send_win) { 2288 if (cong_win <= skb->len) { 2289 *is_cwnd_limited = true; 2290 return true; 2291 } 2292 } else { 2293 if (send_win <= skb->len) { 2294 *is_rwnd_limited = true; 2295 return true; 2296 } 2297 } 2298 2299 /* If this packet won't get more data, do not wait. */ 2300 if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) || 2301 TCP_SKB_CB(skb)->eor) 2302 goto send_now; 2303 2304 return true; 2305 2306 send_now: 2307 return false; 2308 } 2309 2310 static inline void tcp_mtu_check_reprobe(struct sock *sk) 2311 { 2312 struct inet_connection_sock *icsk = inet_csk(sk); 2313 struct tcp_sock *tp = tcp_sk(sk); 2314 struct net *net = sock_net(sk); 2315 u32 interval; 2316 s32 delta; 2317 2318 interval = READ_ONCE(net->ipv4.sysctl_tcp_probe_interval); 2319 delta = tcp_jiffies32 - icsk->icsk_mtup.probe_timestamp; 2320 if (unlikely(delta >= interval * HZ)) { 2321 int mss = tcp_current_mss(sk); 2322 2323 /* Update current search range */ 2324 icsk->icsk_mtup.probe_size = 0; 2325 icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp + 2326 sizeof(struct tcphdr) + 2327 icsk->icsk_af_ops->net_header_len; 2328 icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, mss); 2329 2330 /* Update probe time stamp */ 2331 icsk->icsk_mtup.probe_timestamp = tcp_jiffies32; 2332 } 2333 } 2334 2335 static bool tcp_can_coalesce_send_queue_head(struct sock *sk, int len) 2336 { 2337 struct sk_buff *skb, *next; 2338 2339 skb = tcp_send_head(sk); 2340 tcp_for_write_queue_from_safe(skb, next, sk) { 2341 if (len <= skb->len) 2342 break; 2343 2344 if (unlikely(TCP_SKB_CB(skb)->eor) || 2345 tcp_has_tx_tstamp(skb) || 2346 !skb_pure_zcopy_same(skb, next)) 2347 return false; 2348 2349 len -= skb->len; 2350 } 2351 2352 return true; 2353 } 2354 2355 static int tcp_clone_payload(struct sock *sk, struct sk_buff *to, 2356 int probe_size) 2357 { 2358 skb_frag_t *lastfrag = NULL, *fragto = skb_shinfo(to)->frags; 2359 int i, todo, len = 0, nr_frags = 0; 2360 const struct sk_buff *skb; 2361 2362 if (!sk_wmem_schedule(sk, to->truesize + probe_size)) 2363 return -ENOMEM; 2364 2365 skb_queue_walk(&sk->sk_write_queue, skb) { 2366 const skb_frag_t *fragfrom = skb_shinfo(skb)->frags; 2367 2368 if (skb_headlen(skb)) 2369 return -EINVAL; 2370 2371 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, fragfrom++) { 2372 if (len >= probe_size) 2373 goto commit; 2374 todo = min_t(int, skb_frag_size(fragfrom), 2375 probe_size - len); 2376 len += todo; 2377 if (lastfrag && 2378 skb_frag_page(fragfrom) == skb_frag_page(lastfrag) && 2379 skb_frag_off(fragfrom) == skb_frag_off(lastfrag) + 2380 skb_frag_size(lastfrag)) { 2381 skb_frag_size_add(lastfrag, todo); 2382 continue; 2383 } 2384 if (unlikely(nr_frags == MAX_SKB_FRAGS)) 2385 return -E2BIG; 2386 skb_frag_page_copy(fragto, fragfrom); 2387 skb_frag_off_copy(fragto, fragfrom); 2388 skb_frag_size_set(fragto, todo); 2389 nr_frags++; 2390 lastfrag = fragto++; 2391 } 2392 } 2393 commit: 2394 WARN_ON_ONCE(len != probe_size); 2395 for (i = 0; i < nr_frags; i++) 2396 skb_frag_ref(to, i); 2397 2398 skb_shinfo(to)->nr_frags = nr_frags; 2399 to->truesize += probe_size; 2400 to->len += probe_size; 2401 to->data_len += probe_size; 2402 __skb_header_release(to); 2403 return 0; 2404 } 2405 2406 /* Create a new MTU probe if we are ready. 2407 * MTU probe is regularly attempting to increase the path MTU by 2408 * deliberately sending larger packets. This discovers routing 2409 * changes resulting in larger path MTUs. 2410 * 2411 * Returns 0 if we should wait to probe (no cwnd available), 2412 * 1 if a probe was sent, 2413 * -1 otherwise 2414 */ 2415 static int tcp_mtu_probe(struct sock *sk) 2416 { 2417 struct inet_connection_sock *icsk = inet_csk(sk); 2418 struct tcp_sock *tp = tcp_sk(sk); 2419 struct sk_buff *skb, *nskb, *next; 2420 struct net *net = sock_net(sk); 2421 int probe_size; 2422 int size_needed; 2423 int copy, len; 2424 int mss_now; 2425 int interval; 2426 2427 /* Not currently probing/verifying, 2428 * not in recovery, 2429 * have enough cwnd, and 2430 * not SACKing (the variable headers throw things off) 2431 */ 2432 if (likely(!icsk->icsk_mtup.enabled || 2433 icsk->icsk_mtup.probe_size || 2434 inet_csk(sk)->icsk_ca_state != TCP_CA_Open || 2435 tcp_snd_cwnd(tp) < 11 || 2436 tp->rx_opt.num_sacks || tp->rx_opt.dsack)) 2437 return -1; 2438 2439 /* Use binary search for probe_size between tcp_mss_base, 2440 * and current mss_clamp. if (search_high - search_low) 2441 * smaller than a threshold, backoff from probing. 2442 */ 2443 mss_now = tcp_current_mss(sk); 2444 probe_size = tcp_mtu_to_mss(sk, (icsk->icsk_mtup.search_high + 2445 icsk->icsk_mtup.search_low) >> 1); 2446 size_needed = probe_size + (tp->reordering + 1) * tp->mss_cache; 2447 interval = icsk->icsk_mtup.search_high - icsk->icsk_mtup.search_low; 2448 /* When misfortune happens, we are reprobing actively, 2449 * and then reprobe timer has expired. We stick with current 2450 * probing process by not resetting search range to its orignal. 2451 */ 2452 if (probe_size > tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_high) || 2453 interval < READ_ONCE(net->ipv4.sysctl_tcp_probe_threshold)) { 2454 /* Check whether enough time has elaplased for 2455 * another round of probing. 2456 */ 2457 tcp_mtu_check_reprobe(sk); 2458 return -1; 2459 } 2460 2461 /* Have enough data in the send queue to probe? */ 2462 if (tp->write_seq - tp->snd_nxt < size_needed) 2463 return -1; 2464 2465 if (tp->snd_wnd < size_needed) 2466 return -1; 2467 if (after(tp->snd_nxt + size_needed, tcp_wnd_end(tp))) 2468 return 0; 2469 2470 /* Do we need to wait to drain cwnd? With none in flight, don't stall */ 2471 if (tcp_packets_in_flight(tp) + 2 > tcp_snd_cwnd(tp)) { 2472 if (!tcp_packets_in_flight(tp)) 2473 return -1; 2474 else 2475 return 0; 2476 } 2477 2478 if (!tcp_can_coalesce_send_queue_head(sk, probe_size)) 2479 return -1; 2480 2481 /* We're allowed to probe. Build it now. */ 2482 nskb = tcp_stream_alloc_skb(sk, GFP_ATOMIC, false); 2483 if (!nskb) 2484 return -1; 2485 2486 /* build the payload, and be prepared to abort if this fails. */ 2487 if (tcp_clone_payload(sk, nskb, probe_size)) { 2488 tcp_skb_tsorted_anchor_cleanup(nskb); 2489 consume_skb(nskb); 2490 return -1; 2491 } 2492 sk_wmem_queued_add(sk, nskb->truesize); 2493 sk_mem_charge(sk, nskb->truesize); 2494 2495 skb = tcp_send_head(sk); 2496 skb_copy_decrypted(nskb, skb); 2497 mptcp_skb_ext_copy(nskb, skb); 2498 2499 TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq; 2500 TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size; 2501 TCP_SKB_CB(nskb)->tcp_flags = TCPHDR_ACK; 2502 2503 tcp_insert_write_queue_before(nskb, skb, sk); 2504 tcp_highest_sack_replace(sk, skb, nskb); 2505 2506 len = 0; 2507 tcp_for_write_queue_from_safe(skb, next, sk) { 2508 copy = min_t(int, skb->len, probe_size - len); 2509 2510 if (skb->len <= copy) { 2511 /* We've eaten all the data from this skb. 2512 * Throw it away. */ 2513 TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags; 2514 /* If this is the last SKB we copy and eor is set 2515 * we need to propagate it to the new skb. 2516 */ 2517 TCP_SKB_CB(nskb)->eor = TCP_SKB_CB(skb)->eor; 2518 tcp_skb_collapse_tstamp(nskb, skb); 2519 tcp_unlink_write_queue(skb, sk); 2520 tcp_wmem_free_skb(sk, skb); 2521 } else { 2522 TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags & 2523 ~(TCPHDR_FIN|TCPHDR_PSH); 2524 __pskb_trim_head(skb, copy); 2525 tcp_set_skb_tso_segs(skb, mss_now); 2526 TCP_SKB_CB(skb)->seq += copy; 2527 } 2528 2529 len += copy; 2530 2531 if (len >= probe_size) 2532 break; 2533 } 2534 tcp_init_tso_segs(nskb, nskb->len); 2535 2536 /* We're ready to send. If this fails, the probe will 2537 * be resegmented into mss-sized pieces by tcp_write_xmit(). 2538 */ 2539 if (!tcp_transmit_skb(sk, nskb, 1, GFP_ATOMIC)) { 2540 /* Decrement cwnd here because we are sending 2541 * effectively two packets. */ 2542 tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) - 1); 2543 tcp_event_new_data_sent(sk, nskb); 2544 2545 icsk->icsk_mtup.probe_size = tcp_mss_to_mtu(sk, nskb->len); 2546 tp->mtu_probe.probe_seq_start = TCP_SKB_CB(nskb)->seq; 2547 tp->mtu_probe.probe_seq_end = TCP_SKB_CB(nskb)->end_seq; 2548 2549 return 1; 2550 } 2551 2552 return -1; 2553 } 2554 2555 static bool tcp_pacing_check(struct sock *sk) 2556 { 2557 struct tcp_sock *tp = tcp_sk(sk); 2558 2559 if (!tcp_needs_internal_pacing(sk)) 2560 return false; 2561 2562 if (tp->tcp_wstamp_ns <= tp->tcp_clock_cache) 2563 return false; 2564 2565 if (!hrtimer_is_queued(&tp->pacing_timer)) { 2566 hrtimer_start(&tp->pacing_timer, 2567 ns_to_ktime(tp->tcp_wstamp_ns), 2568 HRTIMER_MODE_ABS_PINNED_SOFT); 2569 sock_hold(sk); 2570 } 2571 return true; 2572 } 2573 2574 static bool tcp_rtx_queue_empty_or_single_skb(const struct sock *sk) 2575 { 2576 const struct rb_node *node = sk->tcp_rtx_queue.rb_node; 2577 2578 /* No skb in the rtx queue. */ 2579 if (!node) 2580 return true; 2581 2582 /* Only one skb in rtx queue. */ 2583 return !node->rb_left && !node->rb_right; 2584 } 2585 2586 /* TCP Small Queues : 2587 * Control number of packets in qdisc/devices to two packets / or ~1 ms. 2588 * (These limits are doubled for retransmits) 2589 * This allows for : 2590 * - better RTT estimation and ACK scheduling 2591 * - faster recovery 2592 * - high rates 2593 * Alas, some drivers / subsystems require a fair amount 2594 * of queued bytes to ensure line rate. 2595 * One example is wifi aggregation (802.11 AMPDU) 2596 */ 2597 static bool tcp_small_queue_check(struct sock *sk, const struct sk_buff *skb, 2598 unsigned int factor) 2599 { 2600 unsigned long limit; 2601 2602 limit = max_t(unsigned long, 2603 2 * skb->truesize, 2604 READ_ONCE(sk->sk_pacing_rate) >> READ_ONCE(sk->sk_pacing_shift)); 2605 if (sk->sk_pacing_status == SK_PACING_NONE) 2606 limit = min_t(unsigned long, limit, 2607 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_limit_output_bytes)); 2608 limit <<= factor; 2609 2610 if (static_branch_unlikely(&tcp_tx_delay_enabled) && 2611 tcp_sk(sk)->tcp_tx_delay) { 2612 u64 extra_bytes = (u64)READ_ONCE(sk->sk_pacing_rate) * 2613 tcp_sk(sk)->tcp_tx_delay; 2614 2615 /* TSQ is based on skb truesize sum (sk_wmem_alloc), so we 2616 * approximate our needs assuming an ~100% skb->truesize overhead. 2617 * USEC_PER_SEC is approximated by 2^20. 2618 * do_div(extra_bytes, USEC_PER_SEC/2) is replaced by a right shift. 2619 */ 2620 extra_bytes >>= (20 - 1); 2621 limit += extra_bytes; 2622 } 2623 if (refcount_read(&sk->sk_wmem_alloc) > limit) { 2624 /* Always send skb if rtx queue is empty or has one skb. 2625 * No need to wait for TX completion to call us back, 2626 * after softirq/tasklet schedule. 2627 * This helps when TX completions are delayed too much. 2628 */ 2629 if (tcp_rtx_queue_empty_or_single_skb(sk)) 2630 return false; 2631 2632 set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags); 2633 /* It is possible TX completion already happened 2634 * before we set TSQ_THROTTLED, so we must 2635 * test again the condition. 2636 */ 2637 smp_mb__after_atomic(); 2638 if (refcount_read(&sk->sk_wmem_alloc) > limit) 2639 return true; 2640 } 2641 return false; 2642 } 2643 2644 static void tcp_chrono_set(struct tcp_sock *tp, const enum tcp_chrono new) 2645 { 2646 const u32 now = tcp_jiffies32; 2647 enum tcp_chrono old = tp->chrono_type; 2648 2649 if (old > TCP_CHRONO_UNSPEC) 2650 tp->chrono_stat[old - 1] += now - tp->chrono_start; 2651 tp->chrono_start = now; 2652 tp->chrono_type = new; 2653 } 2654 2655 void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type) 2656 { 2657 struct tcp_sock *tp = tcp_sk(sk); 2658 2659 /* If there are multiple conditions worthy of tracking in a 2660 * chronograph then the highest priority enum takes precedence 2661 * over the other conditions. So that if something "more interesting" 2662 * starts happening, stop the previous chrono and start a new one. 2663 */ 2664 if (type > tp->chrono_type) 2665 tcp_chrono_set(tp, type); 2666 } 2667 2668 void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type) 2669 { 2670 struct tcp_sock *tp = tcp_sk(sk); 2671 2672 2673 /* There are multiple conditions worthy of tracking in a 2674 * chronograph, so that the highest priority enum takes 2675 * precedence over the other conditions (see tcp_chrono_start). 2676 * If a condition stops, we only stop chrono tracking if 2677 * it's the "most interesting" or current chrono we are 2678 * tracking and starts busy chrono if we have pending data. 2679 */ 2680 if (tcp_rtx_and_write_queues_empty(sk)) 2681 tcp_chrono_set(tp, TCP_CHRONO_UNSPEC); 2682 else if (type == tp->chrono_type) 2683 tcp_chrono_set(tp, TCP_CHRONO_BUSY); 2684 } 2685 2686 /* This routine writes packets to the network. It advances the 2687 * send_head. This happens as incoming acks open up the remote 2688 * window for us. 2689 * 2690 * LARGESEND note: !tcp_urg_mode is overkill, only frames between 2691 * snd_up-64k-mss .. snd_up cannot be large. However, taking into 2692 * account rare use of URG, this is not a big flaw. 2693 * 2694 * Send at most one packet when push_one > 0. Temporarily ignore 2695 * cwnd limit to force at most one packet out when push_one == 2. 2696 2697 * Returns true, if no segments are in flight and we have queued segments, 2698 * but cannot send anything now because of SWS or another problem. 2699 */ 2700 static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, 2701 int push_one, gfp_t gfp) 2702 { 2703 struct tcp_sock *tp = tcp_sk(sk); 2704 struct sk_buff *skb; 2705 unsigned int tso_segs, sent_pkts; 2706 int cwnd_quota; 2707 int result; 2708 bool is_cwnd_limited = false, is_rwnd_limited = false; 2709 u32 max_segs; 2710 2711 sent_pkts = 0; 2712 2713 tcp_mstamp_refresh(tp); 2714 if (!push_one) { 2715 /* Do MTU probing. */ 2716 result = tcp_mtu_probe(sk); 2717 if (!result) { 2718 return false; 2719 } else if (result > 0) { 2720 sent_pkts = 1; 2721 } 2722 } 2723 2724 max_segs = tcp_tso_segs(sk, mss_now); 2725 while ((skb = tcp_send_head(sk))) { 2726 unsigned int limit; 2727 2728 if (unlikely(tp->repair) && tp->repair_queue == TCP_SEND_QUEUE) { 2729 /* "skb_mstamp_ns" is used as a start point for the retransmit timer */ 2730 tp->tcp_wstamp_ns = tp->tcp_clock_cache; 2731 skb_set_delivery_time(skb, tp->tcp_wstamp_ns, true); 2732 list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue); 2733 tcp_init_tso_segs(skb, mss_now); 2734 goto repair; /* Skip network transmission */ 2735 } 2736 2737 if (tcp_pacing_check(sk)) 2738 break; 2739 2740 tso_segs = tcp_init_tso_segs(skb, mss_now); 2741 BUG_ON(!tso_segs); 2742 2743 cwnd_quota = tcp_cwnd_test(tp, skb); 2744 if (!cwnd_quota) { 2745 if (push_one == 2) 2746 /* Force out a loss probe pkt. */ 2747 cwnd_quota = 1; 2748 else 2749 break; 2750 } 2751 2752 if (unlikely(!tcp_snd_wnd_test(tp, skb, mss_now))) { 2753 is_rwnd_limited = true; 2754 break; 2755 } 2756 2757 if (tso_segs == 1) { 2758 if (unlikely(!tcp_nagle_test(tp, skb, mss_now, 2759 (tcp_skb_is_last(sk, skb) ? 2760 nonagle : TCP_NAGLE_PUSH)))) 2761 break; 2762 } else { 2763 if (!push_one && 2764 tcp_tso_should_defer(sk, skb, &is_cwnd_limited, 2765 &is_rwnd_limited, max_segs)) 2766 break; 2767 } 2768 2769 limit = mss_now; 2770 if (tso_segs > 1 && !tcp_urg_mode(tp)) 2771 limit = tcp_mss_split_point(sk, skb, mss_now, 2772 min_t(unsigned int, 2773 cwnd_quota, 2774 max_segs), 2775 nonagle); 2776 2777 if (skb->len > limit && 2778 unlikely(tso_fragment(sk, skb, limit, mss_now, gfp))) 2779 break; 2780 2781 if (tcp_small_queue_check(sk, skb, 0)) 2782 break; 2783 2784 /* Argh, we hit an empty skb(), presumably a thread 2785 * is sleeping in sendmsg()/sk_stream_wait_memory(). 2786 * We do not want to send a pure-ack packet and have 2787 * a strange looking rtx queue with empty packet(s). 2788 */ 2789 if (TCP_SKB_CB(skb)->end_seq == TCP_SKB_CB(skb)->seq) 2790 break; 2791 2792 if (unlikely(tcp_transmit_skb(sk, skb, 1, gfp))) 2793 break; 2794 2795 repair: 2796 /* Advance the send_head. This one is sent out. 2797 * This call will increment packets_out. 2798 */ 2799 tcp_event_new_data_sent(sk, skb); 2800 2801 tcp_minshall_update(tp, mss_now, skb); 2802 sent_pkts += tcp_skb_pcount(skb); 2803 2804 if (push_one) 2805 break; 2806 } 2807 2808 if (is_rwnd_limited) 2809 tcp_chrono_start(sk, TCP_CHRONO_RWND_LIMITED); 2810 else 2811 tcp_chrono_stop(sk, TCP_CHRONO_RWND_LIMITED); 2812 2813 is_cwnd_limited |= (tcp_packets_in_flight(tp) >= tcp_snd_cwnd(tp)); 2814 if (likely(sent_pkts || is_cwnd_limited)) 2815 tcp_cwnd_validate(sk, is_cwnd_limited); 2816 2817 if (likely(sent_pkts)) { 2818 if (tcp_in_cwnd_reduction(sk)) 2819 tp->prr_out += sent_pkts; 2820 2821 /* Send one loss probe per tail loss episode. */ 2822 if (push_one != 2) 2823 tcp_schedule_loss_probe(sk, false); 2824 return false; 2825 } 2826 return !tp->packets_out && !tcp_write_queue_empty(sk); 2827 } 2828 2829 bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto) 2830 { 2831 struct inet_connection_sock *icsk = inet_csk(sk); 2832 struct tcp_sock *tp = tcp_sk(sk); 2833 u32 timeout, timeout_us, rto_delta_us; 2834 int early_retrans; 2835 2836 /* Don't do any loss probe on a Fast Open connection before 3WHS 2837 * finishes. 2838 */ 2839 if (rcu_access_pointer(tp->fastopen_rsk)) 2840 return false; 2841 2842 early_retrans = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_early_retrans); 2843 /* Schedule a loss probe in 2*RTT for SACK capable connections 2844 * not in loss recovery, that are either limited by cwnd or application. 2845 */ 2846 if ((early_retrans != 3 && early_retrans != 4) || 2847 !tp->packets_out || !tcp_is_sack(tp) || 2848 (icsk->icsk_ca_state != TCP_CA_Open && 2849 icsk->icsk_ca_state != TCP_CA_CWR)) 2850 return false; 2851 2852 /* Probe timeout is 2*rtt. Add minimum RTO to account 2853 * for delayed ack when there's one outstanding packet. If no RTT 2854 * sample is available then probe after TCP_TIMEOUT_INIT. 2855 */ 2856 if (tp->srtt_us) { 2857 timeout_us = tp->srtt_us >> 2; 2858 if (tp->packets_out == 1) 2859 timeout_us += tcp_rto_min_us(sk); 2860 else 2861 timeout_us += TCP_TIMEOUT_MIN_US; 2862 timeout = usecs_to_jiffies(timeout_us); 2863 } else { 2864 timeout = TCP_TIMEOUT_INIT; 2865 } 2866 2867 /* If the RTO formula yields an earlier time, then use that time. */ 2868 rto_delta_us = advancing_rto ? 2869 jiffies_to_usecs(inet_csk(sk)->icsk_rto) : 2870 tcp_rto_delta_us(sk); /* How far in future is RTO? */ 2871 if (rto_delta_us > 0) 2872 timeout = min_t(u32, timeout, usecs_to_jiffies(rto_delta_us)); 2873 2874 tcp_reset_xmit_timer(sk, ICSK_TIME_LOSS_PROBE, timeout, TCP_RTO_MAX); 2875 return true; 2876 } 2877 2878 /* Thanks to skb fast clones, we can detect if a prior transmit of 2879 * a packet is still in a qdisc or driver queue. 2880 * In this case, there is very little point doing a retransmit ! 2881 */ 2882 static bool skb_still_in_host_queue(struct sock *sk, 2883 const struct sk_buff *skb) 2884 { 2885 if (unlikely(skb_fclone_busy(sk, skb))) { 2886 set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags); 2887 smp_mb__after_atomic(); 2888 if (skb_fclone_busy(sk, skb)) { 2889 NET_INC_STATS(sock_net(sk), 2890 LINUX_MIB_TCPSPURIOUS_RTX_HOSTQUEUES); 2891 return true; 2892 } 2893 } 2894 return false; 2895 } 2896 2897 /* When probe timeout (PTO) fires, try send a new segment if possible, else 2898 * retransmit the last segment. 2899 */ 2900 void tcp_send_loss_probe(struct sock *sk) 2901 { 2902 struct tcp_sock *tp = tcp_sk(sk); 2903 struct sk_buff *skb; 2904 int pcount; 2905 int mss = tcp_current_mss(sk); 2906 2907 /* At most one outstanding TLP */ 2908 if (tp->tlp_high_seq) 2909 goto rearm_timer; 2910 2911 tp->tlp_retrans = 0; 2912 skb = tcp_send_head(sk); 2913 if (skb && tcp_snd_wnd_test(tp, skb, mss)) { 2914 pcount = tp->packets_out; 2915 tcp_write_xmit(sk, mss, TCP_NAGLE_OFF, 2, GFP_ATOMIC); 2916 if (tp->packets_out > pcount) 2917 goto probe_sent; 2918 goto rearm_timer; 2919 } 2920 skb = skb_rb_last(&sk->tcp_rtx_queue); 2921 if (unlikely(!skb)) { 2922 WARN_ONCE(tp->packets_out, 2923 "invalid inflight: %u state %u cwnd %u mss %d\n", 2924 tp->packets_out, sk->sk_state, tcp_snd_cwnd(tp), mss); 2925 inet_csk(sk)->icsk_pending = 0; 2926 return; 2927 } 2928 2929 if (skb_still_in_host_queue(sk, skb)) 2930 goto rearm_timer; 2931 2932 pcount = tcp_skb_pcount(skb); 2933 if (WARN_ON(!pcount)) 2934 goto rearm_timer; 2935 2936 if ((pcount > 1) && (skb->len > (pcount - 1) * mss)) { 2937 if (unlikely(tcp_fragment(sk, TCP_FRAG_IN_RTX_QUEUE, skb, 2938 (pcount - 1) * mss, mss, 2939 GFP_ATOMIC))) 2940 goto rearm_timer; 2941 skb = skb_rb_next(skb); 2942 } 2943 2944 if (WARN_ON(!skb || !tcp_skb_pcount(skb))) 2945 goto rearm_timer; 2946 2947 if (__tcp_retransmit_skb(sk, skb, 1)) 2948 goto rearm_timer; 2949 2950 tp->tlp_retrans = 1; 2951 2952 probe_sent: 2953 /* Record snd_nxt for loss detection. */ 2954 tp->tlp_high_seq = tp->snd_nxt; 2955 2956 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPLOSSPROBES); 2957 /* Reset s.t. tcp_rearm_rto will restart timer from now */ 2958 inet_csk(sk)->icsk_pending = 0; 2959 rearm_timer: 2960 tcp_rearm_rto(sk); 2961 } 2962 2963 /* Push out any pending frames which were held back due to 2964 * TCP_CORK or attempt at coalescing tiny packets. 2965 * The socket must be locked by the caller. 2966 */ 2967 void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss, 2968 int nonagle) 2969 { 2970 /* If we are closed, the bytes will have to remain here. 2971 * In time closedown will finish, we empty the write queue and 2972 * all will be happy. 2973 */ 2974 if (unlikely(sk->sk_state == TCP_CLOSE)) 2975 return; 2976 2977 if (tcp_write_xmit(sk, cur_mss, nonagle, 0, 2978 sk_gfp_mask(sk, GFP_ATOMIC))) 2979 tcp_check_probe_timer(sk); 2980 } 2981 2982 /* Send _single_ skb sitting at the send head. This function requires 2983 * true push pending frames to setup probe timer etc. 2984 */ 2985 void tcp_push_one(struct sock *sk, unsigned int mss_now) 2986 { 2987 struct sk_buff *skb = tcp_send_head(sk); 2988 2989 BUG_ON(!skb || skb->len < mss_now); 2990 2991 tcp_write_xmit(sk, mss_now, TCP_NAGLE_PUSH, 1, sk->sk_allocation); 2992 } 2993 2994 /* This function returns the amount that we can raise the 2995 * usable window based on the following constraints 2996 * 2997 * 1. The window can never be shrunk once it is offered (RFC 793) 2998 * 2. We limit memory per socket 2999 * 3000 * RFC 1122: 3001 * "the suggested [SWS] avoidance algorithm for the receiver is to keep 3002 * RECV.NEXT + RCV.WIN fixed until: 3003 * RCV.BUFF - RCV.USER - RCV.WINDOW >= min(1/2 RCV.BUFF, MSS)" 3004 * 3005 * i.e. don't raise the right edge of the window until you can raise 3006 * it at least MSS bytes. 3007 * 3008 * Unfortunately, the recommended algorithm breaks header prediction, 3009 * since header prediction assumes th->window stays fixed. 3010 * 3011 * Strictly speaking, keeping th->window fixed violates the receiver 3012 * side SWS prevention criteria. The problem is that under this rule 3013 * a stream of single byte packets will cause the right side of the 3014 * window to always advance by a single byte. 3015 * 3016 * Of course, if the sender implements sender side SWS prevention 3017 * then this will not be a problem. 3018 * 3019 * BSD seems to make the following compromise: 3020 * 3021 * If the free space is less than the 1/4 of the maximum 3022 * space available and the free space is less than 1/2 mss, 3023 * then set the window to 0. 3024 * [ Actually, bsd uses MSS and 1/4 of maximal _window_ ] 3025 * Otherwise, just prevent the window from shrinking 3026 * and from being larger than the largest representable value. 3027 * 3028 * This prevents incremental opening of the window in the regime 3029 * where TCP is limited by the speed of the reader side taking 3030 * data out of the TCP receive queue. It does nothing about 3031 * those cases where the window is constrained on the sender side 3032 * because the pipeline is full. 3033 * 3034 * BSD also seems to "accidentally" limit itself to windows that are a 3035 * multiple of MSS, at least until the free space gets quite small. 3036 * This would appear to be a side effect of the mbuf implementation. 3037 * Combining these two algorithms results in the observed behavior 3038 * of having a fixed window size at almost all times. 3039 * 3040 * Below we obtain similar behavior by forcing the offered window to 3041 * a multiple of the mss when it is feasible to do so. 3042 * 3043 * Note, we don't "adjust" for TIMESTAMP or SACK option bytes. 3044 * Regular options like TIMESTAMP are taken into account. 3045 */ 3046 u32 __tcp_select_window(struct sock *sk) 3047 { 3048 struct inet_connection_sock *icsk = inet_csk(sk); 3049 struct tcp_sock *tp = tcp_sk(sk); 3050 struct net *net = sock_net(sk); 3051 /* MSS for the peer's data. Previous versions used mss_clamp 3052 * here. I don't know if the value based on our guesses 3053 * of peer's MSS is better for the performance. It's more correct 3054 * but may be worse for the performance because of rcv_mss 3055 * fluctuations. --SAW 1998/11/1 3056 */ 3057 int mss = icsk->icsk_ack.rcv_mss; 3058 int free_space = tcp_space(sk); 3059 int allowed_space = tcp_full_space(sk); 3060 int full_space, window; 3061 3062 if (sk_is_mptcp(sk)) 3063 mptcp_space(sk, &free_space, &allowed_space); 3064 3065 full_space = min_t(int, tp->window_clamp, allowed_space); 3066 3067 if (unlikely(mss > full_space)) { 3068 mss = full_space; 3069 if (mss <= 0) 3070 return 0; 3071 } 3072 3073 /* Only allow window shrink if the sysctl is enabled and we have 3074 * a non-zero scaling factor in effect. 3075 */ 3076 if (READ_ONCE(net->ipv4.sysctl_tcp_shrink_window) && tp->rx_opt.rcv_wscale) 3077 goto shrink_window_allowed; 3078 3079 /* do not allow window to shrink */ 3080 3081 if (free_space < (full_space >> 1)) { 3082 icsk->icsk_ack.quick = 0; 3083 3084 if (tcp_under_memory_pressure(sk)) 3085 tcp_adjust_rcv_ssthresh(sk); 3086 3087 /* free_space might become our new window, make sure we don't 3088 * increase it due to wscale. 3089 */ 3090 free_space = round_down(free_space, 1 << tp->rx_opt.rcv_wscale); 3091 3092 /* if free space is less than mss estimate, or is below 1/16th 3093 * of the maximum allowed, try to move to zero-window, else 3094 * tcp_clamp_window() will grow rcv buf up to tcp_rmem[2], and 3095 * new incoming data is dropped due to memory limits. 3096 * With large window, mss test triggers way too late in order 3097 * to announce zero window in time before rmem limit kicks in. 3098 */ 3099 if (free_space < (allowed_space >> 4) || free_space < mss) 3100 return 0; 3101 } 3102 3103 if (free_space > tp->rcv_ssthresh) 3104 free_space = tp->rcv_ssthresh; 3105 3106 /* Don't do rounding if we are using window scaling, since the 3107 * scaled window will not line up with the MSS boundary anyway. 3108 */ 3109 if (tp->rx_opt.rcv_wscale) { 3110 window = free_space; 3111 3112 /* Advertise enough space so that it won't get scaled away. 3113 * Import case: prevent zero window announcement if 3114 * 1<<rcv_wscale > mss. 3115 */ 3116 window = ALIGN(window, (1 << tp->rx_opt.rcv_wscale)); 3117 } else { 3118 window = tp->rcv_wnd; 3119 /* Get the largest window that is a nice multiple of mss. 3120 * Window clamp already applied above. 3121 * If our current window offering is within 1 mss of the 3122 * free space we just keep it. This prevents the divide 3123 * and multiply from happening most of the time. 3124 * We also don't do any window rounding when the free space 3125 * is too small. 3126 */ 3127 if (window <= free_space - mss || window > free_space) 3128 window = rounddown(free_space, mss); 3129 else if (mss == full_space && 3130 free_space > window + (full_space >> 1)) 3131 window = free_space; 3132 } 3133 3134 return window; 3135 3136 shrink_window_allowed: 3137 /* new window should always be an exact multiple of scaling factor */ 3138 free_space = round_down(free_space, 1 << tp->rx_opt.rcv_wscale); 3139 3140 if (free_space < (full_space >> 1)) { 3141 icsk->icsk_ack.quick = 0; 3142 3143 if (tcp_under_memory_pressure(sk)) 3144 tcp_adjust_rcv_ssthresh(sk); 3145 3146 /* if free space is too low, return a zero window */ 3147 if (free_space < (allowed_space >> 4) || free_space < mss || 3148 free_space < (1 << tp->rx_opt.rcv_wscale)) 3149 return 0; 3150 } 3151 3152 if (free_space > tp->rcv_ssthresh) { 3153 free_space = tp->rcv_ssthresh; 3154 /* new window should always be an exact multiple of scaling factor 3155 * 3156 * For this case, we ALIGN "up" (increase free_space) because 3157 * we know free_space is not zero here, it has been reduced from 3158 * the memory-based limit, and rcv_ssthresh is not a hard limit 3159 * (unlike sk_rcvbuf). 3160 */ 3161 free_space = ALIGN(free_space, (1 << tp->rx_opt.rcv_wscale)); 3162 } 3163 3164 return free_space; 3165 } 3166 3167 void tcp_skb_collapse_tstamp(struct sk_buff *skb, 3168 const struct sk_buff *next_skb) 3169 { 3170 if (unlikely(tcp_has_tx_tstamp(next_skb))) { 3171 const struct skb_shared_info *next_shinfo = 3172 skb_shinfo(next_skb); 3173 struct skb_shared_info *shinfo = skb_shinfo(skb); 3174 3175 shinfo->tx_flags |= next_shinfo->tx_flags & SKBTX_ANY_TSTAMP; 3176 shinfo->tskey = next_shinfo->tskey; 3177 TCP_SKB_CB(skb)->txstamp_ack |= 3178 TCP_SKB_CB(next_skb)->txstamp_ack; 3179 } 3180 } 3181 3182 /* Collapses two adjacent SKB's during retransmission. */ 3183 static bool tcp_collapse_retrans(struct sock *sk, struct sk_buff *skb) 3184 { 3185 struct tcp_sock *tp = tcp_sk(sk); 3186 struct sk_buff *next_skb = skb_rb_next(skb); 3187 int next_skb_size; 3188 3189 next_skb_size = next_skb->len; 3190 3191 BUG_ON(tcp_skb_pcount(skb) != 1 || tcp_skb_pcount(next_skb) != 1); 3192 3193 if (next_skb_size && !tcp_skb_shift(skb, next_skb, 1, next_skb_size)) 3194 return false; 3195 3196 tcp_highest_sack_replace(sk, next_skb, skb); 3197 3198 /* Update sequence range on original skb. */ 3199 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(next_skb)->end_seq; 3200 3201 /* Merge over control information. This moves PSH/FIN etc. over */ 3202 TCP_SKB_CB(skb)->tcp_flags |= TCP_SKB_CB(next_skb)->tcp_flags; 3203 3204 /* All done, get rid of second SKB and account for it so 3205 * packet counting does not break. 3206 */ 3207 TCP_SKB_CB(skb)->sacked |= TCP_SKB_CB(next_skb)->sacked & TCPCB_EVER_RETRANS; 3208 TCP_SKB_CB(skb)->eor = TCP_SKB_CB(next_skb)->eor; 3209 3210 /* changed transmit queue under us so clear hints */ 3211 tcp_clear_retrans_hints_partial(tp); 3212 if (next_skb == tp->retransmit_skb_hint) 3213 tp->retransmit_skb_hint = skb; 3214 3215 tcp_adjust_pcount(sk, next_skb, tcp_skb_pcount(next_skb)); 3216 3217 tcp_skb_collapse_tstamp(skb, next_skb); 3218 3219 tcp_rtx_queue_unlink_and_free(next_skb, sk); 3220 return true; 3221 } 3222 3223 /* Check if coalescing SKBs is legal. */ 3224 static bool tcp_can_collapse(const struct sock *sk, const struct sk_buff *skb) 3225 { 3226 if (tcp_skb_pcount(skb) > 1) 3227 return false; 3228 if (skb_cloned(skb)) 3229 return false; 3230 /* Some heuristics for collapsing over SACK'd could be invented */ 3231 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED) 3232 return false; 3233 3234 return true; 3235 } 3236 3237 /* Collapse packets in the retransmit queue to make to create 3238 * less packets on the wire. This is only done on retransmission. 3239 */ 3240 static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *to, 3241 int space) 3242 { 3243 struct tcp_sock *tp = tcp_sk(sk); 3244 struct sk_buff *skb = to, *tmp; 3245 bool first = true; 3246 3247 if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_retrans_collapse)) 3248 return; 3249 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN) 3250 return; 3251 3252 skb_rbtree_walk_from_safe(skb, tmp) { 3253 if (!tcp_can_collapse(sk, skb)) 3254 break; 3255 3256 if (!tcp_skb_can_collapse(to, skb)) 3257 break; 3258 3259 space -= skb->len; 3260 3261 if (first) { 3262 first = false; 3263 continue; 3264 } 3265 3266 if (space < 0) 3267 break; 3268 3269 if (after(TCP_SKB_CB(skb)->end_seq, tcp_wnd_end(tp))) 3270 break; 3271 3272 if (!tcp_collapse_retrans(sk, to)) 3273 break; 3274 } 3275 } 3276 3277 /* This retransmits one SKB. Policy decisions and retransmit queue 3278 * state updates are done by the caller. Returns non-zero if an 3279 * error occurred which prevented the send. 3280 */ 3281 int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs) 3282 { 3283 struct inet_connection_sock *icsk = inet_csk(sk); 3284 struct tcp_sock *tp = tcp_sk(sk); 3285 unsigned int cur_mss; 3286 int diff, len, err; 3287 int avail_wnd; 3288 3289 /* Inconclusive MTU probe */ 3290 if (icsk->icsk_mtup.probe_size) 3291 icsk->icsk_mtup.probe_size = 0; 3292 3293 if (skb_still_in_host_queue(sk, skb)) 3294 return -EBUSY; 3295 3296 if (before(TCP_SKB_CB(skb)->seq, tp->snd_una)) { 3297 if (unlikely(before(TCP_SKB_CB(skb)->end_seq, tp->snd_una))) { 3298 WARN_ON_ONCE(1); 3299 return -EINVAL; 3300 } 3301 if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq)) 3302 return -ENOMEM; 3303 } 3304 3305 if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk)) 3306 return -EHOSTUNREACH; /* Routing failure or similar. */ 3307 3308 cur_mss = tcp_current_mss(sk); 3309 avail_wnd = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq; 3310 3311 /* If receiver has shrunk his window, and skb is out of 3312 * new window, do not retransmit it. The exception is the 3313 * case, when window is shrunk to zero. In this case 3314 * our retransmit of one segment serves as a zero window probe. 3315 */ 3316 if (avail_wnd <= 0) { 3317 if (TCP_SKB_CB(skb)->seq != tp->snd_una) 3318 return -EAGAIN; 3319 avail_wnd = cur_mss; 3320 } 3321 3322 len = cur_mss * segs; 3323 if (len > avail_wnd) { 3324 len = rounddown(avail_wnd, cur_mss); 3325 if (!len) 3326 len = avail_wnd; 3327 } 3328 if (skb->len > len) { 3329 if (tcp_fragment(sk, TCP_FRAG_IN_RTX_QUEUE, skb, len, 3330 cur_mss, GFP_ATOMIC)) 3331 return -ENOMEM; /* We'll try again later. */ 3332 } else { 3333 if (skb_unclone_keeptruesize(skb, GFP_ATOMIC)) 3334 return -ENOMEM; 3335 3336 diff = tcp_skb_pcount(skb); 3337 tcp_set_skb_tso_segs(skb, cur_mss); 3338 diff -= tcp_skb_pcount(skb); 3339 if (diff) 3340 tcp_adjust_pcount(sk, skb, diff); 3341 avail_wnd = min_t(int, avail_wnd, cur_mss); 3342 if (skb->len < avail_wnd) 3343 tcp_retrans_try_collapse(sk, skb, avail_wnd); 3344 } 3345 3346 /* RFC3168, section 6.1.1.1. ECN fallback */ 3347 if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN_ECN) == TCPHDR_SYN_ECN) 3348 tcp_ecn_clear_syn(sk, skb); 3349 3350 /* Update global and local TCP statistics. */ 3351 segs = tcp_skb_pcount(skb); 3352 TCP_ADD_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS, segs); 3353 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN) 3354 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS); 3355 tp->total_retrans += segs; 3356 tp->bytes_retrans += skb->len; 3357 3358 /* make sure skb->data is aligned on arches that require it 3359 * and check if ack-trimming & collapsing extended the headroom 3360 * beyond what csum_start can cover. 3361 */ 3362 if (unlikely((NET_IP_ALIGN && ((unsigned long)skb->data & 3)) || 3363 skb_headroom(skb) >= 0xFFFF)) { 3364 struct sk_buff *nskb; 3365 3366 tcp_skb_tsorted_save(skb) { 3367 nskb = __pskb_copy(skb, MAX_TCP_HEADER, GFP_ATOMIC); 3368 if (nskb) { 3369 nskb->dev = NULL; 3370 err = tcp_transmit_skb(sk, nskb, 0, GFP_ATOMIC); 3371 } else { 3372 err = -ENOBUFS; 3373 } 3374 } tcp_skb_tsorted_restore(skb); 3375 3376 if (!err) { 3377 tcp_update_skb_after_send(sk, skb, tp->tcp_wstamp_ns); 3378 tcp_rate_skb_sent(sk, skb); 3379 } 3380 } else { 3381 err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC); 3382 } 3383 3384 /* To avoid taking spuriously low RTT samples based on a timestamp 3385 * for a transmit that never happened, always mark EVER_RETRANS 3386 */ 3387 TCP_SKB_CB(skb)->sacked |= TCPCB_EVER_RETRANS; 3388 3389 if (BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_RETRANS_CB_FLAG)) 3390 tcp_call_bpf_3arg(sk, BPF_SOCK_OPS_RETRANS_CB, 3391 TCP_SKB_CB(skb)->seq, segs, err); 3392 3393 if (likely(!err)) { 3394 trace_tcp_retransmit_skb(sk, skb); 3395 } else if (err != -EBUSY) { 3396 NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPRETRANSFAIL, segs); 3397 } 3398 return err; 3399 } 3400 3401 int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs) 3402 { 3403 struct tcp_sock *tp = tcp_sk(sk); 3404 int err = __tcp_retransmit_skb(sk, skb, segs); 3405 3406 if (err == 0) { 3407 #if FASTRETRANS_DEBUG > 0 3408 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) { 3409 net_dbg_ratelimited("retrans_out leaked\n"); 3410 } 3411 #endif 3412 TCP_SKB_CB(skb)->sacked |= TCPCB_RETRANS; 3413 tp->retrans_out += tcp_skb_pcount(skb); 3414 } 3415 3416 /* Save stamp of the first (attempted) retransmit. */ 3417 if (!tp->retrans_stamp) 3418 tp->retrans_stamp = tcp_skb_timestamp_ts(tp->tcp_usec_ts, skb); 3419 3420 if (tp->undo_retrans < 0) 3421 tp->undo_retrans = 0; 3422 tp->undo_retrans += tcp_skb_pcount(skb); 3423 return err; 3424 } 3425 3426 /* This gets called after a retransmit timeout, and the initially 3427 * retransmitted data is acknowledged. It tries to continue 3428 * resending the rest of the retransmit queue, until either 3429 * we've sent it all or the congestion window limit is reached. 3430 */ 3431 void tcp_xmit_retransmit_queue(struct sock *sk) 3432 { 3433 const struct inet_connection_sock *icsk = inet_csk(sk); 3434 struct sk_buff *skb, *rtx_head, *hole = NULL; 3435 struct tcp_sock *tp = tcp_sk(sk); 3436 bool rearm_timer = false; 3437 u32 max_segs; 3438 int mib_idx; 3439 3440 if (!tp->packets_out) 3441 return; 3442 3443 rtx_head = tcp_rtx_queue_head(sk); 3444 skb = tp->retransmit_skb_hint ?: rtx_head; 3445 max_segs = tcp_tso_segs(sk, tcp_current_mss(sk)); 3446 skb_rbtree_walk_from(skb) { 3447 __u8 sacked; 3448 int segs; 3449 3450 if (tcp_pacing_check(sk)) 3451 break; 3452 3453 /* we could do better than to assign each time */ 3454 if (!hole) 3455 tp->retransmit_skb_hint = skb; 3456 3457 segs = tcp_snd_cwnd(tp) - tcp_packets_in_flight(tp); 3458 if (segs <= 0) 3459 break; 3460 sacked = TCP_SKB_CB(skb)->sacked; 3461 /* In case tcp_shift_skb_data() have aggregated large skbs, 3462 * we need to make sure not sending too bigs TSO packets 3463 */ 3464 segs = min_t(int, segs, max_segs); 3465 3466 if (tp->retrans_out >= tp->lost_out) { 3467 break; 3468 } else if (!(sacked & TCPCB_LOST)) { 3469 if (!hole && !(sacked & (TCPCB_SACKED_RETRANS|TCPCB_SACKED_ACKED))) 3470 hole = skb; 3471 continue; 3472 3473 } else { 3474 if (icsk->icsk_ca_state != TCP_CA_Loss) 3475 mib_idx = LINUX_MIB_TCPFASTRETRANS; 3476 else 3477 mib_idx = LINUX_MIB_TCPSLOWSTARTRETRANS; 3478 } 3479 3480 if (sacked & (TCPCB_SACKED_ACKED|TCPCB_SACKED_RETRANS)) 3481 continue; 3482 3483 if (tcp_small_queue_check(sk, skb, 1)) 3484 break; 3485 3486 if (tcp_retransmit_skb(sk, skb, segs)) 3487 break; 3488 3489 NET_ADD_STATS(sock_net(sk), mib_idx, tcp_skb_pcount(skb)); 3490 3491 if (tcp_in_cwnd_reduction(sk)) 3492 tp->prr_out += tcp_skb_pcount(skb); 3493 3494 if (skb == rtx_head && 3495 icsk->icsk_pending != ICSK_TIME_REO_TIMEOUT) 3496 rearm_timer = true; 3497 3498 } 3499 if (rearm_timer) 3500 tcp_reset_xmit_timer(sk, ICSK_TIME_RETRANS, 3501 inet_csk(sk)->icsk_rto, 3502 TCP_RTO_MAX); 3503 } 3504 3505 /* We allow to exceed memory limits for FIN packets to expedite 3506 * connection tear down and (memory) recovery. 3507 * Otherwise tcp_send_fin() could be tempted to either delay FIN 3508 * or even be forced to close flow without any FIN. 3509 * In general, we want to allow one skb per socket to avoid hangs 3510 * with edge trigger epoll() 3511 */ 3512 void sk_forced_mem_schedule(struct sock *sk, int size) 3513 { 3514 int delta, amt; 3515 3516 delta = size - sk->sk_forward_alloc; 3517 if (delta <= 0) 3518 return; 3519 amt = sk_mem_pages(delta); 3520 sk_forward_alloc_add(sk, amt << PAGE_SHIFT); 3521 sk_memory_allocated_add(sk, amt); 3522 3523 if (mem_cgroup_sockets_enabled && sk->sk_memcg) 3524 mem_cgroup_charge_skmem(sk->sk_memcg, amt, 3525 gfp_memcg_charge() | __GFP_NOFAIL); 3526 } 3527 3528 /* Send a FIN. The caller locks the socket for us. 3529 * We should try to send a FIN packet really hard, but eventually give up. 3530 */ 3531 void tcp_send_fin(struct sock *sk) 3532 { 3533 struct sk_buff *skb, *tskb, *tail = tcp_write_queue_tail(sk); 3534 struct tcp_sock *tp = tcp_sk(sk); 3535 3536 /* Optimization, tack on the FIN if we have one skb in write queue and 3537 * this skb was not yet sent, or we are under memory pressure. 3538 * Note: in the latter case, FIN packet will be sent after a timeout, 3539 * as TCP stack thinks it has already been transmitted. 3540 */ 3541 tskb = tail; 3542 if (!tskb && tcp_under_memory_pressure(sk)) 3543 tskb = skb_rb_last(&sk->tcp_rtx_queue); 3544 3545 if (tskb) { 3546 TCP_SKB_CB(tskb)->tcp_flags |= TCPHDR_FIN; 3547 TCP_SKB_CB(tskb)->end_seq++; 3548 tp->write_seq++; 3549 if (!tail) { 3550 /* This means tskb was already sent. 3551 * Pretend we included the FIN on previous transmit. 3552 * We need to set tp->snd_nxt to the value it would have 3553 * if FIN had been sent. This is because retransmit path 3554 * does not change tp->snd_nxt. 3555 */ 3556 WRITE_ONCE(tp->snd_nxt, tp->snd_nxt + 1); 3557 return; 3558 } 3559 } else { 3560 skb = alloc_skb_fclone(MAX_TCP_HEADER, sk->sk_allocation); 3561 if (unlikely(!skb)) 3562 return; 3563 3564 INIT_LIST_HEAD(&skb->tcp_tsorted_anchor); 3565 skb_reserve(skb, MAX_TCP_HEADER); 3566 sk_forced_mem_schedule(sk, skb->truesize); 3567 /* FIN eats a sequence byte, write_seq advanced by tcp_queue_skb(). */ 3568 tcp_init_nondata_skb(skb, tp->write_seq, 3569 TCPHDR_ACK | TCPHDR_FIN); 3570 tcp_queue_skb(sk, skb); 3571 } 3572 __tcp_push_pending_frames(sk, tcp_current_mss(sk), TCP_NAGLE_OFF); 3573 } 3574 3575 /* We get here when a process closes a file descriptor (either due to 3576 * an explicit close() or as a byproduct of exit()'ing) and there 3577 * was unread data in the receive queue. This behavior is recommended 3578 * by RFC 2525, section 2.17. -DaveM 3579 */ 3580 void tcp_send_active_reset(struct sock *sk, gfp_t priority) 3581 { 3582 struct sk_buff *skb; 3583 3584 TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTRSTS); 3585 3586 /* NOTE: No TCP options attached and we never retransmit this. */ 3587 skb = alloc_skb(MAX_TCP_HEADER, priority); 3588 if (!skb) { 3589 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED); 3590 return; 3591 } 3592 3593 /* Reserve space for headers and prepare control bits. */ 3594 skb_reserve(skb, MAX_TCP_HEADER); 3595 tcp_init_nondata_skb(skb, tcp_acceptable_seq(sk), 3596 TCPHDR_ACK | TCPHDR_RST); 3597 tcp_mstamp_refresh(tcp_sk(sk)); 3598 /* Send it off. */ 3599 if (tcp_transmit_skb(sk, skb, 0, priority)) 3600 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED); 3601 3602 /* skb of trace_tcp_send_reset() keeps the skb that caused RST, 3603 * skb here is different to the troublesome skb, so use NULL 3604 */ 3605 trace_tcp_send_reset(sk, NULL); 3606 } 3607 3608 /* Send a crossed SYN-ACK during socket establishment. 3609 * WARNING: This routine must only be called when we have already sent 3610 * a SYN packet that crossed the incoming SYN that caused this routine 3611 * to get called. If this assumption fails then the initial rcv_wnd 3612 * and rcv_wscale values will not be correct. 3613 */ 3614 int tcp_send_synack(struct sock *sk) 3615 { 3616 struct sk_buff *skb; 3617 3618 skb = tcp_rtx_queue_head(sk); 3619 if (!skb || !(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) { 3620 pr_err("%s: wrong queue state\n", __func__); 3621 return -EFAULT; 3622 } 3623 if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_ACK)) { 3624 if (skb_cloned(skb)) { 3625 struct sk_buff *nskb; 3626 3627 tcp_skb_tsorted_save(skb) { 3628 nskb = skb_copy(skb, GFP_ATOMIC); 3629 } tcp_skb_tsorted_restore(skb); 3630 if (!nskb) 3631 return -ENOMEM; 3632 INIT_LIST_HEAD(&nskb->tcp_tsorted_anchor); 3633 tcp_highest_sack_replace(sk, skb, nskb); 3634 tcp_rtx_queue_unlink_and_free(skb, sk); 3635 __skb_header_release(nskb); 3636 tcp_rbtree_insert(&sk->tcp_rtx_queue, nskb); 3637 sk_wmem_queued_add(sk, nskb->truesize); 3638 sk_mem_charge(sk, nskb->truesize); 3639 skb = nskb; 3640 } 3641 3642 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ACK; 3643 tcp_ecn_send_synack(sk, skb); 3644 } 3645 return tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC); 3646 } 3647 3648 /** 3649 * tcp_make_synack - Allocate one skb and build a SYNACK packet. 3650 * @sk: listener socket 3651 * @dst: dst entry attached to the SYNACK. It is consumed and caller 3652 * should not use it again. 3653 * @req: request_sock pointer 3654 * @foc: cookie for tcp fast open 3655 * @synack_type: Type of synack to prepare 3656 * @syn_skb: SYN packet just received. It could be NULL for rtx case. 3657 */ 3658 struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst, 3659 struct request_sock *req, 3660 struct tcp_fastopen_cookie *foc, 3661 enum tcp_synack_type synack_type, 3662 struct sk_buff *syn_skb) 3663 { 3664 struct inet_request_sock *ireq = inet_rsk(req); 3665 const struct tcp_sock *tp = tcp_sk(sk); 3666 struct tcp_out_options opts; 3667 struct tcp_key key = {}; 3668 struct sk_buff *skb; 3669 int tcp_header_size; 3670 struct tcphdr *th; 3671 int mss; 3672 u64 now; 3673 3674 skb = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC); 3675 if (unlikely(!skb)) { 3676 dst_release(dst); 3677 return NULL; 3678 } 3679 /* Reserve space for headers. */ 3680 skb_reserve(skb, MAX_TCP_HEADER); 3681 3682 switch (synack_type) { 3683 case TCP_SYNACK_NORMAL: 3684 skb_set_owner_w(skb, req_to_sk(req)); 3685 break; 3686 case TCP_SYNACK_COOKIE: 3687 /* Under synflood, we do not attach skb to a socket, 3688 * to avoid false sharing. 3689 */ 3690 break; 3691 case TCP_SYNACK_FASTOPEN: 3692 /* sk is a const pointer, because we want to express multiple 3693 * cpu might call us concurrently. 3694 * sk->sk_wmem_alloc in an atomic, we can promote to rw. 3695 */ 3696 skb_set_owner_w(skb, (struct sock *)sk); 3697 break; 3698 } 3699 skb_dst_set(skb, dst); 3700 3701 mss = tcp_mss_clamp(tp, dst_metric_advmss(dst)); 3702 3703 memset(&opts, 0, sizeof(opts)); 3704 now = tcp_clock_ns(); 3705 #ifdef CONFIG_SYN_COOKIES 3706 if (unlikely(synack_type == TCP_SYNACK_COOKIE && ireq->tstamp_ok)) 3707 skb_set_delivery_time(skb, cookie_init_timestamp(req, now), 3708 true); 3709 else 3710 #endif 3711 { 3712 skb_set_delivery_time(skb, now, true); 3713 if (!tcp_rsk(req)->snt_synack) /* Timestamp first SYNACK */ 3714 tcp_rsk(req)->snt_synack = tcp_skb_timestamp_us(skb); 3715 } 3716 3717 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 3718 rcu_read_lock(); 3719 #endif 3720 if (tcp_rsk_used_ao(req)) { 3721 #ifdef CONFIG_TCP_AO 3722 struct tcp_ao_key *ao_key = NULL; 3723 u8 keyid = tcp_rsk(req)->ao_keyid; 3724 3725 ao_key = tcp_sk(sk)->af_specific->ao_lookup(sk, req_to_sk(req), 3726 keyid, -1); 3727 /* If there is no matching key - avoid sending anything, 3728 * especially usigned segments. It could try harder and lookup 3729 * for another peer-matching key, but the peer has requested 3730 * ao_keyid (RFC5925 RNextKeyID), so let's keep it simple here. 3731 */ 3732 if (unlikely(!ao_key)) { 3733 rcu_read_unlock(); 3734 kfree_skb(skb); 3735 net_warn_ratelimited("TCP-AO: the keyid %u from SYN packet is not present - not sending SYNACK\n", 3736 keyid); 3737 return NULL; 3738 } 3739 key.ao_key = ao_key; 3740 key.type = TCP_KEY_AO; 3741 #endif 3742 } else { 3743 #ifdef CONFIG_TCP_MD5SIG 3744 key.md5_key = tcp_rsk(req)->af_specific->req_md5_lookup(sk, 3745 req_to_sk(req)); 3746 if (key.md5_key) 3747 key.type = TCP_KEY_MD5; 3748 #endif 3749 } 3750 skb_set_hash(skb, READ_ONCE(tcp_rsk(req)->txhash), PKT_HASH_TYPE_L4); 3751 /* bpf program will be interested in the tcp_flags */ 3752 TCP_SKB_CB(skb)->tcp_flags = TCPHDR_SYN | TCPHDR_ACK; 3753 tcp_header_size = tcp_synack_options(sk, req, mss, skb, &opts, 3754 &key, foc, synack_type, syn_skb) 3755 + sizeof(*th); 3756 3757 skb_push(skb, tcp_header_size); 3758 skb_reset_transport_header(skb); 3759 3760 th = (struct tcphdr *)skb->data; 3761 memset(th, 0, sizeof(struct tcphdr)); 3762 th->syn = 1; 3763 th->ack = 1; 3764 tcp_ecn_make_synack(req, th); 3765 th->source = htons(ireq->ir_num); 3766 th->dest = ireq->ir_rmt_port; 3767 skb->mark = ireq->ir_mark; 3768 skb->ip_summed = CHECKSUM_PARTIAL; 3769 th->seq = htonl(tcp_rsk(req)->snt_isn); 3770 /* XXX data is queued and acked as is. No buffer/window check */ 3771 th->ack_seq = htonl(tcp_rsk(req)->rcv_nxt); 3772 3773 /* RFC1323: The window in SYN & SYN/ACK segments is never scaled. */ 3774 th->window = htons(min(req->rsk_rcv_wnd, 65535U)); 3775 tcp_options_write(th, NULL, tcp_rsk(req), &opts, &key); 3776 th->doff = (tcp_header_size >> 2); 3777 TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTSEGS); 3778 3779 /* Okay, we have all we need - do the md5 hash if needed */ 3780 if (tcp_key_is_md5(&key)) { 3781 #ifdef CONFIG_TCP_MD5SIG 3782 tcp_rsk(req)->af_specific->calc_md5_hash(opts.hash_location, 3783 key.md5_key, req_to_sk(req), skb); 3784 #endif 3785 } else if (tcp_key_is_ao(&key)) { 3786 #ifdef CONFIG_TCP_AO 3787 tcp_rsk(req)->af_specific->ao_synack_hash(opts.hash_location, 3788 key.ao_key, req, skb, 3789 opts.hash_location - (u8 *)th, 0); 3790 #endif 3791 } 3792 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 3793 rcu_read_unlock(); 3794 #endif 3795 3796 bpf_skops_write_hdr_opt((struct sock *)sk, skb, req, syn_skb, 3797 synack_type, &opts); 3798 3799 skb_set_delivery_time(skb, now, true); 3800 tcp_add_tx_delay(skb, tp); 3801 3802 return skb; 3803 } 3804 EXPORT_SYMBOL(tcp_make_synack); 3805 3806 static void tcp_ca_dst_init(struct sock *sk, const struct dst_entry *dst) 3807 { 3808 struct inet_connection_sock *icsk = inet_csk(sk); 3809 const struct tcp_congestion_ops *ca; 3810 u32 ca_key = dst_metric(dst, RTAX_CC_ALGO); 3811 3812 if (ca_key == TCP_CA_UNSPEC) 3813 return; 3814 3815 rcu_read_lock(); 3816 ca = tcp_ca_find_key(ca_key); 3817 if (likely(ca && bpf_try_module_get(ca, ca->owner))) { 3818 bpf_module_put(icsk->icsk_ca_ops, icsk->icsk_ca_ops->owner); 3819 icsk->icsk_ca_dst_locked = tcp_ca_dst_locked(dst); 3820 icsk->icsk_ca_ops = ca; 3821 } 3822 rcu_read_unlock(); 3823 } 3824 3825 /* Do all connect socket setups that can be done AF independent. */ 3826 static void tcp_connect_init(struct sock *sk) 3827 { 3828 const struct dst_entry *dst = __sk_dst_get(sk); 3829 struct tcp_sock *tp = tcp_sk(sk); 3830 __u8 rcv_wscale; 3831 u32 rcv_wnd; 3832 3833 /* We'll fix this up when we get a response from the other end. 3834 * See tcp_input.c:tcp_rcv_state_process case TCP_SYN_SENT. 3835 */ 3836 tp->tcp_header_len = sizeof(struct tcphdr); 3837 if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_timestamps)) 3838 tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED; 3839 3840 tcp_ao_connect_init(sk); 3841 3842 /* If user gave his TCP_MAXSEG, record it to clamp */ 3843 if (tp->rx_opt.user_mss) 3844 tp->rx_opt.mss_clamp = tp->rx_opt.user_mss; 3845 tp->max_window = 0; 3846 tcp_mtup_init(sk); 3847 tcp_sync_mss(sk, dst_mtu(dst)); 3848 3849 tcp_ca_dst_init(sk, dst); 3850 3851 if (!tp->window_clamp) 3852 tp->window_clamp = dst_metric(dst, RTAX_WINDOW); 3853 tp->advmss = tcp_mss_clamp(tp, dst_metric_advmss(dst)); 3854 3855 tcp_initialize_rcv_mss(sk); 3856 3857 /* limit the window selection if the user enforce a smaller rx buffer */ 3858 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK && 3859 (tp->window_clamp > tcp_full_space(sk) || tp->window_clamp == 0)) 3860 tp->window_clamp = tcp_full_space(sk); 3861 3862 rcv_wnd = tcp_rwnd_init_bpf(sk); 3863 if (rcv_wnd == 0) 3864 rcv_wnd = dst_metric(dst, RTAX_INITRWND); 3865 3866 tcp_select_initial_window(sk, tcp_full_space(sk), 3867 tp->advmss - (tp->rx_opt.ts_recent_stamp ? tp->tcp_header_len - sizeof(struct tcphdr) : 0), 3868 &tp->rcv_wnd, 3869 &tp->window_clamp, 3870 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_window_scaling), 3871 &rcv_wscale, 3872 rcv_wnd); 3873 3874 tp->rx_opt.rcv_wscale = rcv_wscale; 3875 tp->rcv_ssthresh = tp->rcv_wnd; 3876 3877 WRITE_ONCE(sk->sk_err, 0); 3878 sock_reset_flag(sk, SOCK_DONE); 3879 tp->snd_wnd = 0; 3880 tcp_init_wl(tp, 0); 3881 tcp_write_queue_purge(sk); 3882 tp->snd_una = tp->write_seq; 3883 tp->snd_sml = tp->write_seq; 3884 tp->snd_up = tp->write_seq; 3885 WRITE_ONCE(tp->snd_nxt, tp->write_seq); 3886 3887 if (likely(!tp->repair)) 3888 tp->rcv_nxt = 0; 3889 else 3890 tp->rcv_tstamp = tcp_jiffies32; 3891 tp->rcv_wup = tp->rcv_nxt; 3892 WRITE_ONCE(tp->copied_seq, tp->rcv_nxt); 3893 3894 inet_csk(sk)->icsk_rto = tcp_timeout_init(sk); 3895 inet_csk(sk)->icsk_retransmits = 0; 3896 tcp_clear_retrans(tp); 3897 } 3898 3899 static void tcp_connect_queue_skb(struct sock *sk, struct sk_buff *skb) 3900 { 3901 struct tcp_sock *tp = tcp_sk(sk); 3902 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb); 3903 3904 tcb->end_seq += skb->len; 3905 __skb_header_release(skb); 3906 sk_wmem_queued_add(sk, skb->truesize); 3907 sk_mem_charge(sk, skb->truesize); 3908 WRITE_ONCE(tp->write_seq, tcb->end_seq); 3909 tp->packets_out += tcp_skb_pcount(skb); 3910 } 3911 3912 /* Build and send a SYN with data and (cached) Fast Open cookie. However, 3913 * queue a data-only packet after the regular SYN, such that regular SYNs 3914 * are retransmitted on timeouts. Also if the remote SYN-ACK acknowledges 3915 * only the SYN sequence, the data are retransmitted in the first ACK. 3916 * If cookie is not cached or other error occurs, falls back to send a 3917 * regular SYN with Fast Open cookie request option. 3918 */ 3919 static int tcp_send_syn_data(struct sock *sk, struct sk_buff *syn) 3920 { 3921 struct inet_connection_sock *icsk = inet_csk(sk); 3922 struct tcp_sock *tp = tcp_sk(sk); 3923 struct tcp_fastopen_request *fo = tp->fastopen_req; 3924 struct page_frag *pfrag = sk_page_frag(sk); 3925 struct sk_buff *syn_data; 3926 int space, err = 0; 3927 3928 tp->rx_opt.mss_clamp = tp->advmss; /* If MSS is not cached */ 3929 if (!tcp_fastopen_cookie_check(sk, &tp->rx_opt.mss_clamp, &fo->cookie)) 3930 goto fallback; 3931 3932 /* MSS for SYN-data is based on cached MSS and bounded by PMTU and 3933 * user-MSS. Reserve maximum option space for middleboxes that add 3934 * private TCP options. The cost is reduced data space in SYN :( 3935 */ 3936 tp->rx_opt.mss_clamp = tcp_mss_clamp(tp, tp->rx_opt.mss_clamp); 3937 /* Sync mss_cache after updating the mss_clamp */ 3938 tcp_sync_mss(sk, icsk->icsk_pmtu_cookie); 3939 3940 space = __tcp_mtu_to_mss(sk, icsk->icsk_pmtu_cookie) - 3941 MAX_TCP_OPTION_SPACE; 3942 3943 space = min_t(size_t, space, fo->size); 3944 3945 if (space && 3946 !skb_page_frag_refill(min_t(size_t, space, PAGE_SIZE), 3947 pfrag, sk->sk_allocation)) 3948 goto fallback; 3949 syn_data = tcp_stream_alloc_skb(sk, sk->sk_allocation, false); 3950 if (!syn_data) 3951 goto fallback; 3952 memcpy(syn_data->cb, syn->cb, sizeof(syn->cb)); 3953 if (space) { 3954 space = min_t(size_t, space, pfrag->size - pfrag->offset); 3955 space = tcp_wmem_schedule(sk, space); 3956 } 3957 if (space) { 3958 space = copy_page_from_iter(pfrag->page, pfrag->offset, 3959 space, &fo->data->msg_iter); 3960 if (unlikely(!space)) { 3961 tcp_skb_tsorted_anchor_cleanup(syn_data); 3962 kfree_skb(syn_data); 3963 goto fallback; 3964 } 3965 skb_fill_page_desc(syn_data, 0, pfrag->page, 3966 pfrag->offset, space); 3967 page_ref_inc(pfrag->page); 3968 pfrag->offset += space; 3969 skb_len_add(syn_data, space); 3970 skb_zcopy_set(syn_data, fo->uarg, NULL); 3971 } 3972 /* No more data pending in inet_wait_for_connect() */ 3973 if (space == fo->size) 3974 fo->data = NULL; 3975 fo->copied = space; 3976 3977 tcp_connect_queue_skb(sk, syn_data); 3978 if (syn_data->len) 3979 tcp_chrono_start(sk, TCP_CHRONO_BUSY); 3980 3981 err = tcp_transmit_skb(sk, syn_data, 1, sk->sk_allocation); 3982 3983 skb_set_delivery_time(syn, syn_data->skb_mstamp_ns, true); 3984 3985 /* Now full SYN+DATA was cloned and sent (or not), 3986 * remove the SYN from the original skb (syn_data) 3987 * we keep in write queue in case of a retransmit, as we 3988 * also have the SYN packet (with no data) in the same queue. 3989 */ 3990 TCP_SKB_CB(syn_data)->seq++; 3991 TCP_SKB_CB(syn_data)->tcp_flags = TCPHDR_ACK | TCPHDR_PSH; 3992 if (!err) { 3993 tp->syn_data = (fo->copied > 0); 3994 tcp_rbtree_insert(&sk->tcp_rtx_queue, syn_data); 3995 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT); 3996 goto done; 3997 } 3998 3999 /* data was not sent, put it in write_queue */ 4000 __skb_queue_tail(&sk->sk_write_queue, syn_data); 4001 tp->packets_out -= tcp_skb_pcount(syn_data); 4002 4003 fallback: 4004 /* Send a regular SYN with Fast Open cookie request option */ 4005 if (fo->cookie.len > 0) 4006 fo->cookie.len = 0; 4007 err = tcp_transmit_skb(sk, syn, 1, sk->sk_allocation); 4008 if (err) 4009 tp->syn_fastopen = 0; 4010 done: 4011 fo->cookie.len = -1; /* Exclude Fast Open option for SYN retries */ 4012 return err; 4013 } 4014 4015 /* Build a SYN and send it off. */ 4016 int tcp_connect(struct sock *sk) 4017 { 4018 struct tcp_sock *tp = tcp_sk(sk); 4019 struct sk_buff *buff; 4020 int err; 4021 4022 tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_CONNECT_CB, 0, NULL); 4023 4024 #if defined(CONFIG_TCP_MD5SIG) && defined(CONFIG_TCP_AO) 4025 /* Has to be checked late, after setting daddr/saddr/ops. 4026 * Return error if the peer has both a md5 and a tcp-ao key 4027 * configured as this is ambiguous. 4028 */ 4029 if (unlikely(rcu_dereference_protected(tp->md5sig_info, 4030 lockdep_sock_is_held(sk)))) { 4031 bool needs_ao = !!tp->af_specific->ao_lookup(sk, sk, -1, -1); 4032 bool needs_md5 = !!tp->af_specific->md5_lookup(sk, sk); 4033 struct tcp_ao_info *ao_info; 4034 4035 ao_info = rcu_dereference_check(tp->ao_info, 4036 lockdep_sock_is_held(sk)); 4037 if (ao_info) { 4038 /* This is an extra check: tcp_ao_required() in 4039 * tcp_v{4,6}_parse_md5_keys() should prevent adding 4040 * md5 keys on ao_required socket. 4041 */ 4042 needs_ao |= ao_info->ao_required; 4043 WARN_ON_ONCE(ao_info->ao_required && needs_md5); 4044 } 4045 if (needs_md5 && needs_ao) 4046 return -EKEYREJECTED; 4047 4048 /* If we have a matching md5 key and no matching tcp-ao key 4049 * then free up ao_info if allocated. 4050 */ 4051 if (needs_md5) { 4052 tcp_ao_destroy_sock(sk, false); 4053 } else if (needs_ao) { 4054 tcp_clear_md5_list(sk); 4055 kfree(rcu_replace_pointer(tp->md5sig_info, NULL, 4056 lockdep_sock_is_held(sk))); 4057 } 4058 } 4059 #endif 4060 #ifdef CONFIG_TCP_AO 4061 if (unlikely(rcu_dereference_protected(tp->ao_info, 4062 lockdep_sock_is_held(sk)))) { 4063 /* Don't allow connecting if ao is configured but no 4064 * matching key is found. 4065 */ 4066 if (!tp->af_specific->ao_lookup(sk, sk, -1, -1)) 4067 return -EKEYREJECTED; 4068 } 4069 #endif 4070 4071 if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk)) 4072 return -EHOSTUNREACH; /* Routing failure or similar. */ 4073 4074 tcp_connect_init(sk); 4075 4076 if (unlikely(tp->repair)) { 4077 tcp_finish_connect(sk, NULL); 4078 return 0; 4079 } 4080 4081 buff = tcp_stream_alloc_skb(sk, sk->sk_allocation, true); 4082 if (unlikely(!buff)) 4083 return -ENOBUFS; 4084 4085 tcp_init_nondata_skb(buff, tp->write_seq++, TCPHDR_SYN); 4086 tcp_mstamp_refresh(tp); 4087 tp->retrans_stamp = tcp_time_stamp_ts(tp); 4088 tcp_connect_queue_skb(sk, buff); 4089 tcp_ecn_send_syn(sk, buff); 4090 tcp_rbtree_insert(&sk->tcp_rtx_queue, buff); 4091 4092 /* Send off SYN; include data in Fast Open. */ 4093 err = tp->fastopen_req ? tcp_send_syn_data(sk, buff) : 4094 tcp_transmit_skb(sk, buff, 1, sk->sk_allocation); 4095 if (err == -ECONNREFUSED) 4096 return err; 4097 4098 /* We change tp->snd_nxt after the tcp_transmit_skb() call 4099 * in order to make this packet get counted in tcpOutSegs. 4100 */ 4101 WRITE_ONCE(tp->snd_nxt, tp->write_seq); 4102 tp->pushed_seq = tp->write_seq; 4103 buff = tcp_send_head(sk); 4104 if (unlikely(buff)) { 4105 WRITE_ONCE(tp->snd_nxt, TCP_SKB_CB(buff)->seq); 4106 tp->pushed_seq = TCP_SKB_CB(buff)->seq; 4107 } 4108 TCP_INC_STATS(sock_net(sk), TCP_MIB_ACTIVEOPENS); 4109 4110 /* Timer for repeating the SYN until an answer. */ 4111 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, 4112 inet_csk(sk)->icsk_rto, TCP_RTO_MAX); 4113 return 0; 4114 } 4115 EXPORT_SYMBOL(tcp_connect); 4116 4117 u32 tcp_delack_max(const struct sock *sk) 4118 { 4119 const struct dst_entry *dst = __sk_dst_get(sk); 4120 u32 delack_max = inet_csk(sk)->icsk_delack_max; 4121 4122 if (dst && dst_metric_locked(dst, RTAX_RTO_MIN)) { 4123 u32 rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN); 4124 u32 delack_from_rto_min = max_t(int, 1, rto_min - 1); 4125 4126 delack_max = min_t(u32, delack_max, delack_from_rto_min); 4127 } 4128 return delack_max; 4129 } 4130 4131 /* Send out a delayed ack, the caller does the policy checking 4132 * to see if we should even be here. See tcp_input.c:tcp_ack_snd_check() 4133 * for details. 4134 */ 4135 void tcp_send_delayed_ack(struct sock *sk) 4136 { 4137 struct inet_connection_sock *icsk = inet_csk(sk); 4138 int ato = icsk->icsk_ack.ato; 4139 unsigned long timeout; 4140 4141 if (ato > TCP_DELACK_MIN) { 4142 const struct tcp_sock *tp = tcp_sk(sk); 4143 int max_ato = HZ / 2; 4144 4145 if (inet_csk_in_pingpong_mode(sk) || 4146 (icsk->icsk_ack.pending & ICSK_ACK_PUSHED)) 4147 max_ato = TCP_DELACK_MAX; 4148 4149 /* Slow path, intersegment interval is "high". */ 4150 4151 /* If some rtt estimate is known, use it to bound delayed ack. 4152 * Do not use inet_csk(sk)->icsk_rto here, use results of rtt measurements 4153 * directly. 4154 */ 4155 if (tp->srtt_us) { 4156 int rtt = max_t(int, usecs_to_jiffies(tp->srtt_us >> 3), 4157 TCP_DELACK_MIN); 4158 4159 if (rtt < max_ato) 4160 max_ato = rtt; 4161 } 4162 4163 ato = min(ato, max_ato); 4164 } 4165 4166 ato = min_t(u32, ato, tcp_delack_max(sk)); 4167 4168 /* Stay within the limit we were given */ 4169 timeout = jiffies + ato; 4170 4171 /* Use new timeout only if there wasn't a older one earlier. */ 4172 if (icsk->icsk_ack.pending & ICSK_ACK_TIMER) { 4173 /* If delack timer is about to expire, send ACK now. */ 4174 if (time_before_eq(icsk->icsk_ack.timeout, jiffies + (ato >> 2))) { 4175 tcp_send_ack(sk); 4176 return; 4177 } 4178 4179 if (!time_before(timeout, icsk->icsk_ack.timeout)) 4180 timeout = icsk->icsk_ack.timeout; 4181 } 4182 icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER; 4183 icsk->icsk_ack.timeout = timeout; 4184 sk_reset_timer(sk, &icsk->icsk_delack_timer, timeout); 4185 } 4186 4187 /* This routine sends an ack and also updates the window. */ 4188 void __tcp_send_ack(struct sock *sk, u32 rcv_nxt) 4189 { 4190 struct sk_buff *buff; 4191 4192 /* If we have been reset, we may not send again. */ 4193 if (sk->sk_state == TCP_CLOSE) 4194 return; 4195 4196 /* We are not putting this on the write queue, so 4197 * tcp_transmit_skb() will set the ownership to this 4198 * sock. 4199 */ 4200 buff = alloc_skb(MAX_TCP_HEADER, 4201 sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN)); 4202 if (unlikely(!buff)) { 4203 struct inet_connection_sock *icsk = inet_csk(sk); 4204 unsigned long delay; 4205 4206 delay = TCP_DELACK_MAX << icsk->icsk_ack.retry; 4207 if (delay < TCP_RTO_MAX) 4208 icsk->icsk_ack.retry++; 4209 inet_csk_schedule_ack(sk); 4210 icsk->icsk_ack.ato = TCP_ATO_MIN; 4211 inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK, delay, TCP_RTO_MAX); 4212 return; 4213 } 4214 4215 /* Reserve space for headers and prepare control bits. */ 4216 skb_reserve(buff, MAX_TCP_HEADER); 4217 tcp_init_nondata_skb(buff, tcp_acceptable_seq(sk), TCPHDR_ACK); 4218 4219 /* We do not want pure acks influencing TCP Small Queues or fq/pacing 4220 * too much. 4221 * SKB_TRUESIZE(max(1 .. 66, MAX_TCP_HEADER)) is unfortunately ~784 4222 */ 4223 skb_set_tcp_pure_ack(buff); 4224 4225 /* Send it off, this clears delayed acks for us. */ 4226 __tcp_transmit_skb(sk, buff, 0, (__force gfp_t)0, rcv_nxt); 4227 } 4228 EXPORT_SYMBOL_GPL(__tcp_send_ack); 4229 4230 void tcp_send_ack(struct sock *sk) 4231 { 4232 __tcp_send_ack(sk, tcp_sk(sk)->rcv_nxt); 4233 } 4234 4235 /* This routine sends a packet with an out of date sequence 4236 * number. It assumes the other end will try to ack it. 4237 * 4238 * Question: what should we make while urgent mode? 4239 * 4.4BSD forces sending single byte of data. We cannot send 4240 * out of window data, because we have SND.NXT==SND.MAX... 4241 * 4242 * Current solution: to send TWO zero-length segments in urgent mode: 4243 * one is with SEG.SEQ=SND.UNA to deliver urgent pointer, another is 4244 * out-of-date with SND.UNA-1 to probe window. 4245 */ 4246 static int tcp_xmit_probe_skb(struct sock *sk, int urgent, int mib) 4247 { 4248 struct tcp_sock *tp = tcp_sk(sk); 4249 struct sk_buff *skb; 4250 4251 /* We don't queue it, tcp_transmit_skb() sets ownership. */ 4252 skb = alloc_skb(MAX_TCP_HEADER, 4253 sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN)); 4254 if (!skb) 4255 return -1; 4256 4257 /* Reserve space for headers and set control bits. */ 4258 skb_reserve(skb, MAX_TCP_HEADER); 4259 /* Use a previous sequence. This should cause the other 4260 * end to send an ack. Don't queue or clone SKB, just 4261 * send it. 4262 */ 4263 tcp_init_nondata_skb(skb, tp->snd_una - !urgent, TCPHDR_ACK); 4264 NET_INC_STATS(sock_net(sk), mib); 4265 return tcp_transmit_skb(sk, skb, 0, (__force gfp_t)0); 4266 } 4267 4268 /* Called from setsockopt( ... TCP_REPAIR ) */ 4269 void tcp_send_window_probe(struct sock *sk) 4270 { 4271 if (sk->sk_state == TCP_ESTABLISHED) { 4272 tcp_sk(sk)->snd_wl1 = tcp_sk(sk)->rcv_nxt - 1; 4273 tcp_mstamp_refresh(tcp_sk(sk)); 4274 tcp_xmit_probe_skb(sk, 0, LINUX_MIB_TCPWINPROBE); 4275 } 4276 } 4277 4278 /* Initiate keepalive or window probe from timer. */ 4279 int tcp_write_wakeup(struct sock *sk, int mib) 4280 { 4281 struct tcp_sock *tp = tcp_sk(sk); 4282 struct sk_buff *skb; 4283 4284 if (sk->sk_state == TCP_CLOSE) 4285 return -1; 4286 4287 skb = tcp_send_head(sk); 4288 if (skb && before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp))) { 4289 int err; 4290 unsigned int mss = tcp_current_mss(sk); 4291 unsigned int seg_size = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq; 4292 4293 if (before(tp->pushed_seq, TCP_SKB_CB(skb)->end_seq)) 4294 tp->pushed_seq = TCP_SKB_CB(skb)->end_seq; 4295 4296 /* We are probing the opening of a window 4297 * but the window size is != 0 4298 * must have been a result SWS avoidance ( sender ) 4299 */ 4300 if (seg_size < TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq || 4301 skb->len > mss) { 4302 seg_size = min(seg_size, mss); 4303 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH; 4304 if (tcp_fragment(sk, TCP_FRAG_IN_WRITE_QUEUE, 4305 skb, seg_size, mss, GFP_ATOMIC)) 4306 return -1; 4307 } else if (!tcp_skb_pcount(skb)) 4308 tcp_set_skb_tso_segs(skb, mss); 4309 4310 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH; 4311 err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC); 4312 if (!err) 4313 tcp_event_new_data_sent(sk, skb); 4314 return err; 4315 } else { 4316 if (between(tp->snd_up, tp->snd_una + 1, tp->snd_una + 0xFFFF)) 4317 tcp_xmit_probe_skb(sk, 1, mib); 4318 return tcp_xmit_probe_skb(sk, 0, mib); 4319 } 4320 } 4321 4322 /* A window probe timeout has occurred. If window is not closed send 4323 * a partial packet else a zero probe. 4324 */ 4325 void tcp_send_probe0(struct sock *sk) 4326 { 4327 struct inet_connection_sock *icsk = inet_csk(sk); 4328 struct tcp_sock *tp = tcp_sk(sk); 4329 struct net *net = sock_net(sk); 4330 unsigned long timeout; 4331 int err; 4332 4333 err = tcp_write_wakeup(sk, LINUX_MIB_TCPWINPROBE); 4334 4335 if (tp->packets_out || tcp_write_queue_empty(sk)) { 4336 /* Cancel probe timer, if it is not required. */ 4337 icsk->icsk_probes_out = 0; 4338 icsk->icsk_backoff = 0; 4339 icsk->icsk_probes_tstamp = 0; 4340 return; 4341 } 4342 4343 icsk->icsk_probes_out++; 4344 if (err <= 0) { 4345 if (icsk->icsk_backoff < READ_ONCE(net->ipv4.sysctl_tcp_retries2)) 4346 icsk->icsk_backoff++; 4347 timeout = tcp_probe0_when(sk, TCP_RTO_MAX); 4348 } else { 4349 /* If packet was not sent due to local congestion, 4350 * Let senders fight for local resources conservatively. 4351 */ 4352 timeout = TCP_RESOURCE_PROBE_INTERVAL; 4353 } 4354 4355 timeout = tcp_clamp_probe0_to_user_timeout(sk, timeout); 4356 tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0, timeout, TCP_RTO_MAX); 4357 } 4358 4359 int tcp_rtx_synack(const struct sock *sk, struct request_sock *req) 4360 { 4361 const struct tcp_request_sock_ops *af_ops = tcp_rsk(req)->af_specific; 4362 struct flowi fl; 4363 int res; 4364 4365 /* Paired with WRITE_ONCE() in sock_setsockopt() */ 4366 if (READ_ONCE(sk->sk_txrehash) == SOCK_TXREHASH_ENABLED) 4367 WRITE_ONCE(tcp_rsk(req)->txhash, net_tx_rndhash()); 4368 res = af_ops->send_synack(sk, NULL, &fl, req, NULL, TCP_SYNACK_NORMAL, 4369 NULL); 4370 if (!res) { 4371 TCP_INC_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS); 4372 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS); 4373 if (unlikely(tcp_passive_fastopen(sk))) { 4374 /* sk has const attribute because listeners are lockless. 4375 * However in this case, we are dealing with a passive fastopen 4376 * socket thus we can change total_retrans value. 4377 */ 4378 tcp_sk_rw(sk)->total_retrans++; 4379 } 4380 trace_tcp_retransmit_synack(sk, req); 4381 } 4382 return res; 4383 } 4384 EXPORT_SYMBOL(tcp_rtx_synack); 4385