1 /* 2 * INET An implementation of the TCP/IP protocol suite for the LINUX 3 * operating system. INET is implemented using the BSD Socket 4 * interface as the means of communication with the user level. 5 * 6 * Implementation of the Transmission Control Protocol(TCP). 7 * 8 * Authors: Ross Biro 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 10 * Mark Evans, <evansmp@uhura.aston.ac.uk> 11 * Corey Minyard <wf-rch!minyard@relay.EU.net> 12 * Florian La Roche, <flla@stud.uni-sb.de> 13 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu> 14 * Linus Torvalds, <torvalds@cs.helsinki.fi> 15 * Alan Cox, <gw4pts@gw4pts.ampr.org> 16 * Matthew Dillon, <dillon@apollo.west.oic.com> 17 * Arnt Gulbrandsen, <agulbra@nvg.unit.no> 18 * Jorge Cwik, <jorge@laser.satlink.net> 19 */ 20 21 /* 22 * Changes: Pedro Roque : Retransmit queue handled by TCP. 23 * : Fragmentation on mtu decrease 24 * : Segment collapse on retransmit 25 * : AF independence 26 * 27 * Linus Torvalds : send_delayed_ack 28 * David S. Miller : Charge memory using the right skb 29 * during syn/ack processing. 30 * David S. Miller : Output engine completely rewritten. 31 * Andrea Arcangeli: SYNACK carry ts_recent in tsecr. 32 * Cacophonix Gaul : draft-minshall-nagle-01 33 * J Hadi Salim : ECN support 34 * 35 */ 36 37 #define pr_fmt(fmt) "TCP: " fmt 38 39 #include <net/tcp.h> 40 41 #include <linux/compiler.h> 42 #include <linux/gfp.h> 43 #include <linux/module.h> 44 45 /* People can turn this off for buggy TCP's found in printers etc. */ 46 int sysctl_tcp_retrans_collapse __read_mostly = 1; 47 48 /* People can turn this on to work with those rare, broken TCPs that 49 * interpret the window field as a signed quantity. 50 */ 51 int sysctl_tcp_workaround_signed_windows __read_mostly = 0; 52 53 /* Default TSQ limit of two TSO segments */ 54 int sysctl_tcp_limit_output_bytes __read_mostly = 131072; 55 56 /* This limits the percentage of the congestion window which we 57 * will allow a single TSO frame to consume. Building TSO frames 58 * which are too large can cause TCP streams to be bursty. 59 */ 60 int sysctl_tcp_tso_win_divisor __read_mostly = 3; 61 62 int sysctl_tcp_mtu_probing __read_mostly = 0; 63 int sysctl_tcp_base_mss __read_mostly = TCP_BASE_MSS; 64 65 /* By default, RFC2861 behavior. */ 66 int sysctl_tcp_slow_start_after_idle __read_mostly = 1; 67 68 unsigned int sysctl_tcp_notsent_lowat __read_mostly = UINT_MAX; 69 EXPORT_SYMBOL(sysctl_tcp_notsent_lowat); 70 71 static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, 72 int push_one, gfp_t gfp); 73 74 /* Account for new data that has been sent to the network. */ 75 static void tcp_event_new_data_sent(struct sock *sk, const struct sk_buff *skb) 76 { 77 struct inet_connection_sock *icsk = inet_csk(sk); 78 struct tcp_sock *tp = tcp_sk(sk); 79 unsigned int prior_packets = tp->packets_out; 80 81 tcp_advance_send_head(sk, skb); 82 tp->snd_nxt = TCP_SKB_CB(skb)->end_seq; 83 84 tp->packets_out += tcp_skb_pcount(skb); 85 if (!prior_packets || icsk->icsk_pending == ICSK_TIME_EARLY_RETRANS || 86 icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) { 87 tcp_rearm_rto(sk); 88 } 89 90 NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT, 91 tcp_skb_pcount(skb)); 92 } 93 94 /* SND.NXT, if window was not shrunk. 95 * If window has been shrunk, what should we make? It is not clear at all. 96 * Using SND.UNA we will fail to open window, SND.NXT is out of window. :-( 97 * Anything in between SND.UNA...SND.UNA+SND.WND also can be already 98 * invalid. OK, let's make this for now: 99 */ 100 static inline __u32 tcp_acceptable_seq(const struct sock *sk) 101 { 102 const struct tcp_sock *tp = tcp_sk(sk); 103 104 if (!before(tcp_wnd_end(tp), tp->snd_nxt)) 105 return tp->snd_nxt; 106 else 107 return tcp_wnd_end(tp); 108 } 109 110 /* Calculate mss to advertise in SYN segment. 111 * RFC1122, RFC1063, draft-ietf-tcpimpl-pmtud-01 state that: 112 * 113 * 1. It is independent of path mtu. 114 * 2. Ideally, it is maximal possible segment size i.e. 65535-40. 115 * 3. For IPv4 it is reasonable to calculate it from maximal MTU of 116 * attached devices, because some buggy hosts are confused by 117 * large MSS. 118 * 4. We do not make 3, we advertise MSS, calculated from first 119 * hop device mtu, but allow to raise it to ip_rt_min_advmss. 120 * This may be overridden via information stored in routing table. 121 * 5. Value 65535 for MSS is valid in IPv6 and means "as large as possible, 122 * probably even Jumbo". 123 */ 124 static __u16 tcp_advertise_mss(struct sock *sk) 125 { 126 struct tcp_sock *tp = tcp_sk(sk); 127 const struct dst_entry *dst = __sk_dst_get(sk); 128 int mss = tp->advmss; 129 130 if (dst) { 131 unsigned int metric = dst_metric_advmss(dst); 132 133 if (metric < mss) { 134 mss = metric; 135 tp->advmss = mss; 136 } 137 } 138 139 return (__u16)mss; 140 } 141 142 /* RFC2861. Reset CWND after idle period longer RTO to "restart window". 143 * This is the first part of cwnd validation mechanism. */ 144 static void tcp_cwnd_restart(struct sock *sk, const struct dst_entry *dst) 145 { 146 struct tcp_sock *tp = tcp_sk(sk); 147 s32 delta = tcp_time_stamp - tp->lsndtime; 148 u32 restart_cwnd = tcp_init_cwnd(tp, dst); 149 u32 cwnd = tp->snd_cwnd; 150 151 tcp_ca_event(sk, CA_EVENT_CWND_RESTART); 152 153 tp->snd_ssthresh = tcp_current_ssthresh(sk); 154 restart_cwnd = min(restart_cwnd, cwnd); 155 156 while ((delta -= inet_csk(sk)->icsk_rto) > 0 && cwnd > restart_cwnd) 157 cwnd >>= 1; 158 tp->snd_cwnd = max(cwnd, restart_cwnd); 159 tp->snd_cwnd_stamp = tcp_time_stamp; 160 tp->snd_cwnd_used = 0; 161 } 162 163 /* Congestion state accounting after a packet has been sent. */ 164 static void tcp_event_data_sent(struct tcp_sock *tp, 165 struct sock *sk) 166 { 167 struct inet_connection_sock *icsk = inet_csk(sk); 168 const u32 now = tcp_time_stamp; 169 const struct dst_entry *dst = __sk_dst_get(sk); 170 171 if (sysctl_tcp_slow_start_after_idle && 172 (!tp->packets_out && (s32)(now - tp->lsndtime) > icsk->icsk_rto)) 173 tcp_cwnd_restart(sk, __sk_dst_get(sk)); 174 175 tp->lsndtime = now; 176 177 /* If it is a reply for ato after last received 178 * packet, enter pingpong mode. 179 */ 180 if ((u32)(now - icsk->icsk_ack.lrcvtime) < icsk->icsk_ack.ato && 181 (!dst || !dst_metric(dst, RTAX_QUICKACK))) 182 icsk->icsk_ack.pingpong = 1; 183 } 184 185 /* Account for an ACK we sent. */ 186 static inline void tcp_event_ack_sent(struct sock *sk, unsigned int pkts) 187 { 188 tcp_dec_quickack_mode(sk, pkts); 189 inet_csk_clear_xmit_timer(sk, ICSK_TIME_DACK); 190 } 191 192 193 u32 tcp_default_init_rwnd(u32 mss) 194 { 195 /* Initial receive window should be twice of TCP_INIT_CWND to 196 * enable proper sending of new unsent data during fast recovery 197 * (RFC 3517, Section 4, NextSeg() rule (2)). Further place a 198 * limit when mss is larger than 1460. 199 */ 200 u32 init_rwnd = TCP_INIT_CWND * 2; 201 202 if (mss > 1460) 203 init_rwnd = max((1460 * init_rwnd) / mss, 2U); 204 return init_rwnd; 205 } 206 207 /* Determine a window scaling and initial window to offer. 208 * Based on the assumption that the given amount of space 209 * will be offered. Store the results in the tp structure. 210 * NOTE: for smooth operation initial space offering should 211 * be a multiple of mss if possible. We assume here that mss >= 1. 212 * This MUST be enforced by all callers. 213 */ 214 void tcp_select_initial_window(int __space, __u32 mss, 215 __u32 *rcv_wnd, __u32 *window_clamp, 216 int wscale_ok, __u8 *rcv_wscale, 217 __u32 init_rcv_wnd) 218 { 219 unsigned int space = (__space < 0 ? 0 : __space); 220 221 /* If no clamp set the clamp to the max possible scaled window */ 222 if (*window_clamp == 0) 223 (*window_clamp) = (65535 << 14); 224 space = min(*window_clamp, space); 225 226 /* Quantize space offering to a multiple of mss if possible. */ 227 if (space > mss) 228 space = (space / mss) * mss; 229 230 /* NOTE: offering an initial window larger than 32767 231 * will break some buggy TCP stacks. If the admin tells us 232 * it is likely we could be speaking with such a buggy stack 233 * we will truncate our initial window offering to 32K-1 234 * unless the remote has sent us a window scaling option, 235 * which we interpret as a sign the remote TCP is not 236 * misinterpreting the window field as a signed quantity. 237 */ 238 if (sysctl_tcp_workaround_signed_windows) 239 (*rcv_wnd) = min(space, MAX_TCP_WINDOW); 240 else 241 (*rcv_wnd) = space; 242 243 (*rcv_wscale) = 0; 244 if (wscale_ok) { 245 /* Set window scaling on max possible window 246 * See RFC1323 for an explanation of the limit to 14 247 */ 248 space = max_t(u32, sysctl_tcp_rmem[2], sysctl_rmem_max); 249 space = min_t(u32, space, *window_clamp); 250 while (space > 65535 && (*rcv_wscale) < 14) { 251 space >>= 1; 252 (*rcv_wscale)++; 253 } 254 } 255 256 if (mss > (1 << *rcv_wscale)) { 257 if (!init_rcv_wnd) /* Use default unless specified otherwise */ 258 init_rcv_wnd = tcp_default_init_rwnd(mss); 259 *rcv_wnd = min(*rcv_wnd, init_rcv_wnd * mss); 260 } 261 262 /* Set the clamp no higher than max representable value */ 263 (*window_clamp) = min(65535U << (*rcv_wscale), *window_clamp); 264 } 265 EXPORT_SYMBOL(tcp_select_initial_window); 266 267 /* Chose a new window to advertise, update state in tcp_sock for the 268 * socket, and return result with RFC1323 scaling applied. The return 269 * value can be stuffed directly into th->window for an outgoing 270 * frame. 271 */ 272 static u16 tcp_select_window(struct sock *sk) 273 { 274 struct tcp_sock *tp = tcp_sk(sk); 275 u32 old_win = tp->rcv_wnd; 276 u32 cur_win = tcp_receive_window(tp); 277 u32 new_win = __tcp_select_window(sk); 278 279 /* Never shrink the offered window */ 280 if (new_win < cur_win) { 281 /* Danger Will Robinson! 282 * Don't update rcv_wup/rcv_wnd here or else 283 * we will not be able to advertise a zero 284 * window in time. --DaveM 285 * 286 * Relax Will Robinson. 287 */ 288 if (new_win == 0) 289 NET_INC_STATS(sock_net(sk), 290 LINUX_MIB_TCPWANTZEROWINDOWADV); 291 new_win = ALIGN(cur_win, 1 << tp->rx_opt.rcv_wscale); 292 } 293 tp->rcv_wnd = new_win; 294 tp->rcv_wup = tp->rcv_nxt; 295 296 /* Make sure we do not exceed the maximum possible 297 * scaled window. 298 */ 299 if (!tp->rx_opt.rcv_wscale && sysctl_tcp_workaround_signed_windows) 300 new_win = min(new_win, MAX_TCP_WINDOW); 301 else 302 new_win = min(new_win, (65535U << tp->rx_opt.rcv_wscale)); 303 304 /* RFC1323 scaling applied */ 305 new_win >>= tp->rx_opt.rcv_wscale; 306 307 /* If we advertise zero window, disable fast path. */ 308 if (new_win == 0) { 309 tp->pred_flags = 0; 310 if (old_win) 311 NET_INC_STATS(sock_net(sk), 312 LINUX_MIB_TCPTOZEROWINDOWADV); 313 } else if (old_win == 0) { 314 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPFROMZEROWINDOWADV); 315 } 316 317 return new_win; 318 } 319 320 /* Packet ECN state for a SYN-ACK */ 321 static void tcp_ecn_send_synack(struct sock *sk, struct sk_buff *skb) 322 { 323 const struct tcp_sock *tp = tcp_sk(sk); 324 325 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_CWR; 326 if (!(tp->ecn_flags & TCP_ECN_OK)) 327 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ECE; 328 else if (tcp_ca_needs_ecn(sk)) 329 INET_ECN_xmit(sk); 330 } 331 332 /* Packet ECN state for a SYN. */ 333 static void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb) 334 { 335 struct tcp_sock *tp = tcp_sk(sk); 336 bool use_ecn = sock_net(sk)->ipv4.sysctl_tcp_ecn == 1 || 337 tcp_ca_needs_ecn(sk); 338 339 if (!use_ecn) { 340 const struct dst_entry *dst = __sk_dst_get(sk); 341 342 if (dst && dst_feature(dst, RTAX_FEATURE_ECN)) 343 use_ecn = true; 344 } 345 346 tp->ecn_flags = 0; 347 348 if (use_ecn) { 349 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ECE | TCPHDR_CWR; 350 tp->ecn_flags = TCP_ECN_OK; 351 if (tcp_ca_needs_ecn(sk)) 352 INET_ECN_xmit(sk); 353 } 354 } 355 356 static void 357 tcp_ecn_make_synack(const struct request_sock *req, struct tcphdr *th, 358 struct sock *sk) 359 { 360 if (inet_rsk(req)->ecn_ok) { 361 th->ece = 1; 362 if (tcp_ca_needs_ecn(sk)) 363 INET_ECN_xmit(sk); 364 } 365 } 366 367 /* Set up ECN state for a packet on a ESTABLISHED socket that is about to 368 * be sent. 369 */ 370 static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb, 371 int tcp_header_len) 372 { 373 struct tcp_sock *tp = tcp_sk(sk); 374 375 if (tp->ecn_flags & TCP_ECN_OK) { 376 /* Not-retransmitted data segment: set ECT and inject CWR. */ 377 if (skb->len != tcp_header_len && 378 !before(TCP_SKB_CB(skb)->seq, tp->snd_nxt)) { 379 INET_ECN_xmit(sk); 380 if (tp->ecn_flags & TCP_ECN_QUEUE_CWR) { 381 tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR; 382 tcp_hdr(skb)->cwr = 1; 383 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN; 384 } 385 } else if (!tcp_ca_needs_ecn(sk)) { 386 /* ACK or retransmitted segment: clear ECT|CE */ 387 INET_ECN_dontxmit(sk); 388 } 389 if (tp->ecn_flags & TCP_ECN_DEMAND_CWR) 390 tcp_hdr(skb)->ece = 1; 391 } 392 } 393 394 /* Constructs common control bits of non-data skb. If SYN/FIN is present, 395 * auto increment end seqno. 396 */ 397 static void tcp_init_nondata_skb(struct sk_buff *skb, u32 seq, u8 flags) 398 { 399 struct skb_shared_info *shinfo = skb_shinfo(skb); 400 401 skb->ip_summed = CHECKSUM_PARTIAL; 402 skb->csum = 0; 403 404 TCP_SKB_CB(skb)->tcp_flags = flags; 405 TCP_SKB_CB(skb)->sacked = 0; 406 407 tcp_skb_pcount_set(skb, 1); 408 shinfo->gso_size = 0; 409 shinfo->gso_type = 0; 410 411 TCP_SKB_CB(skb)->seq = seq; 412 if (flags & (TCPHDR_SYN | TCPHDR_FIN)) 413 seq++; 414 TCP_SKB_CB(skb)->end_seq = seq; 415 } 416 417 static inline bool tcp_urg_mode(const struct tcp_sock *tp) 418 { 419 return tp->snd_una != tp->snd_up; 420 } 421 422 #define OPTION_SACK_ADVERTISE (1 << 0) 423 #define OPTION_TS (1 << 1) 424 #define OPTION_MD5 (1 << 2) 425 #define OPTION_WSCALE (1 << 3) 426 #define OPTION_FAST_OPEN_COOKIE (1 << 8) 427 428 struct tcp_out_options { 429 u16 options; /* bit field of OPTION_* */ 430 u16 mss; /* 0 to disable */ 431 u8 ws; /* window scale, 0 to disable */ 432 u8 num_sack_blocks; /* number of SACK blocks to include */ 433 u8 hash_size; /* bytes in hash_location */ 434 __u8 *hash_location; /* temporary pointer, overloaded */ 435 __u32 tsval, tsecr; /* need to include OPTION_TS */ 436 struct tcp_fastopen_cookie *fastopen_cookie; /* Fast open cookie */ 437 }; 438 439 /* Write previously computed TCP options to the packet. 440 * 441 * Beware: Something in the Internet is very sensitive to the ordering of 442 * TCP options, we learned this through the hard way, so be careful here. 443 * Luckily we can at least blame others for their non-compliance but from 444 * inter-operability perspective it seems that we're somewhat stuck with 445 * the ordering which we have been using if we want to keep working with 446 * those broken things (not that it currently hurts anybody as there isn't 447 * particular reason why the ordering would need to be changed). 448 * 449 * At least SACK_PERM as the first option is known to lead to a disaster 450 * (but it may well be that other scenarios fail similarly). 451 */ 452 static void tcp_options_write(__be32 *ptr, struct tcp_sock *tp, 453 struct tcp_out_options *opts) 454 { 455 u16 options = opts->options; /* mungable copy */ 456 457 if (unlikely(OPTION_MD5 & options)) { 458 *ptr++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | 459 (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG); 460 /* overload cookie hash location */ 461 opts->hash_location = (__u8 *)ptr; 462 ptr += 4; 463 } 464 465 if (unlikely(opts->mss)) { 466 *ptr++ = htonl((TCPOPT_MSS << 24) | 467 (TCPOLEN_MSS << 16) | 468 opts->mss); 469 } 470 471 if (likely(OPTION_TS & options)) { 472 if (unlikely(OPTION_SACK_ADVERTISE & options)) { 473 *ptr++ = htonl((TCPOPT_SACK_PERM << 24) | 474 (TCPOLEN_SACK_PERM << 16) | 475 (TCPOPT_TIMESTAMP << 8) | 476 TCPOLEN_TIMESTAMP); 477 options &= ~OPTION_SACK_ADVERTISE; 478 } else { 479 *ptr++ = htonl((TCPOPT_NOP << 24) | 480 (TCPOPT_NOP << 16) | 481 (TCPOPT_TIMESTAMP << 8) | 482 TCPOLEN_TIMESTAMP); 483 } 484 *ptr++ = htonl(opts->tsval); 485 *ptr++ = htonl(opts->tsecr); 486 } 487 488 if (unlikely(OPTION_SACK_ADVERTISE & options)) { 489 *ptr++ = htonl((TCPOPT_NOP << 24) | 490 (TCPOPT_NOP << 16) | 491 (TCPOPT_SACK_PERM << 8) | 492 TCPOLEN_SACK_PERM); 493 } 494 495 if (unlikely(OPTION_WSCALE & options)) { 496 *ptr++ = htonl((TCPOPT_NOP << 24) | 497 (TCPOPT_WINDOW << 16) | 498 (TCPOLEN_WINDOW << 8) | 499 opts->ws); 500 } 501 502 if (unlikely(opts->num_sack_blocks)) { 503 struct tcp_sack_block *sp = tp->rx_opt.dsack ? 504 tp->duplicate_sack : tp->selective_acks; 505 int this_sack; 506 507 *ptr++ = htonl((TCPOPT_NOP << 24) | 508 (TCPOPT_NOP << 16) | 509 (TCPOPT_SACK << 8) | 510 (TCPOLEN_SACK_BASE + (opts->num_sack_blocks * 511 TCPOLEN_SACK_PERBLOCK))); 512 513 for (this_sack = 0; this_sack < opts->num_sack_blocks; 514 ++this_sack) { 515 *ptr++ = htonl(sp[this_sack].start_seq); 516 *ptr++ = htonl(sp[this_sack].end_seq); 517 } 518 519 tp->rx_opt.dsack = 0; 520 } 521 522 if (unlikely(OPTION_FAST_OPEN_COOKIE & options)) { 523 struct tcp_fastopen_cookie *foc = opts->fastopen_cookie; 524 525 *ptr++ = htonl((TCPOPT_EXP << 24) | 526 ((TCPOLEN_EXP_FASTOPEN_BASE + foc->len) << 16) | 527 TCPOPT_FASTOPEN_MAGIC); 528 529 memcpy(ptr, foc->val, foc->len); 530 if ((foc->len & 3) == 2) { 531 u8 *align = ((u8 *)ptr) + foc->len; 532 align[0] = align[1] = TCPOPT_NOP; 533 } 534 ptr += (foc->len + 3) >> 2; 535 } 536 } 537 538 /* Compute TCP options for SYN packets. This is not the final 539 * network wire format yet. 540 */ 541 static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb, 542 struct tcp_out_options *opts, 543 struct tcp_md5sig_key **md5) 544 { 545 struct tcp_sock *tp = tcp_sk(sk); 546 unsigned int remaining = MAX_TCP_OPTION_SPACE; 547 struct tcp_fastopen_request *fastopen = tp->fastopen_req; 548 549 #ifdef CONFIG_TCP_MD5SIG 550 *md5 = tp->af_specific->md5_lookup(sk, sk); 551 if (*md5) { 552 opts->options |= OPTION_MD5; 553 remaining -= TCPOLEN_MD5SIG_ALIGNED; 554 } 555 #else 556 *md5 = NULL; 557 #endif 558 559 /* We always get an MSS option. The option bytes which will be seen in 560 * normal data packets should timestamps be used, must be in the MSS 561 * advertised. But we subtract them from tp->mss_cache so that 562 * calculations in tcp_sendmsg are simpler etc. So account for this 563 * fact here if necessary. If we don't do this correctly, as a 564 * receiver we won't recognize data packets as being full sized when we 565 * should, and thus we won't abide by the delayed ACK rules correctly. 566 * SACKs don't matter, we never delay an ACK when we have any of those 567 * going out. */ 568 opts->mss = tcp_advertise_mss(sk); 569 remaining -= TCPOLEN_MSS_ALIGNED; 570 571 if (likely(sysctl_tcp_timestamps && *md5 == NULL)) { 572 opts->options |= OPTION_TS; 573 opts->tsval = tcp_skb_timestamp(skb) + tp->tsoffset; 574 opts->tsecr = tp->rx_opt.ts_recent; 575 remaining -= TCPOLEN_TSTAMP_ALIGNED; 576 } 577 if (likely(sysctl_tcp_window_scaling)) { 578 opts->ws = tp->rx_opt.rcv_wscale; 579 opts->options |= OPTION_WSCALE; 580 remaining -= TCPOLEN_WSCALE_ALIGNED; 581 } 582 if (likely(sysctl_tcp_sack)) { 583 opts->options |= OPTION_SACK_ADVERTISE; 584 if (unlikely(!(OPTION_TS & opts->options))) 585 remaining -= TCPOLEN_SACKPERM_ALIGNED; 586 } 587 588 if (fastopen && fastopen->cookie.len >= 0) { 589 u32 need = TCPOLEN_EXP_FASTOPEN_BASE + fastopen->cookie.len; 590 need = (need + 3) & ~3U; /* Align to 32 bits */ 591 if (remaining >= need) { 592 opts->options |= OPTION_FAST_OPEN_COOKIE; 593 opts->fastopen_cookie = &fastopen->cookie; 594 remaining -= need; 595 tp->syn_fastopen = 1; 596 } 597 } 598 599 return MAX_TCP_OPTION_SPACE - remaining; 600 } 601 602 /* Set up TCP options for SYN-ACKs. */ 603 static unsigned int tcp_synack_options(struct sock *sk, 604 struct request_sock *req, 605 unsigned int mss, struct sk_buff *skb, 606 struct tcp_out_options *opts, 607 struct tcp_md5sig_key **md5, 608 struct tcp_fastopen_cookie *foc) 609 { 610 struct inet_request_sock *ireq = inet_rsk(req); 611 unsigned int remaining = MAX_TCP_OPTION_SPACE; 612 613 #ifdef CONFIG_TCP_MD5SIG 614 *md5 = tcp_rsk(req)->af_specific->md5_lookup(sk, req); 615 if (*md5) { 616 opts->options |= OPTION_MD5; 617 remaining -= TCPOLEN_MD5SIG_ALIGNED; 618 619 /* We can't fit any SACK blocks in a packet with MD5 + TS 620 * options. There was discussion about disabling SACK 621 * rather than TS in order to fit in better with old, 622 * buggy kernels, but that was deemed to be unnecessary. 623 */ 624 ireq->tstamp_ok &= !ireq->sack_ok; 625 } 626 #else 627 *md5 = NULL; 628 #endif 629 630 /* We always send an MSS option. */ 631 opts->mss = mss; 632 remaining -= TCPOLEN_MSS_ALIGNED; 633 634 if (likely(ireq->wscale_ok)) { 635 opts->ws = ireq->rcv_wscale; 636 opts->options |= OPTION_WSCALE; 637 remaining -= TCPOLEN_WSCALE_ALIGNED; 638 } 639 if (likely(ireq->tstamp_ok)) { 640 opts->options |= OPTION_TS; 641 opts->tsval = tcp_skb_timestamp(skb); 642 opts->tsecr = req->ts_recent; 643 remaining -= TCPOLEN_TSTAMP_ALIGNED; 644 } 645 if (likely(ireq->sack_ok)) { 646 opts->options |= OPTION_SACK_ADVERTISE; 647 if (unlikely(!ireq->tstamp_ok)) 648 remaining -= TCPOLEN_SACKPERM_ALIGNED; 649 } 650 if (foc != NULL && foc->len >= 0) { 651 u32 need = TCPOLEN_EXP_FASTOPEN_BASE + foc->len; 652 need = (need + 3) & ~3U; /* Align to 32 bits */ 653 if (remaining >= need) { 654 opts->options |= OPTION_FAST_OPEN_COOKIE; 655 opts->fastopen_cookie = foc; 656 remaining -= need; 657 } 658 } 659 660 return MAX_TCP_OPTION_SPACE - remaining; 661 } 662 663 /* Compute TCP options for ESTABLISHED sockets. This is not the 664 * final wire format yet. 665 */ 666 static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb, 667 struct tcp_out_options *opts, 668 struct tcp_md5sig_key **md5) 669 { 670 struct tcp_sock *tp = tcp_sk(sk); 671 unsigned int size = 0; 672 unsigned int eff_sacks; 673 674 opts->options = 0; 675 676 #ifdef CONFIG_TCP_MD5SIG 677 *md5 = tp->af_specific->md5_lookup(sk, sk); 678 if (unlikely(*md5)) { 679 opts->options |= OPTION_MD5; 680 size += TCPOLEN_MD5SIG_ALIGNED; 681 } 682 #else 683 *md5 = NULL; 684 #endif 685 686 if (likely(tp->rx_opt.tstamp_ok)) { 687 opts->options |= OPTION_TS; 688 opts->tsval = skb ? tcp_skb_timestamp(skb) + tp->tsoffset : 0; 689 opts->tsecr = tp->rx_opt.ts_recent; 690 size += TCPOLEN_TSTAMP_ALIGNED; 691 } 692 693 eff_sacks = tp->rx_opt.num_sacks + tp->rx_opt.dsack; 694 if (unlikely(eff_sacks)) { 695 const unsigned int remaining = MAX_TCP_OPTION_SPACE - size; 696 opts->num_sack_blocks = 697 min_t(unsigned int, eff_sacks, 698 (remaining - TCPOLEN_SACK_BASE_ALIGNED) / 699 TCPOLEN_SACK_PERBLOCK); 700 size += TCPOLEN_SACK_BASE_ALIGNED + 701 opts->num_sack_blocks * TCPOLEN_SACK_PERBLOCK; 702 } 703 704 return size; 705 } 706 707 708 /* TCP SMALL QUEUES (TSQ) 709 * 710 * TSQ goal is to keep small amount of skbs per tcp flow in tx queues (qdisc+dev) 711 * to reduce RTT and bufferbloat. 712 * We do this using a special skb destructor (tcp_wfree). 713 * 714 * Its important tcp_wfree() can be replaced by sock_wfree() in the event skb 715 * needs to be reallocated in a driver. 716 * The invariant being skb->truesize subtracted from sk->sk_wmem_alloc 717 * 718 * Since transmit from skb destructor is forbidden, we use a tasklet 719 * to process all sockets that eventually need to send more skbs. 720 * We use one tasklet per cpu, with its own queue of sockets. 721 */ 722 struct tsq_tasklet { 723 struct tasklet_struct tasklet; 724 struct list_head head; /* queue of tcp sockets */ 725 }; 726 static DEFINE_PER_CPU(struct tsq_tasklet, tsq_tasklet); 727 728 static void tcp_tsq_handler(struct sock *sk) 729 { 730 if ((1 << sk->sk_state) & 731 (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_CLOSING | 732 TCPF_CLOSE_WAIT | TCPF_LAST_ACK)) 733 tcp_write_xmit(sk, tcp_current_mss(sk), tcp_sk(sk)->nonagle, 734 0, GFP_ATOMIC); 735 } 736 /* 737 * One tasklet per cpu tries to send more skbs. 738 * We run in tasklet context but need to disable irqs when 739 * transferring tsq->head because tcp_wfree() might 740 * interrupt us (non NAPI drivers) 741 */ 742 static void tcp_tasklet_func(unsigned long data) 743 { 744 struct tsq_tasklet *tsq = (struct tsq_tasklet *)data; 745 LIST_HEAD(list); 746 unsigned long flags; 747 struct list_head *q, *n; 748 struct tcp_sock *tp; 749 struct sock *sk; 750 751 local_irq_save(flags); 752 list_splice_init(&tsq->head, &list); 753 local_irq_restore(flags); 754 755 list_for_each_safe(q, n, &list) { 756 tp = list_entry(q, struct tcp_sock, tsq_node); 757 list_del(&tp->tsq_node); 758 759 sk = (struct sock *)tp; 760 bh_lock_sock(sk); 761 762 if (!sock_owned_by_user(sk)) { 763 tcp_tsq_handler(sk); 764 } else { 765 /* defer the work to tcp_release_cb() */ 766 set_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags); 767 } 768 bh_unlock_sock(sk); 769 770 clear_bit(TSQ_QUEUED, &tp->tsq_flags); 771 sk_free(sk); 772 } 773 } 774 775 #define TCP_DEFERRED_ALL ((1UL << TCP_TSQ_DEFERRED) | \ 776 (1UL << TCP_WRITE_TIMER_DEFERRED) | \ 777 (1UL << TCP_DELACK_TIMER_DEFERRED) | \ 778 (1UL << TCP_MTU_REDUCED_DEFERRED)) 779 /** 780 * tcp_release_cb - tcp release_sock() callback 781 * @sk: socket 782 * 783 * called from release_sock() to perform protocol dependent 784 * actions before socket release. 785 */ 786 void tcp_release_cb(struct sock *sk) 787 { 788 struct tcp_sock *tp = tcp_sk(sk); 789 unsigned long flags, nflags; 790 791 /* perform an atomic operation only if at least one flag is set */ 792 do { 793 flags = tp->tsq_flags; 794 if (!(flags & TCP_DEFERRED_ALL)) 795 return; 796 nflags = flags & ~TCP_DEFERRED_ALL; 797 } while (cmpxchg(&tp->tsq_flags, flags, nflags) != flags); 798 799 if (flags & (1UL << TCP_TSQ_DEFERRED)) 800 tcp_tsq_handler(sk); 801 802 /* Here begins the tricky part : 803 * We are called from release_sock() with : 804 * 1) BH disabled 805 * 2) sk_lock.slock spinlock held 806 * 3) socket owned by us (sk->sk_lock.owned == 1) 807 * 808 * But following code is meant to be called from BH handlers, 809 * so we should keep BH disabled, but early release socket ownership 810 */ 811 sock_release_ownership(sk); 812 813 if (flags & (1UL << TCP_WRITE_TIMER_DEFERRED)) { 814 tcp_write_timer_handler(sk); 815 __sock_put(sk); 816 } 817 if (flags & (1UL << TCP_DELACK_TIMER_DEFERRED)) { 818 tcp_delack_timer_handler(sk); 819 __sock_put(sk); 820 } 821 if (flags & (1UL << TCP_MTU_REDUCED_DEFERRED)) { 822 inet_csk(sk)->icsk_af_ops->mtu_reduced(sk); 823 __sock_put(sk); 824 } 825 } 826 EXPORT_SYMBOL(tcp_release_cb); 827 828 void __init tcp_tasklet_init(void) 829 { 830 int i; 831 832 for_each_possible_cpu(i) { 833 struct tsq_tasklet *tsq = &per_cpu(tsq_tasklet, i); 834 835 INIT_LIST_HEAD(&tsq->head); 836 tasklet_init(&tsq->tasklet, 837 tcp_tasklet_func, 838 (unsigned long)tsq); 839 } 840 } 841 842 /* 843 * Write buffer destructor automatically called from kfree_skb. 844 * We can't xmit new skbs from this context, as we might already 845 * hold qdisc lock. 846 */ 847 void tcp_wfree(struct sk_buff *skb) 848 { 849 struct sock *sk = skb->sk; 850 struct tcp_sock *tp = tcp_sk(sk); 851 int wmem; 852 853 /* Keep one reference on sk_wmem_alloc. 854 * Will be released by sk_free() from here or tcp_tasklet_func() 855 */ 856 wmem = atomic_sub_return(skb->truesize - 1, &sk->sk_wmem_alloc); 857 858 /* If this softirq is serviced by ksoftirqd, we are likely under stress. 859 * Wait until our queues (qdisc + devices) are drained. 860 * This gives : 861 * - less callbacks to tcp_write_xmit(), reducing stress (batches) 862 * - chance for incoming ACK (processed by another cpu maybe) 863 * to migrate this flow (skb->ooo_okay will be eventually set) 864 */ 865 if (wmem >= SKB_TRUESIZE(1) && this_cpu_ksoftirqd() == current) 866 goto out; 867 868 if (test_and_clear_bit(TSQ_THROTTLED, &tp->tsq_flags) && 869 !test_and_set_bit(TSQ_QUEUED, &tp->tsq_flags)) { 870 unsigned long flags; 871 struct tsq_tasklet *tsq; 872 873 /* queue this socket to tasklet queue */ 874 local_irq_save(flags); 875 tsq = this_cpu_ptr(&tsq_tasklet); 876 list_add(&tp->tsq_node, &tsq->head); 877 tasklet_schedule(&tsq->tasklet); 878 local_irq_restore(flags); 879 return; 880 } 881 out: 882 sk_free(sk); 883 } 884 885 /* This routine actually transmits TCP packets queued in by 886 * tcp_do_sendmsg(). This is used by both the initial 887 * transmission and possible later retransmissions. 888 * All SKB's seen here are completely headerless. It is our 889 * job to build the TCP header, and pass the packet down to 890 * IP so it can do the same plus pass the packet off to the 891 * device. 892 * 893 * We are working here with either a clone of the original 894 * SKB, or a fresh unique copy made by the retransmit engine. 895 */ 896 static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it, 897 gfp_t gfp_mask) 898 { 899 const struct inet_connection_sock *icsk = inet_csk(sk); 900 struct inet_sock *inet; 901 struct tcp_sock *tp; 902 struct tcp_skb_cb *tcb; 903 struct tcp_out_options opts; 904 unsigned int tcp_options_size, tcp_header_size; 905 struct tcp_md5sig_key *md5; 906 struct tcphdr *th; 907 int err; 908 909 BUG_ON(!skb || !tcp_skb_pcount(skb)); 910 911 if (clone_it) { 912 skb_mstamp_get(&skb->skb_mstamp); 913 914 if (unlikely(skb_cloned(skb))) 915 skb = pskb_copy(skb, gfp_mask); 916 else 917 skb = skb_clone(skb, gfp_mask); 918 if (unlikely(!skb)) 919 return -ENOBUFS; 920 } 921 922 inet = inet_sk(sk); 923 tp = tcp_sk(sk); 924 tcb = TCP_SKB_CB(skb); 925 memset(&opts, 0, sizeof(opts)); 926 927 if (unlikely(tcb->tcp_flags & TCPHDR_SYN)) 928 tcp_options_size = tcp_syn_options(sk, skb, &opts, &md5); 929 else 930 tcp_options_size = tcp_established_options(sk, skb, &opts, 931 &md5); 932 tcp_header_size = tcp_options_size + sizeof(struct tcphdr); 933 934 if (tcp_packets_in_flight(tp) == 0) 935 tcp_ca_event(sk, CA_EVENT_TX_START); 936 937 /* if no packet is in qdisc/device queue, then allow XPS to select 938 * another queue. We can be called from tcp_tsq_handler() 939 * which holds one reference to sk_wmem_alloc. 940 * 941 * TODO: Ideally, in-flight pure ACK packets should not matter here. 942 * One way to get this would be to set skb->truesize = 2 on them. 943 */ 944 skb->ooo_okay = sk_wmem_alloc_get(sk) < SKB_TRUESIZE(1); 945 946 skb_push(skb, tcp_header_size); 947 skb_reset_transport_header(skb); 948 949 skb_orphan(skb); 950 skb->sk = sk; 951 skb->destructor = tcp_wfree; 952 skb_set_hash_from_sk(skb, sk); 953 atomic_add(skb->truesize, &sk->sk_wmem_alloc); 954 955 /* Build TCP header and checksum it. */ 956 th = tcp_hdr(skb); 957 th->source = inet->inet_sport; 958 th->dest = inet->inet_dport; 959 th->seq = htonl(tcb->seq); 960 th->ack_seq = htonl(tp->rcv_nxt); 961 *(((__be16 *)th) + 6) = htons(((tcp_header_size >> 2) << 12) | 962 tcb->tcp_flags); 963 964 if (unlikely(tcb->tcp_flags & TCPHDR_SYN)) { 965 /* RFC1323: The window in SYN & SYN/ACK segments 966 * is never scaled. 967 */ 968 th->window = htons(min(tp->rcv_wnd, 65535U)); 969 } else { 970 th->window = htons(tcp_select_window(sk)); 971 } 972 th->check = 0; 973 th->urg_ptr = 0; 974 975 /* The urg_mode check is necessary during a below snd_una win probe */ 976 if (unlikely(tcp_urg_mode(tp) && before(tcb->seq, tp->snd_up))) { 977 if (before(tp->snd_up, tcb->seq + 0x10000)) { 978 th->urg_ptr = htons(tp->snd_up - tcb->seq); 979 th->urg = 1; 980 } else if (after(tcb->seq + 0xFFFF, tp->snd_nxt)) { 981 th->urg_ptr = htons(0xFFFF); 982 th->urg = 1; 983 } 984 } 985 986 tcp_options_write((__be32 *)(th + 1), tp, &opts); 987 if (likely((tcb->tcp_flags & TCPHDR_SYN) == 0)) 988 tcp_ecn_send(sk, skb, tcp_header_size); 989 990 #ifdef CONFIG_TCP_MD5SIG 991 /* Calculate the MD5 hash, as we have all we need now */ 992 if (md5) { 993 sk_nocaps_add(sk, NETIF_F_GSO_MASK); 994 tp->af_specific->calc_md5_hash(opts.hash_location, 995 md5, sk, NULL, skb); 996 } 997 #endif 998 999 icsk->icsk_af_ops->send_check(sk, skb); 1000 1001 if (likely(tcb->tcp_flags & TCPHDR_ACK)) 1002 tcp_event_ack_sent(sk, tcp_skb_pcount(skb)); 1003 1004 if (skb->len != tcp_header_size) 1005 tcp_event_data_sent(tp, sk); 1006 1007 if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq) 1008 TCP_ADD_STATS(sock_net(sk), TCP_MIB_OUTSEGS, 1009 tcp_skb_pcount(skb)); 1010 1011 /* OK, its time to fill skb_shinfo(skb)->gso_segs */ 1012 skb_shinfo(skb)->gso_segs = tcp_skb_pcount(skb); 1013 1014 /* Our usage of tstamp should remain private */ 1015 skb->tstamp.tv64 = 0; 1016 1017 /* Cleanup our debris for IP stacks */ 1018 memset(skb->cb, 0, max(sizeof(struct inet_skb_parm), 1019 sizeof(struct inet6_skb_parm))); 1020 1021 err = icsk->icsk_af_ops->queue_xmit(sk, skb, &inet->cork.fl); 1022 1023 if (likely(err <= 0)) 1024 return err; 1025 1026 tcp_enter_cwr(sk); 1027 1028 return net_xmit_eval(err); 1029 } 1030 1031 /* This routine just queues the buffer for sending. 1032 * 1033 * NOTE: probe0 timer is not checked, do not forget tcp_push_pending_frames, 1034 * otherwise socket can stall. 1035 */ 1036 static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb) 1037 { 1038 struct tcp_sock *tp = tcp_sk(sk); 1039 1040 /* Advance write_seq and place onto the write_queue. */ 1041 tp->write_seq = TCP_SKB_CB(skb)->end_seq; 1042 __skb_header_release(skb); 1043 tcp_add_write_queue_tail(sk, skb); 1044 sk->sk_wmem_queued += skb->truesize; 1045 sk_mem_charge(sk, skb->truesize); 1046 } 1047 1048 /* Initialize TSO segments for a packet. */ 1049 static void tcp_set_skb_tso_segs(const struct sock *sk, struct sk_buff *skb, 1050 unsigned int mss_now) 1051 { 1052 struct skb_shared_info *shinfo = skb_shinfo(skb); 1053 1054 /* Make sure we own this skb before messing gso_size/gso_segs */ 1055 WARN_ON_ONCE(skb_cloned(skb)); 1056 1057 if (skb->len <= mss_now || skb->ip_summed == CHECKSUM_NONE) { 1058 /* Avoid the costly divide in the normal 1059 * non-TSO case. 1060 */ 1061 tcp_skb_pcount_set(skb, 1); 1062 shinfo->gso_size = 0; 1063 shinfo->gso_type = 0; 1064 } else { 1065 tcp_skb_pcount_set(skb, DIV_ROUND_UP(skb->len, mss_now)); 1066 shinfo->gso_size = mss_now; 1067 shinfo->gso_type = sk->sk_gso_type; 1068 } 1069 } 1070 1071 /* When a modification to fackets out becomes necessary, we need to check 1072 * skb is counted to fackets_out or not. 1073 */ 1074 static void tcp_adjust_fackets_out(struct sock *sk, const struct sk_buff *skb, 1075 int decr) 1076 { 1077 struct tcp_sock *tp = tcp_sk(sk); 1078 1079 if (!tp->sacked_out || tcp_is_reno(tp)) 1080 return; 1081 1082 if (after(tcp_highest_sack_seq(tp), TCP_SKB_CB(skb)->seq)) 1083 tp->fackets_out -= decr; 1084 } 1085 1086 /* Pcount in the middle of the write queue got changed, we need to do various 1087 * tweaks to fix counters 1088 */ 1089 static void tcp_adjust_pcount(struct sock *sk, const struct sk_buff *skb, int decr) 1090 { 1091 struct tcp_sock *tp = tcp_sk(sk); 1092 1093 tp->packets_out -= decr; 1094 1095 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED) 1096 tp->sacked_out -= decr; 1097 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) 1098 tp->retrans_out -= decr; 1099 if (TCP_SKB_CB(skb)->sacked & TCPCB_LOST) 1100 tp->lost_out -= decr; 1101 1102 /* Reno case is special. Sigh... */ 1103 if (tcp_is_reno(tp) && decr > 0) 1104 tp->sacked_out -= min_t(u32, tp->sacked_out, decr); 1105 1106 tcp_adjust_fackets_out(sk, skb, decr); 1107 1108 if (tp->lost_skb_hint && 1109 before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(tp->lost_skb_hint)->seq) && 1110 (tcp_is_fack(tp) || (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED))) 1111 tp->lost_cnt_hint -= decr; 1112 1113 tcp_verify_left_out(tp); 1114 } 1115 1116 static void tcp_fragment_tstamp(struct sk_buff *skb, struct sk_buff *skb2) 1117 { 1118 struct skb_shared_info *shinfo = skb_shinfo(skb); 1119 1120 if (unlikely(shinfo->tx_flags & SKBTX_ANY_TSTAMP) && 1121 !before(shinfo->tskey, TCP_SKB_CB(skb2)->seq)) { 1122 struct skb_shared_info *shinfo2 = skb_shinfo(skb2); 1123 u8 tsflags = shinfo->tx_flags & SKBTX_ANY_TSTAMP; 1124 1125 shinfo->tx_flags &= ~tsflags; 1126 shinfo2->tx_flags |= tsflags; 1127 swap(shinfo->tskey, shinfo2->tskey); 1128 } 1129 } 1130 1131 /* Function to create two new TCP segments. Shrinks the given segment 1132 * to the specified size and appends a new segment with the rest of the 1133 * packet to the list. This won't be called frequently, I hope. 1134 * Remember, these are still headerless SKBs at this point. 1135 */ 1136 int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len, 1137 unsigned int mss_now, gfp_t gfp) 1138 { 1139 struct tcp_sock *tp = tcp_sk(sk); 1140 struct sk_buff *buff; 1141 int nsize, old_factor; 1142 int nlen; 1143 u8 flags; 1144 1145 if (WARN_ON(len > skb->len)) 1146 return -EINVAL; 1147 1148 nsize = skb_headlen(skb) - len; 1149 if (nsize < 0) 1150 nsize = 0; 1151 1152 if (skb_unclone(skb, gfp)) 1153 return -ENOMEM; 1154 1155 /* Get a new skb... force flag on. */ 1156 buff = sk_stream_alloc_skb(sk, nsize, gfp); 1157 if (buff == NULL) 1158 return -ENOMEM; /* We'll just try again later. */ 1159 1160 sk->sk_wmem_queued += buff->truesize; 1161 sk_mem_charge(sk, buff->truesize); 1162 nlen = skb->len - len - nsize; 1163 buff->truesize += nlen; 1164 skb->truesize -= nlen; 1165 1166 /* Correct the sequence numbers. */ 1167 TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len; 1168 TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq; 1169 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq; 1170 1171 /* PSH and FIN should only be set in the second packet. */ 1172 flags = TCP_SKB_CB(skb)->tcp_flags; 1173 TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH); 1174 TCP_SKB_CB(buff)->tcp_flags = flags; 1175 TCP_SKB_CB(buff)->sacked = TCP_SKB_CB(skb)->sacked; 1176 1177 if (!skb_shinfo(skb)->nr_frags && skb->ip_summed != CHECKSUM_PARTIAL) { 1178 /* Copy and checksum data tail into the new buffer. */ 1179 buff->csum = csum_partial_copy_nocheck(skb->data + len, 1180 skb_put(buff, nsize), 1181 nsize, 0); 1182 1183 skb_trim(skb, len); 1184 1185 skb->csum = csum_block_sub(skb->csum, buff->csum, len); 1186 } else { 1187 skb->ip_summed = CHECKSUM_PARTIAL; 1188 skb_split(skb, buff, len); 1189 } 1190 1191 buff->ip_summed = skb->ip_summed; 1192 1193 buff->tstamp = skb->tstamp; 1194 tcp_fragment_tstamp(skb, buff); 1195 1196 old_factor = tcp_skb_pcount(skb); 1197 1198 /* Fix up tso_factor for both original and new SKB. */ 1199 tcp_set_skb_tso_segs(sk, skb, mss_now); 1200 tcp_set_skb_tso_segs(sk, buff, mss_now); 1201 1202 /* If this packet has been sent out already, we must 1203 * adjust the various packet counters. 1204 */ 1205 if (!before(tp->snd_nxt, TCP_SKB_CB(buff)->end_seq)) { 1206 int diff = old_factor - tcp_skb_pcount(skb) - 1207 tcp_skb_pcount(buff); 1208 1209 if (diff) 1210 tcp_adjust_pcount(sk, skb, diff); 1211 } 1212 1213 /* Link BUFF into the send queue. */ 1214 __skb_header_release(buff); 1215 tcp_insert_write_queue_after(skb, buff, sk); 1216 1217 return 0; 1218 } 1219 1220 /* This is similar to __pskb_pull_head() (it will go to core/skbuff.c 1221 * eventually). The difference is that pulled data not copied, but 1222 * immediately discarded. 1223 */ 1224 static void __pskb_trim_head(struct sk_buff *skb, int len) 1225 { 1226 struct skb_shared_info *shinfo; 1227 int i, k, eat; 1228 1229 eat = min_t(int, len, skb_headlen(skb)); 1230 if (eat) { 1231 __skb_pull(skb, eat); 1232 len -= eat; 1233 if (!len) 1234 return; 1235 } 1236 eat = len; 1237 k = 0; 1238 shinfo = skb_shinfo(skb); 1239 for (i = 0; i < shinfo->nr_frags; i++) { 1240 int size = skb_frag_size(&shinfo->frags[i]); 1241 1242 if (size <= eat) { 1243 skb_frag_unref(skb, i); 1244 eat -= size; 1245 } else { 1246 shinfo->frags[k] = shinfo->frags[i]; 1247 if (eat) { 1248 shinfo->frags[k].page_offset += eat; 1249 skb_frag_size_sub(&shinfo->frags[k], eat); 1250 eat = 0; 1251 } 1252 k++; 1253 } 1254 } 1255 shinfo->nr_frags = k; 1256 1257 skb_reset_tail_pointer(skb); 1258 skb->data_len -= len; 1259 skb->len = skb->data_len; 1260 } 1261 1262 /* Remove acked data from a packet in the transmit queue. */ 1263 int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len) 1264 { 1265 if (skb_unclone(skb, GFP_ATOMIC)) 1266 return -ENOMEM; 1267 1268 __pskb_trim_head(skb, len); 1269 1270 TCP_SKB_CB(skb)->seq += len; 1271 skb->ip_summed = CHECKSUM_PARTIAL; 1272 1273 skb->truesize -= len; 1274 sk->sk_wmem_queued -= len; 1275 sk_mem_uncharge(sk, len); 1276 sock_set_flag(sk, SOCK_QUEUE_SHRUNK); 1277 1278 /* Any change of skb->len requires recalculation of tso factor. */ 1279 if (tcp_skb_pcount(skb) > 1) 1280 tcp_set_skb_tso_segs(sk, skb, tcp_skb_mss(skb)); 1281 1282 return 0; 1283 } 1284 1285 /* Calculate MSS not accounting any TCP options. */ 1286 static inline int __tcp_mtu_to_mss(struct sock *sk, int pmtu) 1287 { 1288 const struct tcp_sock *tp = tcp_sk(sk); 1289 const struct inet_connection_sock *icsk = inet_csk(sk); 1290 int mss_now; 1291 1292 /* Calculate base mss without TCP options: 1293 It is MMS_S - sizeof(tcphdr) of rfc1122 1294 */ 1295 mss_now = pmtu - icsk->icsk_af_ops->net_header_len - sizeof(struct tcphdr); 1296 1297 /* IPv6 adds a frag_hdr in case RTAX_FEATURE_ALLFRAG is set */ 1298 if (icsk->icsk_af_ops->net_frag_header_len) { 1299 const struct dst_entry *dst = __sk_dst_get(sk); 1300 1301 if (dst && dst_allfrag(dst)) 1302 mss_now -= icsk->icsk_af_ops->net_frag_header_len; 1303 } 1304 1305 /* Clamp it (mss_clamp does not include tcp options) */ 1306 if (mss_now > tp->rx_opt.mss_clamp) 1307 mss_now = tp->rx_opt.mss_clamp; 1308 1309 /* Now subtract optional transport overhead */ 1310 mss_now -= icsk->icsk_ext_hdr_len; 1311 1312 /* Then reserve room for full set of TCP options and 8 bytes of data */ 1313 if (mss_now < 48) 1314 mss_now = 48; 1315 return mss_now; 1316 } 1317 1318 /* Calculate MSS. Not accounting for SACKs here. */ 1319 int tcp_mtu_to_mss(struct sock *sk, int pmtu) 1320 { 1321 /* Subtract TCP options size, not including SACKs */ 1322 return __tcp_mtu_to_mss(sk, pmtu) - 1323 (tcp_sk(sk)->tcp_header_len - sizeof(struct tcphdr)); 1324 } 1325 1326 /* Inverse of above */ 1327 int tcp_mss_to_mtu(struct sock *sk, int mss) 1328 { 1329 const struct tcp_sock *tp = tcp_sk(sk); 1330 const struct inet_connection_sock *icsk = inet_csk(sk); 1331 int mtu; 1332 1333 mtu = mss + 1334 tp->tcp_header_len + 1335 icsk->icsk_ext_hdr_len + 1336 icsk->icsk_af_ops->net_header_len; 1337 1338 /* IPv6 adds a frag_hdr in case RTAX_FEATURE_ALLFRAG is set */ 1339 if (icsk->icsk_af_ops->net_frag_header_len) { 1340 const struct dst_entry *dst = __sk_dst_get(sk); 1341 1342 if (dst && dst_allfrag(dst)) 1343 mtu += icsk->icsk_af_ops->net_frag_header_len; 1344 } 1345 return mtu; 1346 } 1347 1348 /* MTU probing init per socket */ 1349 void tcp_mtup_init(struct sock *sk) 1350 { 1351 struct tcp_sock *tp = tcp_sk(sk); 1352 struct inet_connection_sock *icsk = inet_csk(sk); 1353 1354 icsk->icsk_mtup.enabled = sysctl_tcp_mtu_probing > 1; 1355 icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp + sizeof(struct tcphdr) + 1356 icsk->icsk_af_ops->net_header_len; 1357 icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, sysctl_tcp_base_mss); 1358 icsk->icsk_mtup.probe_size = 0; 1359 } 1360 EXPORT_SYMBOL(tcp_mtup_init); 1361 1362 /* This function synchronize snd mss to current pmtu/exthdr set. 1363 1364 tp->rx_opt.user_mss is mss set by user by TCP_MAXSEG. It does NOT counts 1365 for TCP options, but includes only bare TCP header. 1366 1367 tp->rx_opt.mss_clamp is mss negotiated at connection setup. 1368 It is minimum of user_mss and mss received with SYN. 1369 It also does not include TCP options. 1370 1371 inet_csk(sk)->icsk_pmtu_cookie is last pmtu, seen by this function. 1372 1373 tp->mss_cache is current effective sending mss, including 1374 all tcp options except for SACKs. It is evaluated, 1375 taking into account current pmtu, but never exceeds 1376 tp->rx_opt.mss_clamp. 1377 1378 NOTE1. rfc1122 clearly states that advertised MSS 1379 DOES NOT include either tcp or ip options. 1380 1381 NOTE2. inet_csk(sk)->icsk_pmtu_cookie and tp->mss_cache 1382 are READ ONLY outside this function. --ANK (980731) 1383 */ 1384 unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu) 1385 { 1386 struct tcp_sock *tp = tcp_sk(sk); 1387 struct inet_connection_sock *icsk = inet_csk(sk); 1388 int mss_now; 1389 1390 if (icsk->icsk_mtup.search_high > pmtu) 1391 icsk->icsk_mtup.search_high = pmtu; 1392 1393 mss_now = tcp_mtu_to_mss(sk, pmtu); 1394 mss_now = tcp_bound_to_half_wnd(tp, mss_now); 1395 1396 /* And store cached results */ 1397 icsk->icsk_pmtu_cookie = pmtu; 1398 if (icsk->icsk_mtup.enabled) 1399 mss_now = min(mss_now, tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_low)); 1400 tp->mss_cache = mss_now; 1401 1402 return mss_now; 1403 } 1404 EXPORT_SYMBOL(tcp_sync_mss); 1405 1406 /* Compute the current effective MSS, taking SACKs and IP options, 1407 * and even PMTU discovery events into account. 1408 */ 1409 unsigned int tcp_current_mss(struct sock *sk) 1410 { 1411 const struct tcp_sock *tp = tcp_sk(sk); 1412 const struct dst_entry *dst = __sk_dst_get(sk); 1413 u32 mss_now; 1414 unsigned int header_len; 1415 struct tcp_out_options opts; 1416 struct tcp_md5sig_key *md5; 1417 1418 mss_now = tp->mss_cache; 1419 1420 if (dst) { 1421 u32 mtu = dst_mtu(dst); 1422 if (mtu != inet_csk(sk)->icsk_pmtu_cookie) 1423 mss_now = tcp_sync_mss(sk, mtu); 1424 } 1425 1426 header_len = tcp_established_options(sk, NULL, &opts, &md5) + 1427 sizeof(struct tcphdr); 1428 /* The mss_cache is sized based on tp->tcp_header_len, which assumes 1429 * some common options. If this is an odd packet (because we have SACK 1430 * blocks etc) then our calculated header_len will be different, and 1431 * we have to adjust mss_now correspondingly */ 1432 if (header_len != tp->tcp_header_len) { 1433 int delta = (int) header_len - tp->tcp_header_len; 1434 mss_now -= delta; 1435 } 1436 1437 return mss_now; 1438 } 1439 1440 /* RFC2861, slow part. Adjust cwnd, after it was not full during one rto. 1441 * As additional protections, we do not touch cwnd in retransmission phases, 1442 * and if application hit its sndbuf limit recently. 1443 */ 1444 static void tcp_cwnd_application_limited(struct sock *sk) 1445 { 1446 struct tcp_sock *tp = tcp_sk(sk); 1447 1448 if (inet_csk(sk)->icsk_ca_state == TCP_CA_Open && 1449 sk->sk_socket && !test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) { 1450 /* Limited by application or receiver window. */ 1451 u32 init_win = tcp_init_cwnd(tp, __sk_dst_get(sk)); 1452 u32 win_used = max(tp->snd_cwnd_used, init_win); 1453 if (win_used < tp->snd_cwnd) { 1454 tp->snd_ssthresh = tcp_current_ssthresh(sk); 1455 tp->snd_cwnd = (tp->snd_cwnd + win_used) >> 1; 1456 } 1457 tp->snd_cwnd_used = 0; 1458 } 1459 tp->snd_cwnd_stamp = tcp_time_stamp; 1460 } 1461 1462 static void tcp_cwnd_validate(struct sock *sk, bool is_cwnd_limited) 1463 { 1464 struct tcp_sock *tp = tcp_sk(sk); 1465 1466 /* Track the maximum number of outstanding packets in each 1467 * window, and remember whether we were cwnd-limited then. 1468 */ 1469 if (!before(tp->snd_una, tp->max_packets_seq) || 1470 tp->packets_out > tp->max_packets_out) { 1471 tp->max_packets_out = tp->packets_out; 1472 tp->max_packets_seq = tp->snd_nxt; 1473 tp->is_cwnd_limited = is_cwnd_limited; 1474 } 1475 1476 if (tcp_is_cwnd_limited(sk)) { 1477 /* Network is feed fully. */ 1478 tp->snd_cwnd_used = 0; 1479 tp->snd_cwnd_stamp = tcp_time_stamp; 1480 } else { 1481 /* Network starves. */ 1482 if (tp->packets_out > tp->snd_cwnd_used) 1483 tp->snd_cwnd_used = tp->packets_out; 1484 1485 if (sysctl_tcp_slow_start_after_idle && 1486 (s32)(tcp_time_stamp - tp->snd_cwnd_stamp) >= inet_csk(sk)->icsk_rto) 1487 tcp_cwnd_application_limited(sk); 1488 } 1489 } 1490 1491 /* Minshall's variant of the Nagle send check. */ 1492 static bool tcp_minshall_check(const struct tcp_sock *tp) 1493 { 1494 return after(tp->snd_sml, tp->snd_una) && 1495 !after(tp->snd_sml, tp->snd_nxt); 1496 } 1497 1498 /* Update snd_sml if this skb is under mss 1499 * Note that a TSO packet might end with a sub-mss segment 1500 * The test is really : 1501 * if ((skb->len % mss) != 0) 1502 * tp->snd_sml = TCP_SKB_CB(skb)->end_seq; 1503 * But we can avoid doing the divide again given we already have 1504 * skb_pcount = skb->len / mss_now 1505 */ 1506 static void tcp_minshall_update(struct tcp_sock *tp, unsigned int mss_now, 1507 const struct sk_buff *skb) 1508 { 1509 if (skb->len < tcp_skb_pcount(skb) * mss_now) 1510 tp->snd_sml = TCP_SKB_CB(skb)->end_seq; 1511 } 1512 1513 /* Return false, if packet can be sent now without violation Nagle's rules: 1514 * 1. It is full sized. (provided by caller in %partial bool) 1515 * 2. Or it contains FIN. (already checked by caller) 1516 * 3. Or TCP_CORK is not set, and TCP_NODELAY is set. 1517 * 4. Or TCP_CORK is not set, and all sent packets are ACKed. 1518 * With Minshall's modification: all sent small packets are ACKed. 1519 */ 1520 static bool tcp_nagle_check(bool partial, const struct tcp_sock *tp, 1521 int nonagle) 1522 { 1523 return partial && 1524 ((nonagle & TCP_NAGLE_CORK) || 1525 (!nonagle && tp->packets_out && tcp_minshall_check(tp))); 1526 } 1527 1528 /* Return how many segs we'd like on a TSO packet, 1529 * to send one TSO packet per ms 1530 */ 1531 static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now) 1532 { 1533 u32 bytes, segs; 1534 1535 bytes = min(sk->sk_pacing_rate >> 10, 1536 sk->sk_gso_max_size - 1 - MAX_TCP_HEADER); 1537 1538 /* Goal is to send at least one packet per ms, 1539 * not one big TSO packet every 100 ms. 1540 * This preserves ACK clocking and is consistent 1541 * with tcp_tso_should_defer() heuristic. 1542 */ 1543 segs = max_t(u32, bytes / mss_now, sysctl_tcp_min_tso_segs); 1544 1545 return min_t(u32, segs, sk->sk_gso_max_segs); 1546 } 1547 1548 /* Returns the portion of skb which can be sent right away */ 1549 static unsigned int tcp_mss_split_point(const struct sock *sk, 1550 const struct sk_buff *skb, 1551 unsigned int mss_now, 1552 unsigned int max_segs, 1553 int nonagle) 1554 { 1555 const struct tcp_sock *tp = tcp_sk(sk); 1556 u32 partial, needed, window, max_len; 1557 1558 window = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq; 1559 max_len = mss_now * max_segs; 1560 1561 if (likely(max_len <= window && skb != tcp_write_queue_tail(sk))) 1562 return max_len; 1563 1564 needed = min(skb->len, window); 1565 1566 if (max_len <= needed) 1567 return max_len; 1568 1569 partial = needed % mss_now; 1570 /* If last segment is not a full MSS, check if Nagle rules allow us 1571 * to include this last segment in this skb. 1572 * Otherwise, we'll split the skb at last MSS boundary 1573 */ 1574 if (tcp_nagle_check(partial != 0, tp, nonagle)) 1575 return needed - partial; 1576 1577 return needed; 1578 } 1579 1580 /* Can at least one segment of SKB be sent right now, according to the 1581 * congestion window rules? If so, return how many segments are allowed. 1582 */ 1583 static inline unsigned int tcp_cwnd_test(const struct tcp_sock *tp, 1584 const struct sk_buff *skb) 1585 { 1586 u32 in_flight, cwnd, halfcwnd; 1587 1588 /* Don't be strict about the congestion window for the final FIN. */ 1589 if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) && 1590 tcp_skb_pcount(skb) == 1) 1591 return 1; 1592 1593 in_flight = tcp_packets_in_flight(tp); 1594 cwnd = tp->snd_cwnd; 1595 if (in_flight >= cwnd) 1596 return 0; 1597 1598 /* For better scheduling, ensure we have at least 1599 * 2 GSO packets in flight. 1600 */ 1601 halfcwnd = max(cwnd >> 1, 1U); 1602 return min(halfcwnd, cwnd - in_flight); 1603 } 1604 1605 /* Initialize TSO state of a skb. 1606 * This must be invoked the first time we consider transmitting 1607 * SKB onto the wire. 1608 */ 1609 static int tcp_init_tso_segs(const struct sock *sk, struct sk_buff *skb, 1610 unsigned int mss_now) 1611 { 1612 int tso_segs = tcp_skb_pcount(skb); 1613 1614 if (!tso_segs || (tso_segs > 1 && tcp_skb_mss(skb) != mss_now)) { 1615 tcp_set_skb_tso_segs(sk, skb, mss_now); 1616 tso_segs = tcp_skb_pcount(skb); 1617 } 1618 return tso_segs; 1619 } 1620 1621 1622 /* Return true if the Nagle test allows this packet to be 1623 * sent now. 1624 */ 1625 static inline bool tcp_nagle_test(const struct tcp_sock *tp, const struct sk_buff *skb, 1626 unsigned int cur_mss, int nonagle) 1627 { 1628 /* Nagle rule does not apply to frames, which sit in the middle of the 1629 * write_queue (they have no chances to get new data). 1630 * 1631 * This is implemented in the callers, where they modify the 'nonagle' 1632 * argument based upon the location of SKB in the send queue. 1633 */ 1634 if (nonagle & TCP_NAGLE_PUSH) 1635 return true; 1636 1637 /* Don't use the nagle rule for urgent data (or for the final FIN). */ 1638 if (tcp_urg_mode(tp) || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) 1639 return true; 1640 1641 if (!tcp_nagle_check(skb->len < cur_mss, tp, nonagle)) 1642 return true; 1643 1644 return false; 1645 } 1646 1647 /* Does at least the first segment of SKB fit into the send window? */ 1648 static bool tcp_snd_wnd_test(const struct tcp_sock *tp, 1649 const struct sk_buff *skb, 1650 unsigned int cur_mss) 1651 { 1652 u32 end_seq = TCP_SKB_CB(skb)->end_seq; 1653 1654 if (skb->len > cur_mss) 1655 end_seq = TCP_SKB_CB(skb)->seq + cur_mss; 1656 1657 return !after(end_seq, tcp_wnd_end(tp)); 1658 } 1659 1660 /* This checks if the data bearing packet SKB (usually tcp_send_head(sk)) 1661 * should be put on the wire right now. If so, it returns the number of 1662 * packets allowed by the congestion window. 1663 */ 1664 static unsigned int tcp_snd_test(const struct sock *sk, struct sk_buff *skb, 1665 unsigned int cur_mss, int nonagle) 1666 { 1667 const struct tcp_sock *tp = tcp_sk(sk); 1668 unsigned int cwnd_quota; 1669 1670 tcp_init_tso_segs(sk, skb, cur_mss); 1671 1672 if (!tcp_nagle_test(tp, skb, cur_mss, nonagle)) 1673 return 0; 1674 1675 cwnd_quota = tcp_cwnd_test(tp, skb); 1676 if (cwnd_quota && !tcp_snd_wnd_test(tp, skb, cur_mss)) 1677 cwnd_quota = 0; 1678 1679 return cwnd_quota; 1680 } 1681 1682 /* Test if sending is allowed right now. */ 1683 bool tcp_may_send_now(struct sock *sk) 1684 { 1685 const struct tcp_sock *tp = tcp_sk(sk); 1686 struct sk_buff *skb = tcp_send_head(sk); 1687 1688 return skb && 1689 tcp_snd_test(sk, skb, tcp_current_mss(sk), 1690 (tcp_skb_is_last(sk, skb) ? 1691 tp->nonagle : TCP_NAGLE_PUSH)); 1692 } 1693 1694 /* Trim TSO SKB to LEN bytes, put the remaining data into a new packet 1695 * which is put after SKB on the list. It is very much like 1696 * tcp_fragment() except that it may make several kinds of assumptions 1697 * in order to speed up the splitting operation. In particular, we 1698 * know that all the data is in scatter-gather pages, and that the 1699 * packet has never been sent out before (and thus is not cloned). 1700 */ 1701 static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len, 1702 unsigned int mss_now, gfp_t gfp) 1703 { 1704 struct sk_buff *buff; 1705 int nlen = skb->len - len; 1706 u8 flags; 1707 1708 /* All of a TSO frame must be composed of paged data. */ 1709 if (skb->len != skb->data_len) 1710 return tcp_fragment(sk, skb, len, mss_now, gfp); 1711 1712 buff = sk_stream_alloc_skb(sk, 0, gfp); 1713 if (unlikely(buff == NULL)) 1714 return -ENOMEM; 1715 1716 sk->sk_wmem_queued += buff->truesize; 1717 sk_mem_charge(sk, buff->truesize); 1718 buff->truesize += nlen; 1719 skb->truesize -= nlen; 1720 1721 /* Correct the sequence numbers. */ 1722 TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len; 1723 TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq; 1724 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq; 1725 1726 /* PSH and FIN should only be set in the second packet. */ 1727 flags = TCP_SKB_CB(skb)->tcp_flags; 1728 TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH); 1729 TCP_SKB_CB(buff)->tcp_flags = flags; 1730 1731 /* This packet was never sent out yet, so no SACK bits. */ 1732 TCP_SKB_CB(buff)->sacked = 0; 1733 1734 buff->ip_summed = skb->ip_summed = CHECKSUM_PARTIAL; 1735 skb_split(skb, buff, len); 1736 tcp_fragment_tstamp(skb, buff); 1737 1738 /* Fix up tso_factor for both original and new SKB. */ 1739 tcp_set_skb_tso_segs(sk, skb, mss_now); 1740 tcp_set_skb_tso_segs(sk, buff, mss_now); 1741 1742 /* Link BUFF into the send queue. */ 1743 __skb_header_release(buff); 1744 tcp_insert_write_queue_after(skb, buff, sk); 1745 1746 return 0; 1747 } 1748 1749 /* Try to defer sending, if possible, in order to minimize the amount 1750 * of TSO splitting we do. View it as a kind of TSO Nagle test. 1751 * 1752 * This algorithm is from John Heffner. 1753 */ 1754 static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb, 1755 bool *is_cwnd_limited, u32 max_segs) 1756 { 1757 struct tcp_sock *tp = tcp_sk(sk); 1758 const struct inet_connection_sock *icsk = inet_csk(sk); 1759 u32 send_win, cong_win, limit, in_flight; 1760 int win_divisor; 1761 1762 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 1763 goto send_now; 1764 1765 if (icsk->icsk_ca_state != TCP_CA_Open) 1766 goto send_now; 1767 1768 /* Defer for less than two clock ticks. */ 1769 if (tp->tso_deferred && 1770 (((u32)jiffies << 1) >> 1) - (tp->tso_deferred >> 1) > 1) 1771 goto send_now; 1772 1773 in_flight = tcp_packets_in_flight(tp); 1774 1775 BUG_ON(tcp_skb_pcount(skb) <= 1 || (tp->snd_cwnd <= in_flight)); 1776 1777 send_win = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq; 1778 1779 /* From in_flight test above, we know that cwnd > in_flight. */ 1780 cong_win = (tp->snd_cwnd - in_flight) * tp->mss_cache; 1781 1782 limit = min(send_win, cong_win); 1783 1784 /* If a full-sized TSO skb can be sent, do it. */ 1785 if (limit >= max_segs * tp->mss_cache) 1786 goto send_now; 1787 1788 /* Middle in queue won't get any more data, full sendable already? */ 1789 if ((skb != tcp_write_queue_tail(sk)) && (limit >= skb->len)) 1790 goto send_now; 1791 1792 win_divisor = ACCESS_ONCE(sysctl_tcp_tso_win_divisor); 1793 if (win_divisor) { 1794 u32 chunk = min(tp->snd_wnd, tp->snd_cwnd * tp->mss_cache); 1795 1796 /* If at least some fraction of a window is available, 1797 * just use it. 1798 */ 1799 chunk /= win_divisor; 1800 if (limit >= chunk) 1801 goto send_now; 1802 } else { 1803 /* Different approach, try not to defer past a single 1804 * ACK. Receiver should ACK every other full sized 1805 * frame, so if we have space for more than 3 frames 1806 * then send now. 1807 */ 1808 if (limit > tcp_max_tso_deferred_mss(tp) * tp->mss_cache) 1809 goto send_now; 1810 } 1811 1812 /* Ok, it looks like it is advisable to defer. 1813 * Do not rearm the timer if already set to not break TCP ACK clocking. 1814 */ 1815 if (!tp->tso_deferred) 1816 tp->tso_deferred = 1 | (jiffies << 1); 1817 1818 if (cong_win < send_win && cong_win < skb->len) 1819 *is_cwnd_limited = true; 1820 1821 return true; 1822 1823 send_now: 1824 tp->tso_deferred = 0; 1825 return false; 1826 } 1827 1828 /* Create a new MTU probe if we are ready. 1829 * MTU probe is regularly attempting to increase the path MTU by 1830 * deliberately sending larger packets. This discovers routing 1831 * changes resulting in larger path MTUs. 1832 * 1833 * Returns 0 if we should wait to probe (no cwnd available), 1834 * 1 if a probe was sent, 1835 * -1 otherwise 1836 */ 1837 static int tcp_mtu_probe(struct sock *sk) 1838 { 1839 struct tcp_sock *tp = tcp_sk(sk); 1840 struct inet_connection_sock *icsk = inet_csk(sk); 1841 struct sk_buff *skb, *nskb, *next; 1842 int len; 1843 int probe_size; 1844 int size_needed; 1845 int copy; 1846 int mss_now; 1847 1848 /* Not currently probing/verifying, 1849 * not in recovery, 1850 * have enough cwnd, and 1851 * not SACKing (the variable headers throw things off) */ 1852 if (!icsk->icsk_mtup.enabled || 1853 icsk->icsk_mtup.probe_size || 1854 inet_csk(sk)->icsk_ca_state != TCP_CA_Open || 1855 tp->snd_cwnd < 11 || 1856 tp->rx_opt.num_sacks || tp->rx_opt.dsack) 1857 return -1; 1858 1859 /* Very simple search strategy: just double the MSS. */ 1860 mss_now = tcp_current_mss(sk); 1861 probe_size = 2 * tp->mss_cache; 1862 size_needed = probe_size + (tp->reordering + 1) * tp->mss_cache; 1863 if (probe_size > tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_high)) { 1864 /* TODO: set timer for probe_converge_event */ 1865 return -1; 1866 } 1867 1868 /* Have enough data in the send queue to probe? */ 1869 if (tp->write_seq - tp->snd_nxt < size_needed) 1870 return -1; 1871 1872 if (tp->snd_wnd < size_needed) 1873 return -1; 1874 if (after(tp->snd_nxt + size_needed, tcp_wnd_end(tp))) 1875 return 0; 1876 1877 /* Do we need to wait to drain cwnd? With none in flight, don't stall */ 1878 if (tcp_packets_in_flight(tp) + 2 > tp->snd_cwnd) { 1879 if (!tcp_packets_in_flight(tp)) 1880 return -1; 1881 else 1882 return 0; 1883 } 1884 1885 /* We're allowed to probe. Build it now. */ 1886 if ((nskb = sk_stream_alloc_skb(sk, probe_size, GFP_ATOMIC)) == NULL) 1887 return -1; 1888 sk->sk_wmem_queued += nskb->truesize; 1889 sk_mem_charge(sk, nskb->truesize); 1890 1891 skb = tcp_send_head(sk); 1892 1893 TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq; 1894 TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size; 1895 TCP_SKB_CB(nskb)->tcp_flags = TCPHDR_ACK; 1896 TCP_SKB_CB(nskb)->sacked = 0; 1897 nskb->csum = 0; 1898 nskb->ip_summed = skb->ip_summed; 1899 1900 tcp_insert_write_queue_before(nskb, skb, sk); 1901 1902 len = 0; 1903 tcp_for_write_queue_from_safe(skb, next, sk) { 1904 copy = min_t(int, skb->len, probe_size - len); 1905 if (nskb->ip_summed) 1906 skb_copy_bits(skb, 0, skb_put(nskb, copy), copy); 1907 else 1908 nskb->csum = skb_copy_and_csum_bits(skb, 0, 1909 skb_put(nskb, copy), 1910 copy, nskb->csum); 1911 1912 if (skb->len <= copy) { 1913 /* We've eaten all the data from this skb. 1914 * Throw it away. */ 1915 TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags; 1916 tcp_unlink_write_queue(skb, sk); 1917 sk_wmem_free_skb(sk, skb); 1918 } else { 1919 TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags & 1920 ~(TCPHDR_FIN|TCPHDR_PSH); 1921 if (!skb_shinfo(skb)->nr_frags) { 1922 skb_pull(skb, copy); 1923 if (skb->ip_summed != CHECKSUM_PARTIAL) 1924 skb->csum = csum_partial(skb->data, 1925 skb->len, 0); 1926 } else { 1927 __pskb_trim_head(skb, copy); 1928 tcp_set_skb_tso_segs(sk, skb, mss_now); 1929 } 1930 TCP_SKB_CB(skb)->seq += copy; 1931 } 1932 1933 len += copy; 1934 1935 if (len >= probe_size) 1936 break; 1937 } 1938 tcp_init_tso_segs(sk, nskb, nskb->len); 1939 1940 /* We're ready to send. If this fails, the probe will 1941 * be resegmented into mss-sized pieces by tcp_write_xmit(). 1942 */ 1943 if (!tcp_transmit_skb(sk, nskb, 1, GFP_ATOMIC)) { 1944 /* Decrement cwnd here because we are sending 1945 * effectively two packets. */ 1946 tp->snd_cwnd--; 1947 tcp_event_new_data_sent(sk, nskb); 1948 1949 icsk->icsk_mtup.probe_size = tcp_mss_to_mtu(sk, nskb->len); 1950 tp->mtu_probe.probe_seq_start = TCP_SKB_CB(nskb)->seq; 1951 tp->mtu_probe.probe_seq_end = TCP_SKB_CB(nskb)->end_seq; 1952 1953 return 1; 1954 } 1955 1956 return -1; 1957 } 1958 1959 /* This routine writes packets to the network. It advances the 1960 * send_head. This happens as incoming acks open up the remote 1961 * window for us. 1962 * 1963 * LARGESEND note: !tcp_urg_mode is overkill, only frames between 1964 * snd_up-64k-mss .. snd_up cannot be large. However, taking into 1965 * account rare use of URG, this is not a big flaw. 1966 * 1967 * Send at most one packet when push_one > 0. Temporarily ignore 1968 * cwnd limit to force at most one packet out when push_one == 2. 1969 1970 * Returns true, if no segments are in flight and we have queued segments, 1971 * but cannot send anything now because of SWS or another problem. 1972 */ 1973 static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, 1974 int push_one, gfp_t gfp) 1975 { 1976 struct tcp_sock *tp = tcp_sk(sk); 1977 struct sk_buff *skb; 1978 unsigned int tso_segs, sent_pkts; 1979 int cwnd_quota; 1980 int result; 1981 bool is_cwnd_limited = false; 1982 u32 max_segs; 1983 1984 sent_pkts = 0; 1985 1986 if (!push_one) { 1987 /* Do MTU probing. */ 1988 result = tcp_mtu_probe(sk); 1989 if (!result) { 1990 return false; 1991 } else if (result > 0) { 1992 sent_pkts = 1; 1993 } 1994 } 1995 1996 max_segs = tcp_tso_autosize(sk, mss_now); 1997 while ((skb = tcp_send_head(sk))) { 1998 unsigned int limit; 1999 2000 tso_segs = tcp_init_tso_segs(sk, skb, mss_now); 2001 BUG_ON(!tso_segs); 2002 2003 if (unlikely(tp->repair) && tp->repair_queue == TCP_SEND_QUEUE) { 2004 /* "skb_mstamp" is used as a start point for the retransmit timer */ 2005 skb_mstamp_get(&skb->skb_mstamp); 2006 goto repair; /* Skip network transmission */ 2007 } 2008 2009 cwnd_quota = tcp_cwnd_test(tp, skb); 2010 if (!cwnd_quota) { 2011 is_cwnd_limited = true; 2012 if (push_one == 2) 2013 /* Force out a loss probe pkt. */ 2014 cwnd_quota = 1; 2015 else 2016 break; 2017 } 2018 2019 if (unlikely(!tcp_snd_wnd_test(tp, skb, mss_now))) 2020 break; 2021 2022 if (tso_segs == 1 || !max_segs) { 2023 if (unlikely(!tcp_nagle_test(tp, skb, mss_now, 2024 (tcp_skb_is_last(sk, skb) ? 2025 nonagle : TCP_NAGLE_PUSH)))) 2026 break; 2027 } else { 2028 if (!push_one && 2029 tcp_tso_should_defer(sk, skb, &is_cwnd_limited, 2030 max_segs)) 2031 break; 2032 } 2033 2034 limit = mss_now; 2035 if (tso_segs > 1 && max_segs && !tcp_urg_mode(tp)) 2036 limit = tcp_mss_split_point(sk, skb, mss_now, 2037 min_t(unsigned int, 2038 cwnd_quota, 2039 max_segs), 2040 nonagle); 2041 2042 if (skb->len > limit && 2043 unlikely(tso_fragment(sk, skb, limit, mss_now, gfp))) 2044 break; 2045 2046 /* TCP Small Queues : 2047 * Control number of packets in qdisc/devices to two packets / or ~1 ms. 2048 * This allows for : 2049 * - better RTT estimation and ACK scheduling 2050 * - faster recovery 2051 * - high rates 2052 * Alas, some drivers / subsystems require a fair amount 2053 * of queued bytes to ensure line rate. 2054 * One example is wifi aggregation (802.11 AMPDU) 2055 */ 2056 limit = max(2 * skb->truesize, sk->sk_pacing_rate >> 10); 2057 limit = min_t(u32, limit, sysctl_tcp_limit_output_bytes); 2058 2059 if (atomic_read(&sk->sk_wmem_alloc) > limit) { 2060 set_bit(TSQ_THROTTLED, &tp->tsq_flags); 2061 /* It is possible TX completion already happened 2062 * before we set TSQ_THROTTLED, so we must 2063 * test again the condition. 2064 */ 2065 smp_mb__after_atomic(); 2066 if (atomic_read(&sk->sk_wmem_alloc) > limit) 2067 break; 2068 } 2069 2070 if (unlikely(tcp_transmit_skb(sk, skb, 1, gfp))) 2071 break; 2072 2073 repair: 2074 /* Advance the send_head. This one is sent out. 2075 * This call will increment packets_out. 2076 */ 2077 tcp_event_new_data_sent(sk, skb); 2078 2079 tcp_minshall_update(tp, mss_now, skb); 2080 sent_pkts += tcp_skb_pcount(skb); 2081 2082 if (push_one) 2083 break; 2084 } 2085 2086 if (likely(sent_pkts)) { 2087 if (tcp_in_cwnd_reduction(sk)) 2088 tp->prr_out += sent_pkts; 2089 2090 /* Send one loss probe per tail loss episode. */ 2091 if (push_one != 2) 2092 tcp_schedule_loss_probe(sk); 2093 tcp_cwnd_validate(sk, is_cwnd_limited); 2094 return false; 2095 } 2096 return (push_one == 2) || (!tp->packets_out && tcp_send_head(sk)); 2097 } 2098 2099 bool tcp_schedule_loss_probe(struct sock *sk) 2100 { 2101 struct inet_connection_sock *icsk = inet_csk(sk); 2102 struct tcp_sock *tp = tcp_sk(sk); 2103 u32 timeout, tlp_time_stamp, rto_time_stamp; 2104 u32 rtt = usecs_to_jiffies(tp->srtt_us >> 3); 2105 2106 if (WARN_ON(icsk->icsk_pending == ICSK_TIME_EARLY_RETRANS)) 2107 return false; 2108 /* No consecutive loss probes. */ 2109 if (WARN_ON(icsk->icsk_pending == ICSK_TIME_LOSS_PROBE)) { 2110 tcp_rearm_rto(sk); 2111 return false; 2112 } 2113 /* Don't do any loss probe on a Fast Open connection before 3WHS 2114 * finishes. 2115 */ 2116 if (sk->sk_state == TCP_SYN_RECV) 2117 return false; 2118 2119 /* TLP is only scheduled when next timer event is RTO. */ 2120 if (icsk->icsk_pending != ICSK_TIME_RETRANS) 2121 return false; 2122 2123 /* Schedule a loss probe in 2*RTT for SACK capable connections 2124 * in Open state, that are either limited by cwnd or application. 2125 */ 2126 if (sysctl_tcp_early_retrans < 3 || !tp->srtt_us || !tp->packets_out || 2127 !tcp_is_sack(tp) || inet_csk(sk)->icsk_ca_state != TCP_CA_Open) 2128 return false; 2129 2130 if ((tp->snd_cwnd > tcp_packets_in_flight(tp)) && 2131 tcp_send_head(sk)) 2132 return false; 2133 2134 /* Probe timeout is at least 1.5*rtt + TCP_DELACK_MAX to account 2135 * for delayed ack when there's one outstanding packet. 2136 */ 2137 timeout = rtt << 1; 2138 if (tp->packets_out == 1) 2139 timeout = max_t(u32, timeout, 2140 (rtt + (rtt >> 1) + TCP_DELACK_MAX)); 2141 timeout = max_t(u32, timeout, msecs_to_jiffies(10)); 2142 2143 /* If RTO is shorter, just schedule TLP in its place. */ 2144 tlp_time_stamp = tcp_time_stamp + timeout; 2145 rto_time_stamp = (u32)inet_csk(sk)->icsk_timeout; 2146 if ((s32)(tlp_time_stamp - rto_time_stamp) > 0) { 2147 s32 delta = rto_time_stamp - tcp_time_stamp; 2148 if (delta > 0) 2149 timeout = delta; 2150 } 2151 2152 inet_csk_reset_xmit_timer(sk, ICSK_TIME_LOSS_PROBE, timeout, 2153 TCP_RTO_MAX); 2154 return true; 2155 } 2156 2157 /* Thanks to skb fast clones, we can detect if a prior transmit of 2158 * a packet is still in a qdisc or driver queue. 2159 * In this case, there is very little point doing a retransmit ! 2160 * Note: This is called from BH context only. 2161 */ 2162 static bool skb_still_in_host_queue(const struct sock *sk, 2163 const struct sk_buff *skb) 2164 { 2165 if (unlikely(skb_fclone_busy(sk, skb))) { 2166 NET_INC_STATS_BH(sock_net(sk), 2167 LINUX_MIB_TCPSPURIOUS_RTX_HOSTQUEUES); 2168 return true; 2169 } 2170 return false; 2171 } 2172 2173 /* When probe timeout (PTO) fires, send a new segment if one exists, else 2174 * retransmit the last segment. 2175 */ 2176 void tcp_send_loss_probe(struct sock *sk) 2177 { 2178 struct tcp_sock *tp = tcp_sk(sk); 2179 struct sk_buff *skb; 2180 int pcount; 2181 int mss = tcp_current_mss(sk); 2182 int err = -1; 2183 2184 if (tcp_send_head(sk) != NULL) { 2185 err = tcp_write_xmit(sk, mss, TCP_NAGLE_OFF, 2, GFP_ATOMIC); 2186 goto rearm_timer; 2187 } 2188 2189 /* At most one outstanding TLP retransmission. */ 2190 if (tp->tlp_high_seq) 2191 goto rearm_timer; 2192 2193 /* Retransmit last segment. */ 2194 skb = tcp_write_queue_tail(sk); 2195 if (WARN_ON(!skb)) 2196 goto rearm_timer; 2197 2198 if (skb_still_in_host_queue(sk, skb)) 2199 goto rearm_timer; 2200 2201 pcount = tcp_skb_pcount(skb); 2202 if (WARN_ON(!pcount)) 2203 goto rearm_timer; 2204 2205 if ((pcount > 1) && (skb->len > (pcount - 1) * mss)) { 2206 if (unlikely(tcp_fragment(sk, skb, (pcount - 1) * mss, mss, 2207 GFP_ATOMIC))) 2208 goto rearm_timer; 2209 skb = tcp_write_queue_tail(sk); 2210 } 2211 2212 if (WARN_ON(!skb || !tcp_skb_pcount(skb))) 2213 goto rearm_timer; 2214 2215 err = __tcp_retransmit_skb(sk, skb); 2216 2217 /* Record snd_nxt for loss detection. */ 2218 if (likely(!err)) 2219 tp->tlp_high_seq = tp->snd_nxt; 2220 2221 rearm_timer: 2222 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, 2223 inet_csk(sk)->icsk_rto, 2224 TCP_RTO_MAX); 2225 2226 if (likely(!err)) 2227 NET_INC_STATS_BH(sock_net(sk), 2228 LINUX_MIB_TCPLOSSPROBES); 2229 } 2230 2231 /* Push out any pending frames which were held back due to 2232 * TCP_CORK or attempt at coalescing tiny packets. 2233 * The socket must be locked by the caller. 2234 */ 2235 void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss, 2236 int nonagle) 2237 { 2238 /* If we are closed, the bytes will have to remain here. 2239 * In time closedown will finish, we empty the write queue and 2240 * all will be happy. 2241 */ 2242 if (unlikely(sk->sk_state == TCP_CLOSE)) 2243 return; 2244 2245 if (tcp_write_xmit(sk, cur_mss, nonagle, 0, 2246 sk_gfp_atomic(sk, GFP_ATOMIC))) 2247 tcp_check_probe_timer(sk); 2248 } 2249 2250 /* Send _single_ skb sitting at the send head. This function requires 2251 * true push pending frames to setup probe timer etc. 2252 */ 2253 void tcp_push_one(struct sock *sk, unsigned int mss_now) 2254 { 2255 struct sk_buff *skb = tcp_send_head(sk); 2256 2257 BUG_ON(!skb || skb->len < mss_now); 2258 2259 tcp_write_xmit(sk, mss_now, TCP_NAGLE_PUSH, 1, sk->sk_allocation); 2260 } 2261 2262 /* This function returns the amount that we can raise the 2263 * usable window based on the following constraints 2264 * 2265 * 1. The window can never be shrunk once it is offered (RFC 793) 2266 * 2. We limit memory per socket 2267 * 2268 * RFC 1122: 2269 * "the suggested [SWS] avoidance algorithm for the receiver is to keep 2270 * RECV.NEXT + RCV.WIN fixed until: 2271 * RCV.BUFF - RCV.USER - RCV.WINDOW >= min(1/2 RCV.BUFF, MSS)" 2272 * 2273 * i.e. don't raise the right edge of the window until you can raise 2274 * it at least MSS bytes. 2275 * 2276 * Unfortunately, the recommended algorithm breaks header prediction, 2277 * since header prediction assumes th->window stays fixed. 2278 * 2279 * Strictly speaking, keeping th->window fixed violates the receiver 2280 * side SWS prevention criteria. The problem is that under this rule 2281 * a stream of single byte packets will cause the right side of the 2282 * window to always advance by a single byte. 2283 * 2284 * Of course, if the sender implements sender side SWS prevention 2285 * then this will not be a problem. 2286 * 2287 * BSD seems to make the following compromise: 2288 * 2289 * If the free space is less than the 1/4 of the maximum 2290 * space available and the free space is less than 1/2 mss, 2291 * then set the window to 0. 2292 * [ Actually, bsd uses MSS and 1/4 of maximal _window_ ] 2293 * Otherwise, just prevent the window from shrinking 2294 * and from being larger than the largest representable value. 2295 * 2296 * This prevents incremental opening of the window in the regime 2297 * where TCP is limited by the speed of the reader side taking 2298 * data out of the TCP receive queue. It does nothing about 2299 * those cases where the window is constrained on the sender side 2300 * because the pipeline is full. 2301 * 2302 * BSD also seems to "accidentally" limit itself to windows that are a 2303 * multiple of MSS, at least until the free space gets quite small. 2304 * This would appear to be a side effect of the mbuf implementation. 2305 * Combining these two algorithms results in the observed behavior 2306 * of having a fixed window size at almost all times. 2307 * 2308 * Below we obtain similar behavior by forcing the offered window to 2309 * a multiple of the mss when it is feasible to do so. 2310 * 2311 * Note, we don't "adjust" for TIMESTAMP or SACK option bytes. 2312 * Regular options like TIMESTAMP are taken into account. 2313 */ 2314 u32 __tcp_select_window(struct sock *sk) 2315 { 2316 struct inet_connection_sock *icsk = inet_csk(sk); 2317 struct tcp_sock *tp = tcp_sk(sk); 2318 /* MSS for the peer's data. Previous versions used mss_clamp 2319 * here. I don't know if the value based on our guesses 2320 * of peer's MSS is better for the performance. It's more correct 2321 * but may be worse for the performance because of rcv_mss 2322 * fluctuations. --SAW 1998/11/1 2323 */ 2324 int mss = icsk->icsk_ack.rcv_mss; 2325 int free_space = tcp_space(sk); 2326 int allowed_space = tcp_full_space(sk); 2327 int full_space = min_t(int, tp->window_clamp, allowed_space); 2328 int window; 2329 2330 if (mss > full_space) 2331 mss = full_space; 2332 2333 if (free_space < (full_space >> 1)) { 2334 icsk->icsk_ack.quick = 0; 2335 2336 if (sk_under_memory_pressure(sk)) 2337 tp->rcv_ssthresh = min(tp->rcv_ssthresh, 2338 4U * tp->advmss); 2339 2340 /* free_space might become our new window, make sure we don't 2341 * increase it due to wscale. 2342 */ 2343 free_space = round_down(free_space, 1 << tp->rx_opt.rcv_wscale); 2344 2345 /* if free space is less than mss estimate, or is below 1/16th 2346 * of the maximum allowed, try to move to zero-window, else 2347 * tcp_clamp_window() will grow rcv buf up to tcp_rmem[2], and 2348 * new incoming data is dropped due to memory limits. 2349 * With large window, mss test triggers way too late in order 2350 * to announce zero window in time before rmem limit kicks in. 2351 */ 2352 if (free_space < (allowed_space >> 4) || free_space < mss) 2353 return 0; 2354 } 2355 2356 if (free_space > tp->rcv_ssthresh) 2357 free_space = tp->rcv_ssthresh; 2358 2359 /* Don't do rounding if we are using window scaling, since the 2360 * scaled window will not line up with the MSS boundary anyway. 2361 */ 2362 window = tp->rcv_wnd; 2363 if (tp->rx_opt.rcv_wscale) { 2364 window = free_space; 2365 2366 /* Advertise enough space so that it won't get scaled away. 2367 * Import case: prevent zero window announcement if 2368 * 1<<rcv_wscale > mss. 2369 */ 2370 if (((window >> tp->rx_opt.rcv_wscale) << tp->rx_opt.rcv_wscale) != window) 2371 window = (((window >> tp->rx_opt.rcv_wscale) + 1) 2372 << tp->rx_opt.rcv_wscale); 2373 } else { 2374 /* Get the largest window that is a nice multiple of mss. 2375 * Window clamp already applied above. 2376 * If our current window offering is within 1 mss of the 2377 * free space we just keep it. This prevents the divide 2378 * and multiply from happening most of the time. 2379 * We also don't do any window rounding when the free space 2380 * is too small. 2381 */ 2382 if (window <= free_space - mss || window > free_space) 2383 window = (free_space / mss) * mss; 2384 else if (mss == full_space && 2385 free_space > window + (full_space >> 1)) 2386 window = free_space; 2387 } 2388 2389 return window; 2390 } 2391 2392 /* Collapses two adjacent SKB's during retransmission. */ 2393 static void tcp_collapse_retrans(struct sock *sk, struct sk_buff *skb) 2394 { 2395 struct tcp_sock *tp = tcp_sk(sk); 2396 struct sk_buff *next_skb = tcp_write_queue_next(sk, skb); 2397 int skb_size, next_skb_size; 2398 2399 skb_size = skb->len; 2400 next_skb_size = next_skb->len; 2401 2402 BUG_ON(tcp_skb_pcount(skb) != 1 || tcp_skb_pcount(next_skb) != 1); 2403 2404 tcp_highest_sack_combine(sk, next_skb, skb); 2405 2406 tcp_unlink_write_queue(next_skb, sk); 2407 2408 skb_copy_from_linear_data(next_skb, skb_put(skb, next_skb_size), 2409 next_skb_size); 2410 2411 if (next_skb->ip_summed == CHECKSUM_PARTIAL) 2412 skb->ip_summed = CHECKSUM_PARTIAL; 2413 2414 if (skb->ip_summed != CHECKSUM_PARTIAL) 2415 skb->csum = csum_block_add(skb->csum, next_skb->csum, skb_size); 2416 2417 /* Update sequence range on original skb. */ 2418 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(next_skb)->end_seq; 2419 2420 /* Merge over control information. This moves PSH/FIN etc. over */ 2421 TCP_SKB_CB(skb)->tcp_flags |= TCP_SKB_CB(next_skb)->tcp_flags; 2422 2423 /* All done, get rid of second SKB and account for it so 2424 * packet counting does not break. 2425 */ 2426 TCP_SKB_CB(skb)->sacked |= TCP_SKB_CB(next_skb)->sacked & TCPCB_EVER_RETRANS; 2427 2428 /* changed transmit queue under us so clear hints */ 2429 tcp_clear_retrans_hints_partial(tp); 2430 if (next_skb == tp->retransmit_skb_hint) 2431 tp->retransmit_skb_hint = skb; 2432 2433 tcp_adjust_pcount(sk, next_skb, tcp_skb_pcount(next_skb)); 2434 2435 sk_wmem_free_skb(sk, next_skb); 2436 } 2437 2438 /* Check if coalescing SKBs is legal. */ 2439 static bool tcp_can_collapse(const struct sock *sk, const struct sk_buff *skb) 2440 { 2441 if (tcp_skb_pcount(skb) > 1) 2442 return false; 2443 /* TODO: SACK collapsing could be used to remove this condition */ 2444 if (skb_shinfo(skb)->nr_frags != 0) 2445 return false; 2446 if (skb_cloned(skb)) 2447 return false; 2448 if (skb == tcp_send_head(sk)) 2449 return false; 2450 /* Some heurestics for collapsing over SACK'd could be invented */ 2451 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED) 2452 return false; 2453 2454 return true; 2455 } 2456 2457 /* Collapse packets in the retransmit queue to make to create 2458 * less packets on the wire. This is only done on retransmission. 2459 */ 2460 static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *to, 2461 int space) 2462 { 2463 struct tcp_sock *tp = tcp_sk(sk); 2464 struct sk_buff *skb = to, *tmp; 2465 bool first = true; 2466 2467 if (!sysctl_tcp_retrans_collapse) 2468 return; 2469 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN) 2470 return; 2471 2472 tcp_for_write_queue_from_safe(skb, tmp, sk) { 2473 if (!tcp_can_collapse(sk, skb)) 2474 break; 2475 2476 space -= skb->len; 2477 2478 if (first) { 2479 first = false; 2480 continue; 2481 } 2482 2483 if (space < 0) 2484 break; 2485 /* Punt if not enough space exists in the first SKB for 2486 * the data in the second 2487 */ 2488 if (skb->len > skb_availroom(to)) 2489 break; 2490 2491 if (after(TCP_SKB_CB(skb)->end_seq, tcp_wnd_end(tp))) 2492 break; 2493 2494 tcp_collapse_retrans(sk, to); 2495 } 2496 } 2497 2498 /* This retransmits one SKB. Policy decisions and retransmit queue 2499 * state updates are done by the caller. Returns non-zero if an 2500 * error occurred which prevented the send. 2501 */ 2502 int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb) 2503 { 2504 struct tcp_sock *tp = tcp_sk(sk); 2505 struct inet_connection_sock *icsk = inet_csk(sk); 2506 unsigned int cur_mss; 2507 int err; 2508 2509 /* Inconslusive MTU probe */ 2510 if (icsk->icsk_mtup.probe_size) { 2511 icsk->icsk_mtup.probe_size = 0; 2512 } 2513 2514 /* Do not sent more than we queued. 1/4 is reserved for possible 2515 * copying overhead: fragmentation, tunneling, mangling etc. 2516 */ 2517 if (atomic_read(&sk->sk_wmem_alloc) > 2518 min(sk->sk_wmem_queued + (sk->sk_wmem_queued >> 2), sk->sk_sndbuf)) 2519 return -EAGAIN; 2520 2521 if (skb_still_in_host_queue(sk, skb)) 2522 return -EBUSY; 2523 2524 if (before(TCP_SKB_CB(skb)->seq, tp->snd_una)) { 2525 if (before(TCP_SKB_CB(skb)->end_seq, tp->snd_una)) 2526 BUG(); 2527 if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq)) 2528 return -ENOMEM; 2529 } 2530 2531 if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk)) 2532 return -EHOSTUNREACH; /* Routing failure or similar. */ 2533 2534 cur_mss = tcp_current_mss(sk); 2535 2536 /* If receiver has shrunk his window, and skb is out of 2537 * new window, do not retransmit it. The exception is the 2538 * case, when window is shrunk to zero. In this case 2539 * our retransmit serves as a zero window probe. 2540 */ 2541 if (!before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp)) && 2542 TCP_SKB_CB(skb)->seq != tp->snd_una) 2543 return -EAGAIN; 2544 2545 if (skb->len > cur_mss) { 2546 if (tcp_fragment(sk, skb, cur_mss, cur_mss, GFP_ATOMIC)) 2547 return -ENOMEM; /* We'll try again later. */ 2548 } else { 2549 int oldpcount = tcp_skb_pcount(skb); 2550 2551 if (unlikely(oldpcount > 1)) { 2552 if (skb_unclone(skb, GFP_ATOMIC)) 2553 return -ENOMEM; 2554 tcp_init_tso_segs(sk, skb, cur_mss); 2555 tcp_adjust_pcount(sk, skb, oldpcount - tcp_skb_pcount(skb)); 2556 } 2557 } 2558 2559 tcp_retrans_try_collapse(sk, skb, cur_mss); 2560 2561 /* Make a copy, if the first transmission SKB clone we made 2562 * is still in somebody's hands, else make a clone. 2563 */ 2564 2565 /* make sure skb->data is aligned on arches that require it 2566 * and check if ack-trimming & collapsing extended the headroom 2567 * beyond what csum_start can cover. 2568 */ 2569 if (unlikely((NET_IP_ALIGN && ((unsigned long)skb->data & 3)) || 2570 skb_headroom(skb) >= 0xFFFF)) { 2571 struct sk_buff *nskb = __pskb_copy(skb, MAX_TCP_HEADER, 2572 GFP_ATOMIC); 2573 err = nskb ? tcp_transmit_skb(sk, nskb, 0, GFP_ATOMIC) : 2574 -ENOBUFS; 2575 } else { 2576 err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC); 2577 } 2578 2579 if (likely(!err)) { 2580 TCP_SKB_CB(skb)->sacked |= TCPCB_EVER_RETRANS; 2581 /* Update global TCP statistics. */ 2582 TCP_INC_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS); 2583 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN) 2584 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPSYNRETRANS); 2585 tp->total_retrans++; 2586 } 2587 return err; 2588 } 2589 2590 int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb) 2591 { 2592 struct tcp_sock *tp = tcp_sk(sk); 2593 int err = __tcp_retransmit_skb(sk, skb); 2594 2595 if (err == 0) { 2596 #if FASTRETRANS_DEBUG > 0 2597 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) { 2598 net_dbg_ratelimited("retrans_out leaked\n"); 2599 } 2600 #endif 2601 if (!tp->retrans_out) 2602 tp->lost_retrans_low = tp->snd_nxt; 2603 TCP_SKB_CB(skb)->sacked |= TCPCB_RETRANS; 2604 tp->retrans_out += tcp_skb_pcount(skb); 2605 2606 /* Save stamp of the first retransmit. */ 2607 if (!tp->retrans_stamp) 2608 tp->retrans_stamp = tcp_skb_timestamp(skb); 2609 2610 /* snd_nxt is stored to detect loss of retransmitted segment, 2611 * see tcp_input.c tcp_sacktag_write_queue(). 2612 */ 2613 TCP_SKB_CB(skb)->ack_seq = tp->snd_nxt; 2614 } else if (err != -EBUSY) { 2615 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPRETRANSFAIL); 2616 } 2617 2618 if (tp->undo_retrans < 0) 2619 tp->undo_retrans = 0; 2620 tp->undo_retrans += tcp_skb_pcount(skb); 2621 return err; 2622 } 2623 2624 /* Check if we forward retransmits are possible in the current 2625 * window/congestion state. 2626 */ 2627 static bool tcp_can_forward_retransmit(struct sock *sk) 2628 { 2629 const struct inet_connection_sock *icsk = inet_csk(sk); 2630 const struct tcp_sock *tp = tcp_sk(sk); 2631 2632 /* Forward retransmissions are possible only during Recovery. */ 2633 if (icsk->icsk_ca_state != TCP_CA_Recovery) 2634 return false; 2635 2636 /* No forward retransmissions in Reno are possible. */ 2637 if (tcp_is_reno(tp)) 2638 return false; 2639 2640 /* Yeah, we have to make difficult choice between forward transmission 2641 * and retransmission... Both ways have their merits... 2642 * 2643 * For now we do not retransmit anything, while we have some new 2644 * segments to send. In the other cases, follow rule 3 for 2645 * NextSeg() specified in RFC3517. 2646 */ 2647 2648 if (tcp_may_send_now(sk)) 2649 return false; 2650 2651 return true; 2652 } 2653 2654 /* This gets called after a retransmit timeout, and the initially 2655 * retransmitted data is acknowledged. It tries to continue 2656 * resending the rest of the retransmit queue, until either 2657 * we've sent it all or the congestion window limit is reached. 2658 * If doing SACK, the first ACK which comes back for a timeout 2659 * based retransmit packet might feed us FACK information again. 2660 * If so, we use it to avoid unnecessarily retransmissions. 2661 */ 2662 void tcp_xmit_retransmit_queue(struct sock *sk) 2663 { 2664 const struct inet_connection_sock *icsk = inet_csk(sk); 2665 struct tcp_sock *tp = tcp_sk(sk); 2666 struct sk_buff *skb; 2667 struct sk_buff *hole = NULL; 2668 u32 last_lost; 2669 int mib_idx; 2670 int fwd_rexmitting = 0; 2671 2672 if (!tp->packets_out) 2673 return; 2674 2675 if (!tp->lost_out) 2676 tp->retransmit_high = tp->snd_una; 2677 2678 if (tp->retransmit_skb_hint) { 2679 skb = tp->retransmit_skb_hint; 2680 last_lost = TCP_SKB_CB(skb)->end_seq; 2681 if (after(last_lost, tp->retransmit_high)) 2682 last_lost = tp->retransmit_high; 2683 } else { 2684 skb = tcp_write_queue_head(sk); 2685 last_lost = tp->snd_una; 2686 } 2687 2688 tcp_for_write_queue_from(skb, sk) { 2689 __u8 sacked = TCP_SKB_CB(skb)->sacked; 2690 2691 if (skb == tcp_send_head(sk)) 2692 break; 2693 /* we could do better than to assign each time */ 2694 if (hole == NULL) 2695 tp->retransmit_skb_hint = skb; 2696 2697 /* Assume this retransmit will generate 2698 * only one packet for congestion window 2699 * calculation purposes. This works because 2700 * tcp_retransmit_skb() will chop up the 2701 * packet to be MSS sized and all the 2702 * packet counting works out. 2703 */ 2704 if (tcp_packets_in_flight(tp) >= tp->snd_cwnd) 2705 return; 2706 2707 if (fwd_rexmitting) { 2708 begin_fwd: 2709 if (!before(TCP_SKB_CB(skb)->seq, tcp_highest_sack_seq(tp))) 2710 break; 2711 mib_idx = LINUX_MIB_TCPFORWARDRETRANS; 2712 2713 } else if (!before(TCP_SKB_CB(skb)->seq, tp->retransmit_high)) { 2714 tp->retransmit_high = last_lost; 2715 if (!tcp_can_forward_retransmit(sk)) 2716 break; 2717 /* Backtrack if necessary to non-L'ed skb */ 2718 if (hole != NULL) { 2719 skb = hole; 2720 hole = NULL; 2721 } 2722 fwd_rexmitting = 1; 2723 goto begin_fwd; 2724 2725 } else if (!(sacked & TCPCB_LOST)) { 2726 if (hole == NULL && !(sacked & (TCPCB_SACKED_RETRANS|TCPCB_SACKED_ACKED))) 2727 hole = skb; 2728 continue; 2729 2730 } else { 2731 last_lost = TCP_SKB_CB(skb)->end_seq; 2732 if (icsk->icsk_ca_state != TCP_CA_Loss) 2733 mib_idx = LINUX_MIB_TCPFASTRETRANS; 2734 else 2735 mib_idx = LINUX_MIB_TCPSLOWSTARTRETRANS; 2736 } 2737 2738 if (sacked & (TCPCB_SACKED_ACKED|TCPCB_SACKED_RETRANS)) 2739 continue; 2740 2741 if (tcp_retransmit_skb(sk, skb)) 2742 return; 2743 2744 NET_INC_STATS_BH(sock_net(sk), mib_idx); 2745 2746 if (tcp_in_cwnd_reduction(sk)) 2747 tp->prr_out += tcp_skb_pcount(skb); 2748 2749 if (skb == tcp_write_queue_head(sk)) 2750 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, 2751 inet_csk(sk)->icsk_rto, 2752 TCP_RTO_MAX); 2753 } 2754 } 2755 2756 /* Send a fin. The caller locks the socket for us. This cannot be 2757 * allowed to fail queueing a FIN frame under any circumstances. 2758 */ 2759 void tcp_send_fin(struct sock *sk) 2760 { 2761 struct tcp_sock *tp = tcp_sk(sk); 2762 struct sk_buff *skb = tcp_write_queue_tail(sk); 2763 int mss_now; 2764 2765 /* Optimization, tack on the FIN if we have a queue of 2766 * unsent frames. But be careful about outgoing SACKS 2767 * and IP options. 2768 */ 2769 mss_now = tcp_current_mss(sk); 2770 2771 if (tcp_send_head(sk) != NULL) { 2772 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_FIN; 2773 TCP_SKB_CB(skb)->end_seq++; 2774 tp->write_seq++; 2775 } else { 2776 /* Socket is locked, keep trying until memory is available. */ 2777 for (;;) { 2778 skb = alloc_skb_fclone(MAX_TCP_HEADER, 2779 sk->sk_allocation); 2780 if (skb) 2781 break; 2782 yield(); 2783 } 2784 2785 /* Reserve space for headers and prepare control bits. */ 2786 skb_reserve(skb, MAX_TCP_HEADER); 2787 /* FIN eats a sequence byte, write_seq advanced by tcp_queue_skb(). */ 2788 tcp_init_nondata_skb(skb, tp->write_seq, 2789 TCPHDR_ACK | TCPHDR_FIN); 2790 tcp_queue_skb(sk, skb); 2791 } 2792 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_OFF); 2793 } 2794 2795 /* We get here when a process closes a file descriptor (either due to 2796 * an explicit close() or as a byproduct of exit()'ing) and there 2797 * was unread data in the receive queue. This behavior is recommended 2798 * by RFC 2525, section 2.17. -DaveM 2799 */ 2800 void tcp_send_active_reset(struct sock *sk, gfp_t priority) 2801 { 2802 struct sk_buff *skb; 2803 2804 /* NOTE: No TCP options attached and we never retransmit this. */ 2805 skb = alloc_skb(MAX_TCP_HEADER, priority); 2806 if (!skb) { 2807 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED); 2808 return; 2809 } 2810 2811 /* Reserve space for headers and prepare control bits. */ 2812 skb_reserve(skb, MAX_TCP_HEADER); 2813 tcp_init_nondata_skb(skb, tcp_acceptable_seq(sk), 2814 TCPHDR_ACK | TCPHDR_RST); 2815 /* Send it off. */ 2816 if (tcp_transmit_skb(sk, skb, 0, priority)) 2817 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED); 2818 2819 TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTRSTS); 2820 } 2821 2822 /* Send a crossed SYN-ACK during socket establishment. 2823 * WARNING: This routine must only be called when we have already sent 2824 * a SYN packet that crossed the incoming SYN that caused this routine 2825 * to get called. If this assumption fails then the initial rcv_wnd 2826 * and rcv_wscale values will not be correct. 2827 */ 2828 int tcp_send_synack(struct sock *sk) 2829 { 2830 struct sk_buff *skb; 2831 2832 skb = tcp_write_queue_head(sk); 2833 if (skb == NULL || !(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) { 2834 pr_debug("%s: wrong queue state\n", __func__); 2835 return -EFAULT; 2836 } 2837 if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_ACK)) { 2838 if (skb_cloned(skb)) { 2839 struct sk_buff *nskb = skb_copy(skb, GFP_ATOMIC); 2840 if (nskb == NULL) 2841 return -ENOMEM; 2842 tcp_unlink_write_queue(skb, sk); 2843 __skb_header_release(nskb); 2844 __tcp_add_write_queue_head(sk, nskb); 2845 sk_wmem_free_skb(sk, skb); 2846 sk->sk_wmem_queued += nskb->truesize; 2847 sk_mem_charge(sk, nskb->truesize); 2848 skb = nskb; 2849 } 2850 2851 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ACK; 2852 tcp_ecn_send_synack(sk, skb); 2853 } 2854 return tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC); 2855 } 2856 2857 /** 2858 * tcp_make_synack - Prepare a SYN-ACK. 2859 * sk: listener socket 2860 * dst: dst entry attached to the SYNACK 2861 * req: request_sock pointer 2862 * 2863 * Allocate one skb and build a SYNACK packet. 2864 * @dst is consumed : Caller should not use it again. 2865 */ 2866 struct sk_buff *tcp_make_synack(struct sock *sk, struct dst_entry *dst, 2867 struct request_sock *req, 2868 struct tcp_fastopen_cookie *foc) 2869 { 2870 struct tcp_out_options opts; 2871 struct inet_request_sock *ireq = inet_rsk(req); 2872 struct tcp_sock *tp = tcp_sk(sk); 2873 struct tcphdr *th; 2874 struct sk_buff *skb; 2875 struct tcp_md5sig_key *md5; 2876 int tcp_header_size; 2877 int mss; 2878 2879 skb = sock_wmalloc(sk, MAX_TCP_HEADER, 1, GFP_ATOMIC); 2880 if (unlikely(!skb)) { 2881 dst_release(dst); 2882 return NULL; 2883 } 2884 /* Reserve space for headers. */ 2885 skb_reserve(skb, MAX_TCP_HEADER); 2886 2887 skb_dst_set(skb, dst); 2888 security_skb_owned_by(skb, sk); 2889 2890 mss = dst_metric_advmss(dst); 2891 if (tp->rx_opt.user_mss && tp->rx_opt.user_mss < mss) 2892 mss = tp->rx_opt.user_mss; 2893 2894 memset(&opts, 0, sizeof(opts)); 2895 #ifdef CONFIG_SYN_COOKIES 2896 if (unlikely(req->cookie_ts)) 2897 skb->skb_mstamp.stamp_jiffies = cookie_init_timestamp(req); 2898 else 2899 #endif 2900 skb_mstamp_get(&skb->skb_mstamp); 2901 tcp_header_size = tcp_synack_options(sk, req, mss, skb, &opts, &md5, 2902 foc) + sizeof(*th); 2903 2904 skb_push(skb, tcp_header_size); 2905 skb_reset_transport_header(skb); 2906 2907 th = tcp_hdr(skb); 2908 memset(th, 0, sizeof(struct tcphdr)); 2909 th->syn = 1; 2910 th->ack = 1; 2911 tcp_ecn_make_synack(req, th, sk); 2912 th->source = htons(ireq->ir_num); 2913 th->dest = ireq->ir_rmt_port; 2914 /* Setting of flags are superfluous here for callers (and ECE is 2915 * not even correctly set) 2916 */ 2917 tcp_init_nondata_skb(skb, tcp_rsk(req)->snt_isn, 2918 TCPHDR_SYN | TCPHDR_ACK); 2919 2920 th->seq = htonl(TCP_SKB_CB(skb)->seq); 2921 /* XXX data is queued and acked as is. No buffer/window check */ 2922 th->ack_seq = htonl(tcp_rsk(req)->rcv_nxt); 2923 2924 /* RFC1323: The window in SYN & SYN/ACK segments is never scaled. */ 2925 th->window = htons(min(req->rcv_wnd, 65535U)); 2926 tcp_options_write((__be32 *)(th + 1), tp, &opts); 2927 th->doff = (tcp_header_size >> 2); 2928 TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_OUTSEGS); 2929 2930 #ifdef CONFIG_TCP_MD5SIG 2931 /* Okay, we have all we need - do the md5 hash if needed */ 2932 if (md5) { 2933 tcp_rsk(req)->af_specific->calc_md5_hash(opts.hash_location, 2934 md5, NULL, req, skb); 2935 } 2936 #endif 2937 2938 return skb; 2939 } 2940 EXPORT_SYMBOL(tcp_make_synack); 2941 2942 /* Do all connect socket setups that can be done AF independent. */ 2943 static void tcp_connect_init(struct sock *sk) 2944 { 2945 const struct dst_entry *dst = __sk_dst_get(sk); 2946 struct tcp_sock *tp = tcp_sk(sk); 2947 __u8 rcv_wscale; 2948 2949 /* We'll fix this up when we get a response from the other end. 2950 * See tcp_input.c:tcp_rcv_state_process case TCP_SYN_SENT. 2951 */ 2952 tp->tcp_header_len = sizeof(struct tcphdr) + 2953 (sysctl_tcp_timestamps ? TCPOLEN_TSTAMP_ALIGNED : 0); 2954 2955 #ifdef CONFIG_TCP_MD5SIG 2956 if (tp->af_specific->md5_lookup(sk, sk) != NULL) 2957 tp->tcp_header_len += TCPOLEN_MD5SIG_ALIGNED; 2958 #endif 2959 2960 /* If user gave his TCP_MAXSEG, record it to clamp */ 2961 if (tp->rx_opt.user_mss) 2962 tp->rx_opt.mss_clamp = tp->rx_opt.user_mss; 2963 tp->max_window = 0; 2964 tcp_mtup_init(sk); 2965 tcp_sync_mss(sk, dst_mtu(dst)); 2966 2967 if (!tp->window_clamp) 2968 tp->window_clamp = dst_metric(dst, RTAX_WINDOW); 2969 tp->advmss = dst_metric_advmss(dst); 2970 if (tp->rx_opt.user_mss && tp->rx_opt.user_mss < tp->advmss) 2971 tp->advmss = tp->rx_opt.user_mss; 2972 2973 tcp_initialize_rcv_mss(sk); 2974 2975 /* limit the window selection if the user enforce a smaller rx buffer */ 2976 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK && 2977 (tp->window_clamp > tcp_full_space(sk) || tp->window_clamp == 0)) 2978 tp->window_clamp = tcp_full_space(sk); 2979 2980 tcp_select_initial_window(tcp_full_space(sk), 2981 tp->advmss - (tp->rx_opt.ts_recent_stamp ? tp->tcp_header_len - sizeof(struct tcphdr) : 0), 2982 &tp->rcv_wnd, 2983 &tp->window_clamp, 2984 sysctl_tcp_window_scaling, 2985 &rcv_wscale, 2986 dst_metric(dst, RTAX_INITRWND)); 2987 2988 tp->rx_opt.rcv_wscale = rcv_wscale; 2989 tp->rcv_ssthresh = tp->rcv_wnd; 2990 2991 sk->sk_err = 0; 2992 sock_reset_flag(sk, SOCK_DONE); 2993 tp->snd_wnd = 0; 2994 tcp_init_wl(tp, 0); 2995 tp->snd_una = tp->write_seq; 2996 tp->snd_sml = tp->write_seq; 2997 tp->snd_up = tp->write_seq; 2998 tp->snd_nxt = tp->write_seq; 2999 3000 if (likely(!tp->repair)) 3001 tp->rcv_nxt = 0; 3002 else 3003 tp->rcv_tstamp = tcp_time_stamp; 3004 tp->rcv_wup = tp->rcv_nxt; 3005 tp->copied_seq = tp->rcv_nxt; 3006 3007 inet_csk(sk)->icsk_rto = TCP_TIMEOUT_INIT; 3008 inet_csk(sk)->icsk_retransmits = 0; 3009 tcp_clear_retrans(tp); 3010 } 3011 3012 static void tcp_connect_queue_skb(struct sock *sk, struct sk_buff *skb) 3013 { 3014 struct tcp_sock *tp = tcp_sk(sk); 3015 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb); 3016 3017 tcb->end_seq += skb->len; 3018 __skb_header_release(skb); 3019 __tcp_add_write_queue_tail(sk, skb); 3020 sk->sk_wmem_queued += skb->truesize; 3021 sk_mem_charge(sk, skb->truesize); 3022 tp->write_seq = tcb->end_seq; 3023 tp->packets_out += tcp_skb_pcount(skb); 3024 } 3025 3026 /* Build and send a SYN with data and (cached) Fast Open cookie. However, 3027 * queue a data-only packet after the regular SYN, such that regular SYNs 3028 * are retransmitted on timeouts. Also if the remote SYN-ACK acknowledges 3029 * only the SYN sequence, the data are retransmitted in the first ACK. 3030 * If cookie is not cached or other error occurs, falls back to send a 3031 * regular SYN with Fast Open cookie request option. 3032 */ 3033 static int tcp_send_syn_data(struct sock *sk, struct sk_buff *syn) 3034 { 3035 struct tcp_sock *tp = tcp_sk(sk); 3036 struct tcp_fastopen_request *fo = tp->fastopen_req; 3037 int syn_loss = 0, space, err = 0; 3038 unsigned long last_syn_loss = 0; 3039 struct sk_buff *syn_data; 3040 3041 tp->rx_opt.mss_clamp = tp->advmss; /* If MSS is not cached */ 3042 tcp_fastopen_cache_get(sk, &tp->rx_opt.mss_clamp, &fo->cookie, 3043 &syn_loss, &last_syn_loss); 3044 /* Recurring FO SYN losses: revert to regular handshake temporarily */ 3045 if (syn_loss > 1 && 3046 time_before(jiffies, last_syn_loss + (60*HZ << syn_loss))) { 3047 fo->cookie.len = -1; 3048 goto fallback; 3049 } 3050 3051 if (sysctl_tcp_fastopen & TFO_CLIENT_NO_COOKIE) 3052 fo->cookie.len = -1; 3053 else if (fo->cookie.len <= 0) 3054 goto fallback; 3055 3056 /* MSS for SYN-data is based on cached MSS and bounded by PMTU and 3057 * user-MSS. Reserve maximum option space for middleboxes that add 3058 * private TCP options. The cost is reduced data space in SYN :( 3059 */ 3060 if (tp->rx_opt.user_mss && tp->rx_opt.user_mss < tp->rx_opt.mss_clamp) 3061 tp->rx_opt.mss_clamp = tp->rx_opt.user_mss; 3062 space = __tcp_mtu_to_mss(sk, inet_csk(sk)->icsk_pmtu_cookie) - 3063 MAX_TCP_OPTION_SPACE; 3064 3065 space = min_t(size_t, space, fo->size); 3066 3067 /* limit to order-0 allocations */ 3068 space = min_t(size_t, space, SKB_MAX_HEAD(MAX_TCP_HEADER)); 3069 3070 syn_data = sk_stream_alloc_skb(sk, space, sk->sk_allocation); 3071 if (!syn_data) 3072 goto fallback; 3073 syn_data->ip_summed = CHECKSUM_PARTIAL; 3074 memcpy(syn_data->cb, syn->cb, sizeof(syn->cb)); 3075 if (unlikely(memcpy_fromiovecend(skb_put(syn_data, space), 3076 fo->data->msg_iter.iov, 0, space))) { 3077 kfree_skb(syn_data); 3078 goto fallback; 3079 } 3080 3081 /* No more data pending in inet_wait_for_connect() */ 3082 if (space == fo->size) 3083 fo->data = NULL; 3084 fo->copied = space; 3085 3086 tcp_connect_queue_skb(sk, syn_data); 3087 3088 err = tcp_transmit_skb(sk, syn_data, 1, sk->sk_allocation); 3089 3090 syn->skb_mstamp = syn_data->skb_mstamp; 3091 3092 /* Now full SYN+DATA was cloned and sent (or not), 3093 * remove the SYN from the original skb (syn_data) 3094 * we keep in write queue in case of a retransmit, as we 3095 * also have the SYN packet (with no data) in the same queue. 3096 */ 3097 TCP_SKB_CB(syn_data)->seq++; 3098 TCP_SKB_CB(syn_data)->tcp_flags = TCPHDR_ACK | TCPHDR_PSH; 3099 if (!err) { 3100 tp->syn_data = (fo->copied > 0); 3101 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT); 3102 goto done; 3103 } 3104 3105 fallback: 3106 /* Send a regular SYN with Fast Open cookie request option */ 3107 if (fo->cookie.len > 0) 3108 fo->cookie.len = 0; 3109 err = tcp_transmit_skb(sk, syn, 1, sk->sk_allocation); 3110 if (err) 3111 tp->syn_fastopen = 0; 3112 done: 3113 fo->cookie.len = -1; /* Exclude Fast Open option for SYN retries */ 3114 return err; 3115 } 3116 3117 /* Build a SYN and send it off. */ 3118 int tcp_connect(struct sock *sk) 3119 { 3120 struct tcp_sock *tp = tcp_sk(sk); 3121 struct sk_buff *buff; 3122 int err; 3123 3124 tcp_connect_init(sk); 3125 3126 if (unlikely(tp->repair)) { 3127 tcp_finish_connect(sk, NULL); 3128 return 0; 3129 } 3130 3131 buff = sk_stream_alloc_skb(sk, 0, sk->sk_allocation); 3132 if (unlikely(!buff)) 3133 return -ENOBUFS; 3134 3135 tcp_init_nondata_skb(buff, tp->write_seq++, TCPHDR_SYN); 3136 tp->retrans_stamp = tcp_time_stamp; 3137 tcp_connect_queue_skb(sk, buff); 3138 tcp_ecn_send_syn(sk, buff); 3139 3140 /* Send off SYN; include data in Fast Open. */ 3141 err = tp->fastopen_req ? tcp_send_syn_data(sk, buff) : 3142 tcp_transmit_skb(sk, buff, 1, sk->sk_allocation); 3143 if (err == -ECONNREFUSED) 3144 return err; 3145 3146 /* We change tp->snd_nxt after the tcp_transmit_skb() call 3147 * in order to make this packet get counted in tcpOutSegs. 3148 */ 3149 tp->snd_nxt = tp->write_seq; 3150 tp->pushed_seq = tp->write_seq; 3151 TCP_INC_STATS(sock_net(sk), TCP_MIB_ACTIVEOPENS); 3152 3153 /* Timer for repeating the SYN until an answer. */ 3154 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, 3155 inet_csk(sk)->icsk_rto, TCP_RTO_MAX); 3156 return 0; 3157 } 3158 EXPORT_SYMBOL(tcp_connect); 3159 3160 /* Send out a delayed ack, the caller does the policy checking 3161 * to see if we should even be here. See tcp_input.c:tcp_ack_snd_check() 3162 * for details. 3163 */ 3164 void tcp_send_delayed_ack(struct sock *sk) 3165 { 3166 struct inet_connection_sock *icsk = inet_csk(sk); 3167 int ato = icsk->icsk_ack.ato; 3168 unsigned long timeout; 3169 3170 tcp_ca_event(sk, CA_EVENT_DELAYED_ACK); 3171 3172 if (ato > TCP_DELACK_MIN) { 3173 const struct tcp_sock *tp = tcp_sk(sk); 3174 int max_ato = HZ / 2; 3175 3176 if (icsk->icsk_ack.pingpong || 3177 (icsk->icsk_ack.pending & ICSK_ACK_PUSHED)) 3178 max_ato = TCP_DELACK_MAX; 3179 3180 /* Slow path, intersegment interval is "high". */ 3181 3182 /* If some rtt estimate is known, use it to bound delayed ack. 3183 * Do not use inet_csk(sk)->icsk_rto here, use results of rtt measurements 3184 * directly. 3185 */ 3186 if (tp->srtt_us) { 3187 int rtt = max_t(int, usecs_to_jiffies(tp->srtt_us >> 3), 3188 TCP_DELACK_MIN); 3189 3190 if (rtt < max_ato) 3191 max_ato = rtt; 3192 } 3193 3194 ato = min(ato, max_ato); 3195 } 3196 3197 /* Stay within the limit we were given */ 3198 timeout = jiffies + ato; 3199 3200 /* Use new timeout only if there wasn't a older one earlier. */ 3201 if (icsk->icsk_ack.pending & ICSK_ACK_TIMER) { 3202 /* If delack timer was blocked or is about to expire, 3203 * send ACK now. 3204 */ 3205 if (icsk->icsk_ack.blocked || 3206 time_before_eq(icsk->icsk_ack.timeout, jiffies + (ato >> 2))) { 3207 tcp_send_ack(sk); 3208 return; 3209 } 3210 3211 if (!time_before(timeout, icsk->icsk_ack.timeout)) 3212 timeout = icsk->icsk_ack.timeout; 3213 } 3214 icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER; 3215 icsk->icsk_ack.timeout = timeout; 3216 sk_reset_timer(sk, &icsk->icsk_delack_timer, timeout); 3217 } 3218 3219 /* This routine sends an ack and also updates the window. */ 3220 void tcp_send_ack(struct sock *sk) 3221 { 3222 struct sk_buff *buff; 3223 3224 /* If we have been reset, we may not send again. */ 3225 if (sk->sk_state == TCP_CLOSE) 3226 return; 3227 3228 tcp_ca_event(sk, CA_EVENT_NON_DELAYED_ACK); 3229 3230 /* We are not putting this on the write queue, so 3231 * tcp_transmit_skb() will set the ownership to this 3232 * sock. 3233 */ 3234 buff = alloc_skb(MAX_TCP_HEADER, sk_gfp_atomic(sk, GFP_ATOMIC)); 3235 if (buff == NULL) { 3236 inet_csk_schedule_ack(sk); 3237 inet_csk(sk)->icsk_ack.ato = TCP_ATO_MIN; 3238 inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK, 3239 TCP_DELACK_MAX, TCP_RTO_MAX); 3240 return; 3241 } 3242 3243 /* Reserve space for headers and prepare control bits. */ 3244 skb_reserve(buff, MAX_TCP_HEADER); 3245 tcp_init_nondata_skb(buff, tcp_acceptable_seq(sk), TCPHDR_ACK); 3246 3247 /* Send it off, this clears delayed acks for us. */ 3248 skb_mstamp_get(&buff->skb_mstamp); 3249 tcp_transmit_skb(sk, buff, 0, sk_gfp_atomic(sk, GFP_ATOMIC)); 3250 } 3251 EXPORT_SYMBOL_GPL(tcp_send_ack); 3252 3253 /* This routine sends a packet with an out of date sequence 3254 * number. It assumes the other end will try to ack it. 3255 * 3256 * Question: what should we make while urgent mode? 3257 * 4.4BSD forces sending single byte of data. We cannot send 3258 * out of window data, because we have SND.NXT==SND.MAX... 3259 * 3260 * Current solution: to send TWO zero-length segments in urgent mode: 3261 * one is with SEG.SEQ=SND.UNA to deliver urgent pointer, another is 3262 * out-of-date with SND.UNA-1 to probe window. 3263 */ 3264 static int tcp_xmit_probe_skb(struct sock *sk, int urgent) 3265 { 3266 struct tcp_sock *tp = tcp_sk(sk); 3267 struct sk_buff *skb; 3268 3269 /* We don't queue it, tcp_transmit_skb() sets ownership. */ 3270 skb = alloc_skb(MAX_TCP_HEADER, sk_gfp_atomic(sk, GFP_ATOMIC)); 3271 if (skb == NULL) 3272 return -1; 3273 3274 /* Reserve space for headers and set control bits. */ 3275 skb_reserve(skb, MAX_TCP_HEADER); 3276 /* Use a previous sequence. This should cause the other 3277 * end to send an ack. Don't queue or clone SKB, just 3278 * send it. 3279 */ 3280 tcp_init_nondata_skb(skb, tp->snd_una - !urgent, TCPHDR_ACK); 3281 skb_mstamp_get(&skb->skb_mstamp); 3282 return tcp_transmit_skb(sk, skb, 0, GFP_ATOMIC); 3283 } 3284 3285 void tcp_send_window_probe(struct sock *sk) 3286 { 3287 if (sk->sk_state == TCP_ESTABLISHED) { 3288 tcp_sk(sk)->snd_wl1 = tcp_sk(sk)->rcv_nxt - 1; 3289 tcp_xmit_probe_skb(sk, 0); 3290 } 3291 } 3292 3293 /* Initiate keepalive or window probe from timer. */ 3294 int tcp_write_wakeup(struct sock *sk) 3295 { 3296 struct tcp_sock *tp = tcp_sk(sk); 3297 struct sk_buff *skb; 3298 3299 if (sk->sk_state == TCP_CLOSE) 3300 return -1; 3301 3302 if ((skb = tcp_send_head(sk)) != NULL && 3303 before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp))) { 3304 int err; 3305 unsigned int mss = tcp_current_mss(sk); 3306 unsigned int seg_size = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq; 3307 3308 if (before(tp->pushed_seq, TCP_SKB_CB(skb)->end_seq)) 3309 tp->pushed_seq = TCP_SKB_CB(skb)->end_seq; 3310 3311 /* We are probing the opening of a window 3312 * but the window size is != 0 3313 * must have been a result SWS avoidance ( sender ) 3314 */ 3315 if (seg_size < TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq || 3316 skb->len > mss) { 3317 seg_size = min(seg_size, mss); 3318 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH; 3319 if (tcp_fragment(sk, skb, seg_size, mss, GFP_ATOMIC)) 3320 return -1; 3321 } else if (!tcp_skb_pcount(skb)) 3322 tcp_set_skb_tso_segs(sk, skb, mss); 3323 3324 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH; 3325 err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC); 3326 if (!err) 3327 tcp_event_new_data_sent(sk, skb); 3328 return err; 3329 } else { 3330 if (between(tp->snd_up, tp->snd_una + 1, tp->snd_una + 0xFFFF)) 3331 tcp_xmit_probe_skb(sk, 1); 3332 return tcp_xmit_probe_skb(sk, 0); 3333 } 3334 } 3335 3336 /* A window probe timeout has occurred. If window is not closed send 3337 * a partial packet else a zero probe. 3338 */ 3339 void tcp_send_probe0(struct sock *sk) 3340 { 3341 struct inet_connection_sock *icsk = inet_csk(sk); 3342 struct tcp_sock *tp = tcp_sk(sk); 3343 unsigned long probe_max; 3344 int err; 3345 3346 err = tcp_write_wakeup(sk); 3347 3348 if (tp->packets_out || !tcp_send_head(sk)) { 3349 /* Cancel probe timer, if it is not required. */ 3350 icsk->icsk_probes_out = 0; 3351 icsk->icsk_backoff = 0; 3352 return; 3353 } 3354 3355 if (err <= 0) { 3356 if (icsk->icsk_backoff < sysctl_tcp_retries2) 3357 icsk->icsk_backoff++; 3358 icsk->icsk_probes_out++; 3359 probe_max = TCP_RTO_MAX; 3360 } else { 3361 /* If packet was not sent due to local congestion, 3362 * do not backoff and do not remember icsk_probes_out. 3363 * Let local senders to fight for local resources. 3364 * 3365 * Use accumulated backoff yet. 3366 */ 3367 if (!icsk->icsk_probes_out) 3368 icsk->icsk_probes_out = 1; 3369 probe_max = TCP_RESOURCE_PROBE_INTERVAL; 3370 } 3371 inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0, 3372 inet_csk_rto_backoff(icsk, probe_max), 3373 TCP_RTO_MAX); 3374 } 3375 3376 int tcp_rtx_synack(struct sock *sk, struct request_sock *req) 3377 { 3378 const struct tcp_request_sock_ops *af_ops = tcp_rsk(req)->af_specific; 3379 struct flowi fl; 3380 int res; 3381 3382 res = af_ops->send_synack(sk, NULL, &fl, req, 0, NULL); 3383 if (!res) { 3384 TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_RETRANSSEGS); 3385 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPSYNRETRANS); 3386 } 3387 return res; 3388 } 3389 EXPORT_SYMBOL(tcp_rtx_synack); 3390