1 /* 2 * INET An implementation of the TCP/IP protocol suite for the LINUX 3 * operating system. INET is implemented using the BSD Socket 4 * interface as the means of communication with the user level. 5 * 6 * Implementation of the Transmission Control Protocol(TCP). 7 * 8 * Authors: Ross Biro 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 10 * Mark Evans, <evansmp@uhura.aston.ac.uk> 11 * Corey Minyard <wf-rch!minyard@relay.EU.net> 12 * Florian La Roche, <flla@stud.uni-sb.de> 13 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu> 14 * Linus Torvalds, <torvalds@cs.helsinki.fi> 15 * Alan Cox, <gw4pts@gw4pts.ampr.org> 16 * Matthew Dillon, <dillon@apollo.west.oic.com> 17 * Arnt Gulbrandsen, <agulbra@nvg.unit.no> 18 * Jorge Cwik, <jorge@laser.satlink.net> 19 */ 20 21 /* 22 * Changes: Pedro Roque : Retransmit queue handled by TCP. 23 * : Fragmentation on mtu decrease 24 * : Segment collapse on retransmit 25 * : AF independence 26 * 27 * Linus Torvalds : send_delayed_ack 28 * David S. Miller : Charge memory using the right skb 29 * during syn/ack processing. 30 * David S. Miller : Output engine completely rewritten. 31 * Andrea Arcangeli: SYNACK carry ts_recent in tsecr. 32 * Cacophonix Gaul : draft-minshall-nagle-01 33 * J Hadi Salim : ECN support 34 * 35 */ 36 37 #include <net/tcp.h> 38 39 #include <linux/compiler.h> 40 #include <linux/module.h> 41 42 /* People can turn this off for buggy TCP's found in printers etc. */ 43 int sysctl_tcp_retrans_collapse __read_mostly = 1; 44 45 /* People can turn this on to work with those rare, broken TCPs that 46 * interpret the window field as a signed quantity. 47 */ 48 int sysctl_tcp_workaround_signed_windows __read_mostly = 0; 49 50 /* This limits the percentage of the congestion window which we 51 * will allow a single TSO frame to consume. Building TSO frames 52 * which are too large can cause TCP streams to be bursty. 53 */ 54 int sysctl_tcp_tso_win_divisor __read_mostly = 3; 55 56 int sysctl_tcp_mtu_probing __read_mostly = 0; 57 int sysctl_tcp_base_mss __read_mostly = 512; 58 59 /* By default, RFC2861 behavior. */ 60 int sysctl_tcp_slow_start_after_idle __read_mostly = 1; 61 62 static void tcp_event_new_data_sent(struct sock *sk, struct sk_buff *skb) 63 { 64 struct tcp_sock *tp = tcp_sk(sk); 65 unsigned int prior_packets = tp->packets_out; 66 67 tcp_advance_send_head(sk, skb); 68 tp->snd_nxt = TCP_SKB_CB(skb)->end_seq; 69 70 /* Don't override Nagle indefinately with F-RTO */ 71 if (tp->frto_counter == 2) 72 tp->frto_counter = 3; 73 74 tp->packets_out += tcp_skb_pcount(skb); 75 if (!prior_packets) 76 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, 77 inet_csk(sk)->icsk_rto, TCP_RTO_MAX); 78 } 79 80 /* SND.NXT, if window was not shrunk. 81 * If window has been shrunk, what should we make? It is not clear at all. 82 * Using SND.UNA we will fail to open window, SND.NXT is out of window. :-( 83 * Anything in between SND.UNA...SND.UNA+SND.WND also can be already 84 * invalid. OK, let's make this for now: 85 */ 86 static inline __u32 tcp_acceptable_seq(struct sock *sk) 87 { 88 struct tcp_sock *tp = tcp_sk(sk); 89 90 if (!before(tcp_wnd_end(tp), tp->snd_nxt)) 91 return tp->snd_nxt; 92 else 93 return tcp_wnd_end(tp); 94 } 95 96 /* Calculate mss to advertise in SYN segment. 97 * RFC1122, RFC1063, draft-ietf-tcpimpl-pmtud-01 state that: 98 * 99 * 1. It is independent of path mtu. 100 * 2. Ideally, it is maximal possible segment size i.e. 65535-40. 101 * 3. For IPv4 it is reasonable to calculate it from maximal MTU of 102 * attached devices, because some buggy hosts are confused by 103 * large MSS. 104 * 4. We do not make 3, we advertise MSS, calculated from first 105 * hop device mtu, but allow to raise it to ip_rt_min_advmss. 106 * This may be overridden via information stored in routing table. 107 * 5. Value 65535 for MSS is valid in IPv6 and means "as large as possible, 108 * probably even Jumbo". 109 */ 110 static __u16 tcp_advertise_mss(struct sock *sk) 111 { 112 struct tcp_sock *tp = tcp_sk(sk); 113 struct dst_entry *dst = __sk_dst_get(sk); 114 int mss = tp->advmss; 115 116 if (dst && dst_metric(dst, RTAX_ADVMSS) < mss) { 117 mss = dst_metric(dst, RTAX_ADVMSS); 118 tp->advmss = mss; 119 } 120 121 return (__u16)mss; 122 } 123 124 /* RFC2861. Reset CWND after idle period longer RTO to "restart window". 125 * This is the first part of cwnd validation mechanism. */ 126 static void tcp_cwnd_restart(struct sock *sk, struct dst_entry *dst) 127 { 128 struct tcp_sock *tp = tcp_sk(sk); 129 s32 delta = tcp_time_stamp - tp->lsndtime; 130 u32 restart_cwnd = tcp_init_cwnd(tp, dst); 131 u32 cwnd = tp->snd_cwnd; 132 133 tcp_ca_event(sk, CA_EVENT_CWND_RESTART); 134 135 tp->snd_ssthresh = tcp_current_ssthresh(sk); 136 restart_cwnd = min(restart_cwnd, cwnd); 137 138 while ((delta -= inet_csk(sk)->icsk_rto) > 0 && cwnd > restart_cwnd) 139 cwnd >>= 1; 140 tp->snd_cwnd = max(cwnd, restart_cwnd); 141 tp->snd_cwnd_stamp = tcp_time_stamp; 142 tp->snd_cwnd_used = 0; 143 } 144 145 static void tcp_event_data_sent(struct tcp_sock *tp, 146 struct sk_buff *skb, struct sock *sk) 147 { 148 struct inet_connection_sock *icsk = inet_csk(sk); 149 const u32 now = tcp_time_stamp; 150 151 if (sysctl_tcp_slow_start_after_idle && 152 (!tp->packets_out && (s32)(now - tp->lsndtime) > icsk->icsk_rto)) 153 tcp_cwnd_restart(sk, __sk_dst_get(sk)); 154 155 tp->lsndtime = now; 156 157 /* If it is a reply for ato after last received 158 * packet, enter pingpong mode. 159 */ 160 if ((u32)(now - icsk->icsk_ack.lrcvtime) < icsk->icsk_ack.ato) 161 icsk->icsk_ack.pingpong = 1; 162 } 163 164 static inline void tcp_event_ack_sent(struct sock *sk, unsigned int pkts) 165 { 166 tcp_dec_quickack_mode(sk, pkts); 167 inet_csk_clear_xmit_timer(sk, ICSK_TIME_DACK); 168 } 169 170 /* Determine a window scaling and initial window to offer. 171 * Based on the assumption that the given amount of space 172 * will be offered. Store the results in the tp structure. 173 * NOTE: for smooth operation initial space offering should 174 * be a multiple of mss if possible. We assume here that mss >= 1. 175 * This MUST be enforced by all callers. 176 */ 177 void tcp_select_initial_window(int __space, __u32 mss, 178 __u32 *rcv_wnd, __u32 *window_clamp, 179 int wscale_ok, __u8 *rcv_wscale) 180 { 181 unsigned int space = (__space < 0 ? 0 : __space); 182 183 /* If no clamp set the clamp to the max possible scaled window */ 184 if (*window_clamp == 0) 185 (*window_clamp) = (65535 << 14); 186 space = min(*window_clamp, space); 187 188 /* Quantize space offering to a multiple of mss if possible. */ 189 if (space > mss) 190 space = (space / mss) * mss; 191 192 /* NOTE: offering an initial window larger than 32767 193 * will break some buggy TCP stacks. If the admin tells us 194 * it is likely we could be speaking with such a buggy stack 195 * we will truncate our initial window offering to 32K-1 196 * unless the remote has sent us a window scaling option, 197 * which we interpret as a sign the remote TCP is not 198 * misinterpreting the window field as a signed quantity. 199 */ 200 if (sysctl_tcp_workaround_signed_windows) 201 (*rcv_wnd) = min(space, MAX_TCP_WINDOW); 202 else 203 (*rcv_wnd) = space; 204 205 (*rcv_wscale) = 0; 206 if (wscale_ok) { 207 /* Set window scaling on max possible window 208 * See RFC1323 for an explanation of the limit to 14 209 */ 210 space = max_t(u32, sysctl_tcp_rmem[2], sysctl_rmem_max); 211 space = min_t(u32, space, *window_clamp); 212 while (space > 65535 && (*rcv_wscale) < 14) { 213 space >>= 1; 214 (*rcv_wscale)++; 215 } 216 } 217 218 /* Set initial window to value enough for senders, 219 * following RFC2414. Senders, not following this RFC, 220 * will be satisfied with 2. 221 */ 222 if (mss > (1 << *rcv_wscale)) { 223 int init_cwnd = 4; 224 if (mss > 1460 * 3) 225 init_cwnd = 2; 226 else if (mss > 1460) 227 init_cwnd = 3; 228 if (*rcv_wnd > init_cwnd * mss) 229 *rcv_wnd = init_cwnd * mss; 230 } 231 232 /* Set the clamp no higher than max representable value */ 233 (*window_clamp) = min(65535U << (*rcv_wscale), *window_clamp); 234 } 235 236 /* Chose a new window to advertise, update state in tcp_sock for the 237 * socket, and return result with RFC1323 scaling applied. The return 238 * value can be stuffed directly into th->window for an outgoing 239 * frame. 240 */ 241 static u16 tcp_select_window(struct sock *sk) 242 { 243 struct tcp_sock *tp = tcp_sk(sk); 244 u32 cur_win = tcp_receive_window(tp); 245 u32 new_win = __tcp_select_window(sk); 246 247 /* Never shrink the offered window */ 248 if (new_win < cur_win) { 249 /* Danger Will Robinson! 250 * Don't update rcv_wup/rcv_wnd here or else 251 * we will not be able to advertise a zero 252 * window in time. --DaveM 253 * 254 * Relax Will Robinson. 255 */ 256 new_win = ALIGN(cur_win, 1 << tp->rx_opt.rcv_wscale); 257 } 258 tp->rcv_wnd = new_win; 259 tp->rcv_wup = tp->rcv_nxt; 260 261 /* Make sure we do not exceed the maximum possible 262 * scaled window. 263 */ 264 if (!tp->rx_opt.rcv_wscale && sysctl_tcp_workaround_signed_windows) 265 new_win = min(new_win, MAX_TCP_WINDOW); 266 else 267 new_win = min(new_win, (65535U << tp->rx_opt.rcv_wscale)); 268 269 /* RFC1323 scaling applied */ 270 new_win >>= tp->rx_opt.rcv_wscale; 271 272 /* If we advertise zero window, disable fast path. */ 273 if (new_win == 0) 274 tp->pred_flags = 0; 275 276 return new_win; 277 } 278 279 static inline void TCP_ECN_send_synack(struct tcp_sock *tp, struct sk_buff *skb) 280 { 281 TCP_SKB_CB(skb)->flags &= ~TCPCB_FLAG_CWR; 282 if (!(tp->ecn_flags & TCP_ECN_OK)) 283 TCP_SKB_CB(skb)->flags &= ~TCPCB_FLAG_ECE; 284 } 285 286 static inline void TCP_ECN_send_syn(struct sock *sk, struct sk_buff *skb) 287 { 288 struct tcp_sock *tp = tcp_sk(sk); 289 290 tp->ecn_flags = 0; 291 if (sysctl_tcp_ecn) { 292 TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_ECE | TCPCB_FLAG_CWR; 293 tp->ecn_flags = TCP_ECN_OK; 294 } 295 } 296 297 static __inline__ void 298 TCP_ECN_make_synack(struct request_sock *req, struct tcphdr *th) 299 { 300 if (inet_rsk(req)->ecn_ok) 301 th->ece = 1; 302 } 303 304 static inline void TCP_ECN_send(struct sock *sk, struct sk_buff *skb, 305 int tcp_header_len) 306 { 307 struct tcp_sock *tp = tcp_sk(sk); 308 309 if (tp->ecn_flags & TCP_ECN_OK) { 310 /* Not-retransmitted data segment: set ECT and inject CWR. */ 311 if (skb->len != tcp_header_len && 312 !before(TCP_SKB_CB(skb)->seq, tp->snd_nxt)) { 313 INET_ECN_xmit(sk); 314 if (tp->ecn_flags & TCP_ECN_QUEUE_CWR) { 315 tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR; 316 tcp_hdr(skb)->cwr = 1; 317 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN; 318 } 319 } else { 320 /* ACK or retransmitted segment: clear ECT|CE */ 321 INET_ECN_dontxmit(sk); 322 } 323 if (tp->ecn_flags & TCP_ECN_DEMAND_CWR) 324 tcp_hdr(skb)->ece = 1; 325 } 326 } 327 328 /* Constructs common control bits of non-data skb. If SYN/FIN is present, 329 * auto increment end seqno. 330 */ 331 static void tcp_init_nondata_skb(struct sk_buff *skb, u32 seq, u8 flags) 332 { 333 skb->csum = 0; 334 335 TCP_SKB_CB(skb)->flags = flags; 336 TCP_SKB_CB(skb)->sacked = 0; 337 338 skb_shinfo(skb)->gso_segs = 1; 339 skb_shinfo(skb)->gso_size = 0; 340 skb_shinfo(skb)->gso_type = 0; 341 342 TCP_SKB_CB(skb)->seq = seq; 343 if (flags & (TCPCB_FLAG_SYN | TCPCB_FLAG_FIN)) 344 seq++; 345 TCP_SKB_CB(skb)->end_seq = seq; 346 } 347 348 static inline int tcp_urg_mode(const struct tcp_sock *tp) 349 { 350 return tp->snd_una != tp->snd_up; 351 } 352 353 #define OPTION_SACK_ADVERTISE (1 << 0) 354 #define OPTION_TS (1 << 1) 355 #define OPTION_MD5 (1 << 2) 356 357 struct tcp_out_options { 358 u8 options; /* bit field of OPTION_* */ 359 u8 ws; /* window scale, 0 to disable */ 360 u8 num_sack_blocks; /* number of SACK blocks to include */ 361 u16 mss; /* 0 to disable */ 362 __u32 tsval, tsecr; /* need to include OPTION_TS */ 363 }; 364 365 static void tcp_options_write(__be32 *ptr, struct tcp_sock *tp, 366 const struct tcp_out_options *opts, 367 __u8 **md5_hash) { 368 if (unlikely(OPTION_MD5 & opts->options)) { 369 *ptr++ = htonl((TCPOPT_NOP << 24) | 370 (TCPOPT_NOP << 16) | 371 (TCPOPT_MD5SIG << 8) | 372 TCPOLEN_MD5SIG); 373 *md5_hash = (__u8 *)ptr; 374 ptr += 4; 375 } else { 376 *md5_hash = NULL; 377 } 378 379 if (likely(OPTION_TS & opts->options)) { 380 if (unlikely(OPTION_SACK_ADVERTISE & opts->options)) { 381 *ptr++ = htonl((TCPOPT_SACK_PERM << 24) | 382 (TCPOLEN_SACK_PERM << 16) | 383 (TCPOPT_TIMESTAMP << 8) | 384 TCPOLEN_TIMESTAMP); 385 } else { 386 *ptr++ = htonl((TCPOPT_NOP << 24) | 387 (TCPOPT_NOP << 16) | 388 (TCPOPT_TIMESTAMP << 8) | 389 TCPOLEN_TIMESTAMP); 390 } 391 *ptr++ = htonl(opts->tsval); 392 *ptr++ = htonl(opts->tsecr); 393 } 394 395 if (unlikely(opts->mss)) { 396 *ptr++ = htonl((TCPOPT_MSS << 24) | 397 (TCPOLEN_MSS << 16) | 398 opts->mss); 399 } 400 401 if (unlikely(OPTION_SACK_ADVERTISE & opts->options && 402 !(OPTION_TS & opts->options))) { 403 *ptr++ = htonl((TCPOPT_NOP << 24) | 404 (TCPOPT_NOP << 16) | 405 (TCPOPT_SACK_PERM << 8) | 406 TCPOLEN_SACK_PERM); 407 } 408 409 if (unlikely(opts->ws)) { 410 *ptr++ = htonl((TCPOPT_NOP << 24) | 411 (TCPOPT_WINDOW << 16) | 412 (TCPOLEN_WINDOW << 8) | 413 opts->ws); 414 } 415 416 if (unlikely(opts->num_sack_blocks)) { 417 struct tcp_sack_block *sp = tp->rx_opt.dsack ? 418 tp->duplicate_sack : tp->selective_acks; 419 int this_sack; 420 421 *ptr++ = htonl((TCPOPT_NOP << 24) | 422 (TCPOPT_NOP << 16) | 423 (TCPOPT_SACK << 8) | 424 (TCPOLEN_SACK_BASE + (opts->num_sack_blocks * 425 TCPOLEN_SACK_PERBLOCK))); 426 427 for (this_sack = 0; this_sack < opts->num_sack_blocks; 428 ++this_sack) { 429 *ptr++ = htonl(sp[this_sack].start_seq); 430 *ptr++ = htonl(sp[this_sack].end_seq); 431 } 432 433 if (tp->rx_opt.dsack) { 434 tp->rx_opt.dsack = 0; 435 tp->rx_opt.eff_sacks--; 436 } 437 } 438 } 439 440 static unsigned tcp_syn_options(struct sock *sk, struct sk_buff *skb, 441 struct tcp_out_options *opts, 442 struct tcp_md5sig_key **md5) { 443 struct tcp_sock *tp = tcp_sk(sk); 444 unsigned size = 0; 445 446 #ifdef CONFIG_TCP_MD5SIG 447 *md5 = tp->af_specific->md5_lookup(sk, sk); 448 if (*md5) { 449 opts->options |= OPTION_MD5; 450 size += TCPOLEN_MD5SIG_ALIGNED; 451 } 452 #else 453 *md5 = NULL; 454 #endif 455 456 /* We always get an MSS option. The option bytes which will be seen in 457 * normal data packets should timestamps be used, must be in the MSS 458 * advertised. But we subtract them from tp->mss_cache so that 459 * calculations in tcp_sendmsg are simpler etc. So account for this 460 * fact here if necessary. If we don't do this correctly, as a 461 * receiver we won't recognize data packets as being full sized when we 462 * should, and thus we won't abide by the delayed ACK rules correctly. 463 * SACKs don't matter, we never delay an ACK when we have any of those 464 * going out. */ 465 opts->mss = tcp_advertise_mss(sk); 466 size += TCPOLEN_MSS_ALIGNED; 467 468 if (likely(sysctl_tcp_timestamps && *md5 == NULL)) { 469 opts->options |= OPTION_TS; 470 opts->tsval = TCP_SKB_CB(skb)->when; 471 opts->tsecr = tp->rx_opt.ts_recent; 472 size += TCPOLEN_TSTAMP_ALIGNED; 473 } 474 if (likely(sysctl_tcp_window_scaling)) { 475 opts->ws = tp->rx_opt.rcv_wscale; 476 if(likely(opts->ws)) 477 size += TCPOLEN_WSCALE_ALIGNED; 478 } 479 if (likely(sysctl_tcp_sack)) { 480 opts->options |= OPTION_SACK_ADVERTISE; 481 if (unlikely(!(OPTION_TS & opts->options))) 482 size += TCPOLEN_SACKPERM_ALIGNED; 483 } 484 485 return size; 486 } 487 488 static unsigned tcp_synack_options(struct sock *sk, 489 struct request_sock *req, 490 unsigned mss, struct sk_buff *skb, 491 struct tcp_out_options *opts, 492 struct tcp_md5sig_key **md5) { 493 unsigned size = 0; 494 struct inet_request_sock *ireq = inet_rsk(req); 495 char doing_ts; 496 497 #ifdef CONFIG_TCP_MD5SIG 498 *md5 = tcp_rsk(req)->af_specific->md5_lookup(sk, req); 499 if (*md5) { 500 opts->options |= OPTION_MD5; 501 size += TCPOLEN_MD5SIG_ALIGNED; 502 } 503 #else 504 *md5 = NULL; 505 #endif 506 507 /* we can't fit any SACK blocks in a packet with MD5 + TS 508 options. There was discussion about disabling SACK rather than TS in 509 order to fit in better with old, buggy kernels, but that was deemed 510 to be unnecessary. */ 511 doing_ts = ireq->tstamp_ok && !(*md5 && ireq->sack_ok); 512 513 opts->mss = mss; 514 size += TCPOLEN_MSS_ALIGNED; 515 516 if (likely(ireq->wscale_ok)) { 517 opts->ws = ireq->rcv_wscale; 518 if(likely(opts->ws)) 519 size += TCPOLEN_WSCALE_ALIGNED; 520 } 521 if (likely(doing_ts)) { 522 opts->options |= OPTION_TS; 523 opts->tsval = TCP_SKB_CB(skb)->when; 524 opts->tsecr = req->ts_recent; 525 size += TCPOLEN_TSTAMP_ALIGNED; 526 } 527 if (likely(ireq->sack_ok)) { 528 opts->options |= OPTION_SACK_ADVERTISE; 529 if (unlikely(!doing_ts)) 530 size += TCPOLEN_SACKPERM_ALIGNED; 531 } 532 533 return size; 534 } 535 536 static unsigned tcp_established_options(struct sock *sk, struct sk_buff *skb, 537 struct tcp_out_options *opts, 538 struct tcp_md5sig_key **md5) { 539 struct tcp_skb_cb *tcb = skb ? TCP_SKB_CB(skb) : NULL; 540 struct tcp_sock *tp = tcp_sk(sk); 541 unsigned size = 0; 542 543 #ifdef CONFIG_TCP_MD5SIG 544 *md5 = tp->af_specific->md5_lookup(sk, sk); 545 if (unlikely(*md5)) { 546 opts->options |= OPTION_MD5; 547 size += TCPOLEN_MD5SIG_ALIGNED; 548 } 549 #else 550 *md5 = NULL; 551 #endif 552 553 if (likely(tp->rx_opt.tstamp_ok)) { 554 opts->options |= OPTION_TS; 555 opts->tsval = tcb ? tcb->when : 0; 556 opts->tsecr = tp->rx_opt.ts_recent; 557 size += TCPOLEN_TSTAMP_ALIGNED; 558 } 559 560 if (unlikely(tp->rx_opt.eff_sacks)) { 561 const unsigned remaining = MAX_TCP_OPTION_SPACE - size; 562 opts->num_sack_blocks = 563 min_t(unsigned, tp->rx_opt.eff_sacks, 564 (remaining - TCPOLEN_SACK_BASE_ALIGNED) / 565 TCPOLEN_SACK_PERBLOCK); 566 size += TCPOLEN_SACK_BASE_ALIGNED + 567 opts->num_sack_blocks * TCPOLEN_SACK_PERBLOCK; 568 } 569 570 return size; 571 } 572 573 /* This routine actually transmits TCP packets queued in by 574 * tcp_do_sendmsg(). This is used by both the initial 575 * transmission and possible later retransmissions. 576 * All SKB's seen here are completely headerless. It is our 577 * job to build the TCP header, and pass the packet down to 578 * IP so it can do the same plus pass the packet off to the 579 * device. 580 * 581 * We are working here with either a clone of the original 582 * SKB, or a fresh unique copy made by the retransmit engine. 583 */ 584 static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it, 585 gfp_t gfp_mask) 586 { 587 const struct inet_connection_sock *icsk = inet_csk(sk); 588 struct inet_sock *inet; 589 struct tcp_sock *tp; 590 struct tcp_skb_cb *tcb; 591 struct tcp_out_options opts; 592 unsigned tcp_options_size, tcp_header_size; 593 struct tcp_md5sig_key *md5; 594 __u8 *md5_hash_location; 595 struct tcphdr *th; 596 int err; 597 598 BUG_ON(!skb || !tcp_skb_pcount(skb)); 599 600 /* If congestion control is doing timestamping, we must 601 * take such a timestamp before we potentially clone/copy. 602 */ 603 if (icsk->icsk_ca_ops->flags & TCP_CONG_RTT_STAMP) 604 __net_timestamp(skb); 605 606 if (likely(clone_it)) { 607 if (unlikely(skb_cloned(skb))) 608 skb = pskb_copy(skb, gfp_mask); 609 else 610 skb = skb_clone(skb, gfp_mask); 611 if (unlikely(!skb)) 612 return -ENOBUFS; 613 } 614 615 inet = inet_sk(sk); 616 tp = tcp_sk(sk); 617 tcb = TCP_SKB_CB(skb); 618 memset(&opts, 0, sizeof(opts)); 619 620 if (unlikely(tcb->flags & TCPCB_FLAG_SYN)) 621 tcp_options_size = tcp_syn_options(sk, skb, &opts, &md5); 622 else 623 tcp_options_size = tcp_established_options(sk, skb, &opts, 624 &md5); 625 tcp_header_size = tcp_options_size + sizeof(struct tcphdr); 626 627 if (tcp_packets_in_flight(tp) == 0) 628 tcp_ca_event(sk, CA_EVENT_TX_START); 629 630 skb_push(skb, tcp_header_size); 631 skb_reset_transport_header(skb); 632 skb_set_owner_w(skb, sk); 633 634 /* Build TCP header and checksum it. */ 635 th = tcp_hdr(skb); 636 th->source = inet->sport; 637 th->dest = inet->dport; 638 th->seq = htonl(tcb->seq); 639 th->ack_seq = htonl(tp->rcv_nxt); 640 *(((__be16 *)th) + 6) = htons(((tcp_header_size >> 2) << 12) | 641 tcb->flags); 642 643 if (unlikely(tcb->flags & TCPCB_FLAG_SYN)) { 644 /* RFC1323: The window in SYN & SYN/ACK segments 645 * is never scaled. 646 */ 647 th->window = htons(min(tp->rcv_wnd, 65535U)); 648 } else { 649 th->window = htons(tcp_select_window(sk)); 650 } 651 th->check = 0; 652 th->urg_ptr = 0; 653 654 /* The urg_mode check is necessary during a below snd_una win probe */ 655 if (unlikely(tcp_urg_mode(tp) && 656 between(tp->snd_up, tcb->seq + 1, tcb->seq + 0xFFFF))) { 657 th->urg_ptr = htons(tp->snd_up - tcb->seq); 658 th->urg = 1; 659 } 660 661 tcp_options_write((__be32 *)(th + 1), tp, &opts, &md5_hash_location); 662 if (likely((tcb->flags & TCPCB_FLAG_SYN) == 0)) 663 TCP_ECN_send(sk, skb, tcp_header_size); 664 665 #ifdef CONFIG_TCP_MD5SIG 666 /* Calculate the MD5 hash, as we have all we need now */ 667 if (md5) { 668 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 669 tp->af_specific->calc_md5_hash(md5_hash_location, 670 md5, sk, NULL, skb); 671 } 672 #endif 673 674 icsk->icsk_af_ops->send_check(sk, skb->len, skb); 675 676 if (likely(tcb->flags & TCPCB_FLAG_ACK)) 677 tcp_event_ack_sent(sk, tcp_skb_pcount(skb)); 678 679 if (skb->len != tcp_header_size) 680 tcp_event_data_sent(tp, skb, sk); 681 682 if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq) 683 TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTSEGS); 684 685 err = icsk->icsk_af_ops->queue_xmit(skb, 0); 686 if (likely(err <= 0)) 687 return err; 688 689 tcp_enter_cwr(sk, 1); 690 691 return net_xmit_eval(err); 692 } 693 694 /* This routine just queue's the buffer 695 * 696 * NOTE: probe0 timer is not checked, do not forget tcp_push_pending_frames, 697 * otherwise socket can stall. 698 */ 699 static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb) 700 { 701 struct tcp_sock *tp = tcp_sk(sk); 702 703 /* Advance write_seq and place onto the write_queue. */ 704 tp->write_seq = TCP_SKB_CB(skb)->end_seq; 705 skb_header_release(skb); 706 tcp_add_write_queue_tail(sk, skb); 707 sk->sk_wmem_queued += skb->truesize; 708 sk_mem_charge(sk, skb->truesize); 709 } 710 711 static void tcp_set_skb_tso_segs(struct sock *sk, struct sk_buff *skb, 712 unsigned int mss_now) 713 { 714 if (skb->len <= mss_now || !sk_can_gso(sk)) { 715 /* Avoid the costly divide in the normal 716 * non-TSO case. 717 */ 718 skb_shinfo(skb)->gso_segs = 1; 719 skb_shinfo(skb)->gso_size = 0; 720 skb_shinfo(skb)->gso_type = 0; 721 } else { 722 skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(skb->len, mss_now); 723 skb_shinfo(skb)->gso_size = mss_now; 724 skb_shinfo(skb)->gso_type = sk->sk_gso_type; 725 } 726 } 727 728 /* When a modification to fackets out becomes necessary, we need to check 729 * skb is counted to fackets_out or not. 730 */ 731 static void tcp_adjust_fackets_out(struct sock *sk, struct sk_buff *skb, 732 int decr) 733 { 734 struct tcp_sock *tp = tcp_sk(sk); 735 736 if (!tp->sacked_out || tcp_is_reno(tp)) 737 return; 738 739 if (after(tcp_highest_sack_seq(tp), TCP_SKB_CB(skb)->seq)) 740 tp->fackets_out -= decr; 741 } 742 743 /* Function to create two new TCP segments. Shrinks the given segment 744 * to the specified size and appends a new segment with the rest of the 745 * packet to the list. This won't be called frequently, I hope. 746 * Remember, these are still headerless SKBs at this point. 747 */ 748 int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len, 749 unsigned int mss_now) 750 { 751 struct tcp_sock *tp = tcp_sk(sk); 752 struct sk_buff *buff; 753 int nsize, old_factor; 754 int nlen; 755 u16 flags; 756 757 BUG_ON(len > skb->len); 758 759 tcp_clear_retrans_hints_partial(tp); 760 nsize = skb_headlen(skb) - len; 761 if (nsize < 0) 762 nsize = 0; 763 764 if (skb_cloned(skb) && 765 skb_is_nonlinear(skb) && 766 pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) 767 return -ENOMEM; 768 769 /* Get a new skb... force flag on. */ 770 buff = sk_stream_alloc_skb(sk, nsize, GFP_ATOMIC); 771 if (buff == NULL) 772 return -ENOMEM; /* We'll just try again later. */ 773 774 sk->sk_wmem_queued += buff->truesize; 775 sk_mem_charge(sk, buff->truesize); 776 nlen = skb->len - len - nsize; 777 buff->truesize += nlen; 778 skb->truesize -= nlen; 779 780 /* Correct the sequence numbers. */ 781 TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len; 782 TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq; 783 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq; 784 785 /* PSH and FIN should only be set in the second packet. */ 786 flags = TCP_SKB_CB(skb)->flags; 787 TCP_SKB_CB(skb)->flags = flags & ~(TCPCB_FLAG_FIN | TCPCB_FLAG_PSH); 788 TCP_SKB_CB(buff)->flags = flags; 789 TCP_SKB_CB(buff)->sacked = TCP_SKB_CB(skb)->sacked; 790 791 if (!skb_shinfo(skb)->nr_frags && skb->ip_summed != CHECKSUM_PARTIAL) { 792 /* Copy and checksum data tail into the new buffer. */ 793 buff->csum = csum_partial_copy_nocheck(skb->data + len, 794 skb_put(buff, nsize), 795 nsize, 0); 796 797 skb_trim(skb, len); 798 799 skb->csum = csum_block_sub(skb->csum, buff->csum, len); 800 } else { 801 skb->ip_summed = CHECKSUM_PARTIAL; 802 skb_split(skb, buff, len); 803 } 804 805 buff->ip_summed = skb->ip_summed; 806 807 /* Looks stupid, but our code really uses when of 808 * skbs, which it never sent before. --ANK 809 */ 810 TCP_SKB_CB(buff)->when = TCP_SKB_CB(skb)->when; 811 buff->tstamp = skb->tstamp; 812 813 old_factor = tcp_skb_pcount(skb); 814 815 /* Fix up tso_factor for both original and new SKB. */ 816 tcp_set_skb_tso_segs(sk, skb, mss_now); 817 tcp_set_skb_tso_segs(sk, buff, mss_now); 818 819 /* If this packet has been sent out already, we must 820 * adjust the various packet counters. 821 */ 822 if (!before(tp->snd_nxt, TCP_SKB_CB(buff)->end_seq)) { 823 int diff = old_factor - tcp_skb_pcount(skb) - 824 tcp_skb_pcount(buff); 825 826 tp->packets_out -= diff; 827 828 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED) 829 tp->sacked_out -= diff; 830 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) 831 tp->retrans_out -= diff; 832 833 if (TCP_SKB_CB(skb)->sacked & TCPCB_LOST) 834 tp->lost_out -= diff; 835 836 /* Adjust Reno SACK estimate. */ 837 if (tcp_is_reno(tp) && diff > 0) { 838 tcp_dec_pcount_approx_int(&tp->sacked_out, diff); 839 tcp_verify_left_out(tp); 840 } 841 tcp_adjust_fackets_out(sk, skb, diff); 842 } 843 844 /* Link BUFF into the send queue. */ 845 skb_header_release(buff); 846 tcp_insert_write_queue_after(skb, buff, sk); 847 848 return 0; 849 } 850 851 /* This is similar to __pskb_pull_head() (it will go to core/skbuff.c 852 * eventually). The difference is that pulled data not copied, but 853 * immediately discarded. 854 */ 855 static void __pskb_trim_head(struct sk_buff *skb, int len) 856 { 857 int i, k, eat; 858 859 eat = len; 860 k = 0; 861 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 862 if (skb_shinfo(skb)->frags[i].size <= eat) { 863 put_page(skb_shinfo(skb)->frags[i].page); 864 eat -= skb_shinfo(skb)->frags[i].size; 865 } else { 866 skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i]; 867 if (eat) { 868 skb_shinfo(skb)->frags[k].page_offset += eat; 869 skb_shinfo(skb)->frags[k].size -= eat; 870 eat = 0; 871 } 872 k++; 873 } 874 } 875 skb_shinfo(skb)->nr_frags = k; 876 877 skb_reset_tail_pointer(skb); 878 skb->data_len -= len; 879 skb->len = skb->data_len; 880 } 881 882 int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len) 883 { 884 if (skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) 885 return -ENOMEM; 886 887 /* If len == headlen, we avoid __skb_pull to preserve alignment. */ 888 if (unlikely(len < skb_headlen(skb))) 889 __skb_pull(skb, len); 890 else 891 __pskb_trim_head(skb, len - skb_headlen(skb)); 892 893 TCP_SKB_CB(skb)->seq += len; 894 skb->ip_summed = CHECKSUM_PARTIAL; 895 896 skb->truesize -= len; 897 sk->sk_wmem_queued -= len; 898 sk_mem_uncharge(sk, len); 899 sock_set_flag(sk, SOCK_QUEUE_SHRUNK); 900 901 /* Any change of skb->len requires recalculation of tso 902 * factor and mss. 903 */ 904 if (tcp_skb_pcount(skb) > 1) 905 tcp_set_skb_tso_segs(sk, skb, tcp_current_mss(sk, 1)); 906 907 return 0; 908 } 909 910 /* Not accounting for SACKs here. */ 911 int tcp_mtu_to_mss(struct sock *sk, int pmtu) 912 { 913 struct tcp_sock *tp = tcp_sk(sk); 914 struct inet_connection_sock *icsk = inet_csk(sk); 915 int mss_now; 916 917 /* Calculate base mss without TCP options: 918 It is MMS_S - sizeof(tcphdr) of rfc1122 919 */ 920 mss_now = pmtu - icsk->icsk_af_ops->net_header_len - sizeof(struct tcphdr); 921 922 /* Clamp it (mss_clamp does not include tcp options) */ 923 if (mss_now > tp->rx_opt.mss_clamp) 924 mss_now = tp->rx_opt.mss_clamp; 925 926 /* Now subtract optional transport overhead */ 927 mss_now -= icsk->icsk_ext_hdr_len; 928 929 /* Then reserve room for full set of TCP options and 8 bytes of data */ 930 if (mss_now < 48) 931 mss_now = 48; 932 933 /* Now subtract TCP options size, not including SACKs */ 934 mss_now -= tp->tcp_header_len - sizeof(struct tcphdr); 935 936 return mss_now; 937 } 938 939 /* Inverse of above */ 940 int tcp_mss_to_mtu(struct sock *sk, int mss) 941 { 942 struct tcp_sock *tp = tcp_sk(sk); 943 struct inet_connection_sock *icsk = inet_csk(sk); 944 int mtu; 945 946 mtu = mss + 947 tp->tcp_header_len + 948 icsk->icsk_ext_hdr_len + 949 icsk->icsk_af_ops->net_header_len; 950 951 return mtu; 952 } 953 954 void tcp_mtup_init(struct sock *sk) 955 { 956 struct tcp_sock *tp = tcp_sk(sk); 957 struct inet_connection_sock *icsk = inet_csk(sk); 958 959 icsk->icsk_mtup.enabled = sysctl_tcp_mtu_probing > 1; 960 icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp + sizeof(struct tcphdr) + 961 icsk->icsk_af_ops->net_header_len; 962 icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, sysctl_tcp_base_mss); 963 icsk->icsk_mtup.probe_size = 0; 964 } 965 966 /* Bound MSS / TSO packet size with the half of the window */ 967 static int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize) 968 { 969 if (tp->max_window && pktsize > (tp->max_window >> 1)) 970 return max(tp->max_window >> 1, 68U - tp->tcp_header_len); 971 else 972 return pktsize; 973 } 974 975 /* This function synchronize snd mss to current pmtu/exthdr set. 976 977 tp->rx_opt.user_mss is mss set by user by TCP_MAXSEG. It does NOT counts 978 for TCP options, but includes only bare TCP header. 979 980 tp->rx_opt.mss_clamp is mss negotiated at connection setup. 981 It is minimum of user_mss and mss received with SYN. 982 It also does not include TCP options. 983 984 inet_csk(sk)->icsk_pmtu_cookie is last pmtu, seen by this function. 985 986 tp->mss_cache is current effective sending mss, including 987 all tcp options except for SACKs. It is evaluated, 988 taking into account current pmtu, but never exceeds 989 tp->rx_opt.mss_clamp. 990 991 NOTE1. rfc1122 clearly states that advertised MSS 992 DOES NOT include either tcp or ip options. 993 994 NOTE2. inet_csk(sk)->icsk_pmtu_cookie and tp->mss_cache 995 are READ ONLY outside this function. --ANK (980731) 996 */ 997 unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu) 998 { 999 struct tcp_sock *tp = tcp_sk(sk); 1000 struct inet_connection_sock *icsk = inet_csk(sk); 1001 int mss_now; 1002 1003 if (icsk->icsk_mtup.search_high > pmtu) 1004 icsk->icsk_mtup.search_high = pmtu; 1005 1006 mss_now = tcp_mtu_to_mss(sk, pmtu); 1007 mss_now = tcp_bound_to_half_wnd(tp, mss_now); 1008 1009 /* And store cached results */ 1010 icsk->icsk_pmtu_cookie = pmtu; 1011 if (icsk->icsk_mtup.enabled) 1012 mss_now = min(mss_now, tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_low)); 1013 tp->mss_cache = mss_now; 1014 1015 return mss_now; 1016 } 1017 1018 /* Compute the current effective MSS, taking SACKs and IP options, 1019 * and even PMTU discovery events into account. 1020 * 1021 * LARGESEND note: !tcp_urg_mode is overkill, only frames up to snd_up 1022 * cannot be large. However, taking into account rare use of URG, this 1023 * is not a big flaw. 1024 */ 1025 unsigned int tcp_current_mss(struct sock *sk, int large_allowed) 1026 { 1027 struct tcp_sock *tp = tcp_sk(sk); 1028 struct dst_entry *dst = __sk_dst_get(sk); 1029 u32 mss_now; 1030 u16 xmit_size_goal; 1031 int doing_tso = 0; 1032 unsigned header_len; 1033 struct tcp_out_options opts; 1034 struct tcp_md5sig_key *md5; 1035 1036 mss_now = tp->mss_cache; 1037 1038 if (large_allowed && sk_can_gso(sk) && !tcp_urg_mode(tp)) 1039 doing_tso = 1; 1040 1041 if (dst) { 1042 u32 mtu = dst_mtu(dst); 1043 if (mtu != inet_csk(sk)->icsk_pmtu_cookie) 1044 mss_now = tcp_sync_mss(sk, mtu); 1045 } 1046 1047 header_len = tcp_established_options(sk, NULL, &opts, &md5) + 1048 sizeof(struct tcphdr); 1049 /* The mss_cache is sized based on tp->tcp_header_len, which assumes 1050 * some common options. If this is an odd packet (because we have SACK 1051 * blocks etc) then our calculated header_len will be different, and 1052 * we have to adjust mss_now correspondingly */ 1053 if (header_len != tp->tcp_header_len) { 1054 int delta = (int) header_len - tp->tcp_header_len; 1055 mss_now -= delta; 1056 } 1057 1058 xmit_size_goal = mss_now; 1059 1060 if (doing_tso) { 1061 xmit_size_goal = ((sk->sk_gso_max_size - 1) - 1062 inet_csk(sk)->icsk_af_ops->net_header_len - 1063 inet_csk(sk)->icsk_ext_hdr_len - 1064 tp->tcp_header_len); 1065 1066 xmit_size_goal = tcp_bound_to_half_wnd(tp, xmit_size_goal); 1067 xmit_size_goal -= (xmit_size_goal % mss_now); 1068 } 1069 tp->xmit_size_goal = xmit_size_goal; 1070 1071 return mss_now; 1072 } 1073 1074 /* Congestion window validation. (RFC2861) */ 1075 static void tcp_cwnd_validate(struct sock *sk) 1076 { 1077 struct tcp_sock *tp = tcp_sk(sk); 1078 1079 if (tp->packets_out >= tp->snd_cwnd) { 1080 /* Network is feed fully. */ 1081 tp->snd_cwnd_used = 0; 1082 tp->snd_cwnd_stamp = tcp_time_stamp; 1083 } else { 1084 /* Network starves. */ 1085 if (tp->packets_out > tp->snd_cwnd_used) 1086 tp->snd_cwnd_used = tp->packets_out; 1087 1088 if (sysctl_tcp_slow_start_after_idle && 1089 (s32)(tcp_time_stamp - tp->snd_cwnd_stamp) >= inet_csk(sk)->icsk_rto) 1090 tcp_cwnd_application_limited(sk); 1091 } 1092 } 1093 1094 /* Returns the portion of skb which can be sent right away without 1095 * introducing MSS oddities to segment boundaries. In rare cases where 1096 * mss_now != mss_cache, we will request caller to create a small skb 1097 * per input skb which could be mostly avoided here (if desired). 1098 * 1099 * We explicitly want to create a request for splitting write queue tail 1100 * to a small skb for Nagle purposes while avoiding unnecessary modulos, 1101 * thus all the complexity (cwnd_len is always MSS multiple which we 1102 * return whenever allowed by the other factors). Basically we need the 1103 * modulo only when the receiver window alone is the limiting factor or 1104 * when we would be allowed to send the split-due-to-Nagle skb fully. 1105 */ 1106 static unsigned int tcp_mss_split_point(struct sock *sk, struct sk_buff *skb, 1107 unsigned int mss_now, unsigned int cwnd) 1108 { 1109 struct tcp_sock *tp = tcp_sk(sk); 1110 u32 needed, window, cwnd_len; 1111 1112 window = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq; 1113 cwnd_len = mss_now * cwnd; 1114 1115 if (likely(cwnd_len <= window && skb != tcp_write_queue_tail(sk))) 1116 return cwnd_len; 1117 1118 needed = min(skb->len, window); 1119 1120 if (cwnd_len <= needed) 1121 return cwnd_len; 1122 1123 return needed - needed % mss_now; 1124 } 1125 1126 /* Can at least one segment of SKB be sent right now, according to the 1127 * congestion window rules? If so, return how many segments are allowed. 1128 */ 1129 static inline unsigned int tcp_cwnd_test(struct tcp_sock *tp, 1130 struct sk_buff *skb) 1131 { 1132 u32 in_flight, cwnd; 1133 1134 /* Don't be strict about the congestion window for the final FIN. */ 1135 if ((TCP_SKB_CB(skb)->flags & TCPCB_FLAG_FIN) && 1136 tcp_skb_pcount(skb) == 1) 1137 return 1; 1138 1139 in_flight = tcp_packets_in_flight(tp); 1140 cwnd = tp->snd_cwnd; 1141 if (in_flight < cwnd) 1142 return (cwnd - in_flight); 1143 1144 return 0; 1145 } 1146 1147 /* This must be invoked the first time we consider transmitting 1148 * SKB onto the wire. 1149 */ 1150 static int tcp_init_tso_segs(struct sock *sk, struct sk_buff *skb, 1151 unsigned int mss_now) 1152 { 1153 int tso_segs = tcp_skb_pcount(skb); 1154 1155 if (!tso_segs || (tso_segs > 1 && tcp_skb_mss(skb) != mss_now)) { 1156 tcp_set_skb_tso_segs(sk, skb, mss_now); 1157 tso_segs = tcp_skb_pcount(skb); 1158 } 1159 return tso_segs; 1160 } 1161 1162 static inline int tcp_minshall_check(const struct tcp_sock *tp) 1163 { 1164 return after(tp->snd_sml,tp->snd_una) && 1165 !after(tp->snd_sml, tp->snd_nxt); 1166 } 1167 1168 /* Return 0, if packet can be sent now without violation Nagle's rules: 1169 * 1. It is full sized. 1170 * 2. Or it contains FIN. (already checked by caller) 1171 * 3. Or TCP_NODELAY was set. 1172 * 4. Or TCP_CORK is not set, and all sent packets are ACKed. 1173 * With Minshall's modification: all sent small packets are ACKed. 1174 */ 1175 static inline int tcp_nagle_check(const struct tcp_sock *tp, 1176 const struct sk_buff *skb, 1177 unsigned mss_now, int nonagle) 1178 { 1179 return (skb->len < mss_now && 1180 ((nonagle & TCP_NAGLE_CORK) || 1181 (!nonagle && tp->packets_out && tcp_minshall_check(tp)))); 1182 } 1183 1184 /* Return non-zero if the Nagle test allows this packet to be 1185 * sent now. 1186 */ 1187 static inline int tcp_nagle_test(struct tcp_sock *tp, struct sk_buff *skb, 1188 unsigned int cur_mss, int nonagle) 1189 { 1190 /* Nagle rule does not apply to frames, which sit in the middle of the 1191 * write_queue (they have no chances to get new data). 1192 * 1193 * This is implemented in the callers, where they modify the 'nonagle' 1194 * argument based upon the location of SKB in the send queue. 1195 */ 1196 if (nonagle & TCP_NAGLE_PUSH) 1197 return 1; 1198 1199 /* Don't use the nagle rule for urgent data (or for the final FIN). 1200 * Nagle can be ignored during F-RTO too (see RFC4138). 1201 */ 1202 if (tcp_urg_mode(tp) || (tp->frto_counter == 2) || 1203 (TCP_SKB_CB(skb)->flags & TCPCB_FLAG_FIN)) 1204 return 1; 1205 1206 if (!tcp_nagle_check(tp, skb, cur_mss, nonagle)) 1207 return 1; 1208 1209 return 0; 1210 } 1211 1212 /* Does at least the first segment of SKB fit into the send window? */ 1213 static inline int tcp_snd_wnd_test(struct tcp_sock *tp, struct sk_buff *skb, 1214 unsigned int cur_mss) 1215 { 1216 u32 end_seq = TCP_SKB_CB(skb)->end_seq; 1217 1218 if (skb->len > cur_mss) 1219 end_seq = TCP_SKB_CB(skb)->seq + cur_mss; 1220 1221 return !after(end_seq, tcp_wnd_end(tp)); 1222 } 1223 1224 /* This checks if the data bearing packet SKB (usually tcp_send_head(sk)) 1225 * should be put on the wire right now. If so, it returns the number of 1226 * packets allowed by the congestion window. 1227 */ 1228 static unsigned int tcp_snd_test(struct sock *sk, struct sk_buff *skb, 1229 unsigned int cur_mss, int nonagle) 1230 { 1231 struct tcp_sock *tp = tcp_sk(sk); 1232 unsigned int cwnd_quota; 1233 1234 tcp_init_tso_segs(sk, skb, cur_mss); 1235 1236 if (!tcp_nagle_test(tp, skb, cur_mss, nonagle)) 1237 return 0; 1238 1239 cwnd_quota = tcp_cwnd_test(tp, skb); 1240 if (cwnd_quota && !tcp_snd_wnd_test(tp, skb, cur_mss)) 1241 cwnd_quota = 0; 1242 1243 return cwnd_quota; 1244 } 1245 1246 int tcp_may_send_now(struct sock *sk) 1247 { 1248 struct tcp_sock *tp = tcp_sk(sk); 1249 struct sk_buff *skb = tcp_send_head(sk); 1250 1251 return (skb && 1252 tcp_snd_test(sk, skb, tcp_current_mss(sk, 1), 1253 (tcp_skb_is_last(sk, skb) ? 1254 tp->nonagle : TCP_NAGLE_PUSH))); 1255 } 1256 1257 /* Trim TSO SKB to LEN bytes, put the remaining data into a new packet 1258 * which is put after SKB on the list. It is very much like 1259 * tcp_fragment() except that it may make several kinds of assumptions 1260 * in order to speed up the splitting operation. In particular, we 1261 * know that all the data is in scatter-gather pages, and that the 1262 * packet has never been sent out before (and thus is not cloned). 1263 */ 1264 static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len, 1265 unsigned int mss_now) 1266 { 1267 struct sk_buff *buff; 1268 int nlen = skb->len - len; 1269 u16 flags; 1270 1271 /* All of a TSO frame must be composed of paged data. */ 1272 if (skb->len != skb->data_len) 1273 return tcp_fragment(sk, skb, len, mss_now); 1274 1275 buff = sk_stream_alloc_skb(sk, 0, GFP_ATOMIC); 1276 if (unlikely(buff == NULL)) 1277 return -ENOMEM; 1278 1279 sk->sk_wmem_queued += buff->truesize; 1280 sk_mem_charge(sk, buff->truesize); 1281 buff->truesize += nlen; 1282 skb->truesize -= nlen; 1283 1284 /* Correct the sequence numbers. */ 1285 TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len; 1286 TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq; 1287 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq; 1288 1289 /* PSH and FIN should only be set in the second packet. */ 1290 flags = TCP_SKB_CB(skb)->flags; 1291 TCP_SKB_CB(skb)->flags = flags & ~(TCPCB_FLAG_FIN | TCPCB_FLAG_PSH); 1292 TCP_SKB_CB(buff)->flags = flags; 1293 1294 /* This packet was never sent out yet, so no SACK bits. */ 1295 TCP_SKB_CB(buff)->sacked = 0; 1296 1297 buff->ip_summed = skb->ip_summed = CHECKSUM_PARTIAL; 1298 skb_split(skb, buff, len); 1299 1300 /* Fix up tso_factor for both original and new SKB. */ 1301 tcp_set_skb_tso_segs(sk, skb, mss_now); 1302 tcp_set_skb_tso_segs(sk, buff, mss_now); 1303 1304 /* Link BUFF into the send queue. */ 1305 skb_header_release(buff); 1306 tcp_insert_write_queue_after(skb, buff, sk); 1307 1308 return 0; 1309 } 1310 1311 /* Try to defer sending, if possible, in order to minimize the amount 1312 * of TSO splitting we do. View it as a kind of TSO Nagle test. 1313 * 1314 * This algorithm is from John Heffner. 1315 */ 1316 static int tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb) 1317 { 1318 struct tcp_sock *tp = tcp_sk(sk); 1319 const struct inet_connection_sock *icsk = inet_csk(sk); 1320 u32 send_win, cong_win, limit, in_flight; 1321 1322 if (TCP_SKB_CB(skb)->flags & TCPCB_FLAG_FIN) 1323 goto send_now; 1324 1325 if (icsk->icsk_ca_state != TCP_CA_Open) 1326 goto send_now; 1327 1328 /* Defer for less than two clock ticks. */ 1329 if (tp->tso_deferred && 1330 ((jiffies << 1) >> 1) - (tp->tso_deferred >> 1) > 1) 1331 goto send_now; 1332 1333 in_flight = tcp_packets_in_flight(tp); 1334 1335 BUG_ON(tcp_skb_pcount(skb) <= 1 || (tp->snd_cwnd <= in_flight)); 1336 1337 send_win = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq; 1338 1339 /* From in_flight test above, we know that cwnd > in_flight. */ 1340 cong_win = (tp->snd_cwnd - in_flight) * tp->mss_cache; 1341 1342 limit = min(send_win, cong_win); 1343 1344 /* If a full-sized TSO skb can be sent, do it. */ 1345 if (limit >= sk->sk_gso_max_size) 1346 goto send_now; 1347 1348 if (sysctl_tcp_tso_win_divisor) { 1349 u32 chunk = min(tp->snd_wnd, tp->snd_cwnd * tp->mss_cache); 1350 1351 /* If at least some fraction of a window is available, 1352 * just use it. 1353 */ 1354 chunk /= sysctl_tcp_tso_win_divisor; 1355 if (limit >= chunk) 1356 goto send_now; 1357 } else { 1358 /* Different approach, try not to defer past a single 1359 * ACK. Receiver should ACK every other full sized 1360 * frame, so if we have space for more than 3 frames 1361 * then send now. 1362 */ 1363 if (limit > tcp_max_burst(tp) * tp->mss_cache) 1364 goto send_now; 1365 } 1366 1367 /* Ok, it looks like it is advisable to defer. */ 1368 tp->tso_deferred = 1 | (jiffies << 1); 1369 1370 return 1; 1371 1372 send_now: 1373 tp->tso_deferred = 0; 1374 return 0; 1375 } 1376 1377 /* Create a new MTU probe if we are ready. 1378 * Returns 0 if we should wait to probe (no cwnd available), 1379 * 1 if a probe was sent, 1380 * -1 otherwise 1381 */ 1382 static int tcp_mtu_probe(struct sock *sk) 1383 { 1384 struct tcp_sock *tp = tcp_sk(sk); 1385 struct inet_connection_sock *icsk = inet_csk(sk); 1386 struct sk_buff *skb, *nskb, *next; 1387 int len; 1388 int probe_size; 1389 int size_needed; 1390 int copy; 1391 int mss_now; 1392 1393 /* Not currently probing/verifying, 1394 * not in recovery, 1395 * have enough cwnd, and 1396 * not SACKing (the variable headers throw things off) */ 1397 if (!icsk->icsk_mtup.enabled || 1398 icsk->icsk_mtup.probe_size || 1399 inet_csk(sk)->icsk_ca_state != TCP_CA_Open || 1400 tp->snd_cwnd < 11 || 1401 tp->rx_opt.eff_sacks) 1402 return -1; 1403 1404 /* Very simple search strategy: just double the MSS. */ 1405 mss_now = tcp_current_mss(sk, 0); 1406 probe_size = 2 * tp->mss_cache; 1407 size_needed = probe_size + (tp->reordering + 1) * tp->mss_cache; 1408 if (probe_size > tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_high)) { 1409 /* TODO: set timer for probe_converge_event */ 1410 return -1; 1411 } 1412 1413 /* Have enough data in the send queue to probe? */ 1414 if (tp->write_seq - tp->snd_nxt < size_needed) 1415 return -1; 1416 1417 if (tp->snd_wnd < size_needed) 1418 return -1; 1419 if (after(tp->snd_nxt + size_needed, tcp_wnd_end(tp))) 1420 return 0; 1421 1422 /* Do we need to wait to drain cwnd? With none in flight, don't stall */ 1423 if (tcp_packets_in_flight(tp) + 2 > tp->snd_cwnd) { 1424 if (!tcp_packets_in_flight(tp)) 1425 return -1; 1426 else 1427 return 0; 1428 } 1429 1430 /* We're allowed to probe. Build it now. */ 1431 if ((nskb = sk_stream_alloc_skb(sk, probe_size, GFP_ATOMIC)) == NULL) 1432 return -1; 1433 sk->sk_wmem_queued += nskb->truesize; 1434 sk_mem_charge(sk, nskb->truesize); 1435 1436 skb = tcp_send_head(sk); 1437 1438 TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq; 1439 TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size; 1440 TCP_SKB_CB(nskb)->flags = TCPCB_FLAG_ACK; 1441 TCP_SKB_CB(nskb)->sacked = 0; 1442 nskb->csum = 0; 1443 nskb->ip_summed = skb->ip_summed; 1444 1445 tcp_insert_write_queue_before(nskb, skb, sk); 1446 1447 len = 0; 1448 tcp_for_write_queue_from_safe(skb, next, sk) { 1449 copy = min_t(int, skb->len, probe_size - len); 1450 if (nskb->ip_summed) 1451 skb_copy_bits(skb, 0, skb_put(nskb, copy), copy); 1452 else 1453 nskb->csum = skb_copy_and_csum_bits(skb, 0, 1454 skb_put(nskb, copy), 1455 copy, nskb->csum); 1456 1457 if (skb->len <= copy) { 1458 /* We've eaten all the data from this skb. 1459 * Throw it away. */ 1460 TCP_SKB_CB(nskb)->flags |= TCP_SKB_CB(skb)->flags; 1461 tcp_unlink_write_queue(skb, sk); 1462 sk_wmem_free_skb(sk, skb); 1463 } else { 1464 TCP_SKB_CB(nskb)->flags |= TCP_SKB_CB(skb)->flags & 1465 ~(TCPCB_FLAG_FIN|TCPCB_FLAG_PSH); 1466 if (!skb_shinfo(skb)->nr_frags) { 1467 skb_pull(skb, copy); 1468 if (skb->ip_summed != CHECKSUM_PARTIAL) 1469 skb->csum = csum_partial(skb->data, 1470 skb->len, 0); 1471 } else { 1472 __pskb_trim_head(skb, copy); 1473 tcp_set_skb_tso_segs(sk, skb, mss_now); 1474 } 1475 TCP_SKB_CB(skb)->seq += copy; 1476 } 1477 1478 len += copy; 1479 1480 if (len >= probe_size) 1481 break; 1482 } 1483 tcp_init_tso_segs(sk, nskb, nskb->len); 1484 1485 /* We're ready to send. If this fails, the probe will 1486 * be resegmented into mss-sized pieces by tcp_write_xmit(). */ 1487 TCP_SKB_CB(nskb)->when = tcp_time_stamp; 1488 if (!tcp_transmit_skb(sk, nskb, 1, GFP_ATOMIC)) { 1489 /* Decrement cwnd here because we are sending 1490 * effectively two packets. */ 1491 tp->snd_cwnd--; 1492 tcp_event_new_data_sent(sk, nskb); 1493 1494 icsk->icsk_mtup.probe_size = tcp_mss_to_mtu(sk, nskb->len); 1495 tp->mtu_probe.probe_seq_start = TCP_SKB_CB(nskb)->seq; 1496 tp->mtu_probe.probe_seq_end = TCP_SKB_CB(nskb)->end_seq; 1497 1498 return 1; 1499 } 1500 1501 return -1; 1502 } 1503 1504 /* This routine writes packets to the network. It advances the 1505 * send_head. This happens as incoming acks open up the remote 1506 * window for us. 1507 * 1508 * Returns 1, if no segments are in flight and we have queued segments, but 1509 * cannot send anything now because of SWS or another problem. 1510 */ 1511 static int tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle) 1512 { 1513 struct tcp_sock *tp = tcp_sk(sk); 1514 struct sk_buff *skb; 1515 unsigned int tso_segs, sent_pkts; 1516 int cwnd_quota; 1517 int result; 1518 1519 /* If we are closed, the bytes will have to remain here. 1520 * In time closedown will finish, we empty the write queue and all 1521 * will be happy. 1522 */ 1523 if (unlikely(sk->sk_state == TCP_CLOSE)) 1524 return 0; 1525 1526 sent_pkts = 0; 1527 1528 /* Do MTU probing. */ 1529 if ((result = tcp_mtu_probe(sk)) == 0) { 1530 return 0; 1531 } else if (result > 0) { 1532 sent_pkts = 1; 1533 } 1534 1535 while ((skb = tcp_send_head(sk))) { 1536 unsigned int limit; 1537 1538 tso_segs = tcp_init_tso_segs(sk, skb, mss_now); 1539 BUG_ON(!tso_segs); 1540 1541 cwnd_quota = tcp_cwnd_test(tp, skb); 1542 if (!cwnd_quota) 1543 break; 1544 1545 if (unlikely(!tcp_snd_wnd_test(tp, skb, mss_now))) 1546 break; 1547 1548 if (tso_segs == 1) { 1549 if (unlikely(!tcp_nagle_test(tp, skb, mss_now, 1550 (tcp_skb_is_last(sk, skb) ? 1551 nonagle : TCP_NAGLE_PUSH)))) 1552 break; 1553 } else { 1554 if (tcp_tso_should_defer(sk, skb)) 1555 break; 1556 } 1557 1558 limit = mss_now; 1559 if (tso_segs > 1) 1560 limit = tcp_mss_split_point(sk, skb, mss_now, 1561 cwnd_quota); 1562 1563 if (skb->len > limit && 1564 unlikely(tso_fragment(sk, skb, limit, mss_now))) 1565 break; 1566 1567 TCP_SKB_CB(skb)->when = tcp_time_stamp; 1568 1569 if (unlikely(tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC))) 1570 break; 1571 1572 /* Advance the send_head. This one is sent out. 1573 * This call will increment packets_out. 1574 */ 1575 tcp_event_new_data_sent(sk, skb); 1576 1577 tcp_minshall_update(tp, mss_now, skb); 1578 sent_pkts++; 1579 } 1580 1581 if (likely(sent_pkts)) { 1582 tcp_cwnd_validate(sk); 1583 return 0; 1584 } 1585 return !tp->packets_out && tcp_send_head(sk); 1586 } 1587 1588 /* Push out any pending frames which were held back due to 1589 * TCP_CORK or attempt at coalescing tiny packets. 1590 * The socket must be locked by the caller. 1591 */ 1592 void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss, 1593 int nonagle) 1594 { 1595 struct sk_buff *skb = tcp_send_head(sk); 1596 1597 if (skb) { 1598 if (tcp_write_xmit(sk, cur_mss, nonagle)) 1599 tcp_check_probe_timer(sk); 1600 } 1601 } 1602 1603 /* Send _single_ skb sitting at the send head. This function requires 1604 * true push pending frames to setup probe timer etc. 1605 */ 1606 void tcp_push_one(struct sock *sk, unsigned int mss_now) 1607 { 1608 struct sk_buff *skb = tcp_send_head(sk); 1609 unsigned int tso_segs, cwnd_quota; 1610 1611 BUG_ON(!skb || skb->len < mss_now); 1612 1613 tso_segs = tcp_init_tso_segs(sk, skb, mss_now); 1614 cwnd_quota = tcp_snd_test(sk, skb, mss_now, TCP_NAGLE_PUSH); 1615 1616 if (likely(cwnd_quota)) { 1617 unsigned int limit; 1618 1619 BUG_ON(!tso_segs); 1620 1621 limit = mss_now; 1622 if (tso_segs > 1) 1623 limit = tcp_mss_split_point(sk, skb, mss_now, 1624 cwnd_quota); 1625 1626 if (skb->len > limit && 1627 unlikely(tso_fragment(sk, skb, limit, mss_now))) 1628 return; 1629 1630 /* Send it out now. */ 1631 TCP_SKB_CB(skb)->when = tcp_time_stamp; 1632 1633 if (likely(!tcp_transmit_skb(sk, skb, 1, sk->sk_allocation))) { 1634 tcp_event_new_data_sent(sk, skb); 1635 tcp_cwnd_validate(sk); 1636 return; 1637 } 1638 } 1639 } 1640 1641 /* This function returns the amount that we can raise the 1642 * usable window based on the following constraints 1643 * 1644 * 1. The window can never be shrunk once it is offered (RFC 793) 1645 * 2. We limit memory per socket 1646 * 1647 * RFC 1122: 1648 * "the suggested [SWS] avoidance algorithm for the receiver is to keep 1649 * RECV.NEXT + RCV.WIN fixed until: 1650 * RCV.BUFF - RCV.USER - RCV.WINDOW >= min(1/2 RCV.BUFF, MSS)" 1651 * 1652 * i.e. don't raise the right edge of the window until you can raise 1653 * it at least MSS bytes. 1654 * 1655 * Unfortunately, the recommended algorithm breaks header prediction, 1656 * since header prediction assumes th->window stays fixed. 1657 * 1658 * Strictly speaking, keeping th->window fixed violates the receiver 1659 * side SWS prevention criteria. The problem is that under this rule 1660 * a stream of single byte packets will cause the right side of the 1661 * window to always advance by a single byte. 1662 * 1663 * Of course, if the sender implements sender side SWS prevention 1664 * then this will not be a problem. 1665 * 1666 * BSD seems to make the following compromise: 1667 * 1668 * If the free space is less than the 1/4 of the maximum 1669 * space available and the free space is less than 1/2 mss, 1670 * then set the window to 0. 1671 * [ Actually, bsd uses MSS and 1/4 of maximal _window_ ] 1672 * Otherwise, just prevent the window from shrinking 1673 * and from being larger than the largest representable value. 1674 * 1675 * This prevents incremental opening of the window in the regime 1676 * where TCP is limited by the speed of the reader side taking 1677 * data out of the TCP receive queue. It does nothing about 1678 * those cases where the window is constrained on the sender side 1679 * because the pipeline is full. 1680 * 1681 * BSD also seems to "accidentally" limit itself to windows that are a 1682 * multiple of MSS, at least until the free space gets quite small. 1683 * This would appear to be a side effect of the mbuf implementation. 1684 * Combining these two algorithms results in the observed behavior 1685 * of having a fixed window size at almost all times. 1686 * 1687 * Below we obtain similar behavior by forcing the offered window to 1688 * a multiple of the mss when it is feasible to do so. 1689 * 1690 * Note, we don't "adjust" for TIMESTAMP or SACK option bytes. 1691 * Regular options like TIMESTAMP are taken into account. 1692 */ 1693 u32 __tcp_select_window(struct sock *sk) 1694 { 1695 struct inet_connection_sock *icsk = inet_csk(sk); 1696 struct tcp_sock *tp = tcp_sk(sk); 1697 /* MSS for the peer's data. Previous versions used mss_clamp 1698 * here. I don't know if the value based on our guesses 1699 * of peer's MSS is better for the performance. It's more correct 1700 * but may be worse for the performance because of rcv_mss 1701 * fluctuations. --SAW 1998/11/1 1702 */ 1703 int mss = icsk->icsk_ack.rcv_mss; 1704 int free_space = tcp_space(sk); 1705 int full_space = min_t(int, tp->window_clamp, tcp_full_space(sk)); 1706 int window; 1707 1708 if (mss > full_space) 1709 mss = full_space; 1710 1711 if (free_space < (full_space >> 1)) { 1712 icsk->icsk_ack.quick = 0; 1713 1714 if (tcp_memory_pressure) 1715 tp->rcv_ssthresh = min(tp->rcv_ssthresh, 1716 4U * tp->advmss); 1717 1718 if (free_space < mss) 1719 return 0; 1720 } 1721 1722 if (free_space > tp->rcv_ssthresh) 1723 free_space = tp->rcv_ssthresh; 1724 1725 /* Don't do rounding if we are using window scaling, since the 1726 * scaled window will not line up with the MSS boundary anyway. 1727 */ 1728 window = tp->rcv_wnd; 1729 if (tp->rx_opt.rcv_wscale) { 1730 window = free_space; 1731 1732 /* Advertise enough space so that it won't get scaled away. 1733 * Import case: prevent zero window announcement if 1734 * 1<<rcv_wscale > mss. 1735 */ 1736 if (((window >> tp->rx_opt.rcv_wscale) << tp->rx_opt.rcv_wscale) != window) 1737 window = (((window >> tp->rx_opt.rcv_wscale) + 1) 1738 << tp->rx_opt.rcv_wscale); 1739 } else { 1740 /* Get the largest window that is a nice multiple of mss. 1741 * Window clamp already applied above. 1742 * If our current window offering is within 1 mss of the 1743 * free space we just keep it. This prevents the divide 1744 * and multiply from happening most of the time. 1745 * We also don't do any window rounding when the free space 1746 * is too small. 1747 */ 1748 if (window <= free_space - mss || window > free_space) 1749 window = (free_space / mss) * mss; 1750 else if (mss == full_space && 1751 free_space > window + (full_space >> 1)) 1752 window = free_space; 1753 } 1754 1755 return window; 1756 } 1757 1758 /* Attempt to collapse two adjacent SKB's during retransmission. */ 1759 static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *skb, 1760 int mss_now) 1761 { 1762 struct tcp_sock *tp = tcp_sk(sk); 1763 struct sk_buff *next_skb = tcp_write_queue_next(sk, skb); 1764 int skb_size, next_skb_size; 1765 u16 flags; 1766 1767 /* The first test we must make is that neither of these two 1768 * SKB's are still referenced by someone else. 1769 */ 1770 if (skb_cloned(skb) || skb_cloned(next_skb)) 1771 return; 1772 1773 skb_size = skb->len; 1774 next_skb_size = next_skb->len; 1775 flags = TCP_SKB_CB(skb)->flags; 1776 1777 /* Also punt if next skb has been SACK'd. */ 1778 if (TCP_SKB_CB(next_skb)->sacked & TCPCB_SACKED_ACKED) 1779 return; 1780 1781 /* Next skb is out of window. */ 1782 if (after(TCP_SKB_CB(next_skb)->end_seq, tcp_wnd_end(tp))) 1783 return; 1784 1785 /* Punt if not enough space exists in the first SKB for 1786 * the data in the second, or the total combined payload 1787 * would exceed the MSS. 1788 */ 1789 if ((next_skb_size > skb_tailroom(skb)) || 1790 ((skb_size + next_skb_size) > mss_now)) 1791 return; 1792 1793 BUG_ON(tcp_skb_pcount(skb) != 1 || tcp_skb_pcount(next_skb) != 1); 1794 1795 tcp_highest_sack_combine(sk, next_skb, skb); 1796 1797 /* Ok. We will be able to collapse the packet. */ 1798 tcp_unlink_write_queue(next_skb, sk); 1799 1800 skb_copy_from_linear_data(next_skb, skb_put(skb, next_skb_size), 1801 next_skb_size); 1802 1803 if (next_skb->ip_summed == CHECKSUM_PARTIAL) 1804 skb->ip_summed = CHECKSUM_PARTIAL; 1805 1806 if (skb->ip_summed != CHECKSUM_PARTIAL) 1807 skb->csum = csum_block_add(skb->csum, next_skb->csum, skb_size); 1808 1809 /* Update sequence range on original skb. */ 1810 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(next_skb)->end_seq; 1811 1812 /* Merge over control information. */ 1813 flags |= TCP_SKB_CB(next_skb)->flags; /* This moves PSH/FIN etc. over */ 1814 TCP_SKB_CB(skb)->flags = flags; 1815 1816 /* All done, get rid of second SKB and account for it so 1817 * packet counting does not break. 1818 */ 1819 TCP_SKB_CB(skb)->sacked |= TCP_SKB_CB(next_skb)->sacked & TCPCB_EVER_RETRANS; 1820 if (TCP_SKB_CB(next_skb)->sacked & TCPCB_SACKED_RETRANS) 1821 tp->retrans_out -= tcp_skb_pcount(next_skb); 1822 if (TCP_SKB_CB(next_skb)->sacked & TCPCB_LOST) 1823 tp->lost_out -= tcp_skb_pcount(next_skb); 1824 /* Reno case is special. Sigh... */ 1825 if (tcp_is_reno(tp) && tp->sacked_out) 1826 tcp_dec_pcount_approx(&tp->sacked_out, next_skb); 1827 1828 tcp_adjust_fackets_out(sk, next_skb, tcp_skb_pcount(next_skb)); 1829 tp->packets_out -= tcp_skb_pcount(next_skb); 1830 1831 /* changed transmit queue under us so clear hints */ 1832 tcp_clear_retrans_hints_partial(tp); 1833 if (next_skb == tp->retransmit_skb_hint) 1834 tp->retransmit_skb_hint = skb; 1835 1836 sk_wmem_free_skb(sk, next_skb); 1837 } 1838 1839 /* Do a simple retransmit without using the backoff mechanisms in 1840 * tcp_timer. This is used for path mtu discovery. 1841 * The socket is already locked here. 1842 */ 1843 void tcp_simple_retransmit(struct sock *sk) 1844 { 1845 const struct inet_connection_sock *icsk = inet_csk(sk); 1846 struct tcp_sock *tp = tcp_sk(sk); 1847 struct sk_buff *skb; 1848 unsigned int mss = tcp_current_mss(sk, 0); 1849 u32 prior_lost = tp->lost_out; 1850 1851 tcp_for_write_queue(skb, sk) { 1852 if (skb == tcp_send_head(sk)) 1853 break; 1854 if (skb->len > mss && 1855 !(TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)) { 1856 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) { 1857 TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_RETRANS; 1858 tp->retrans_out -= tcp_skb_pcount(skb); 1859 } 1860 tcp_skb_mark_lost_uncond_verify(tp, skb); 1861 } 1862 } 1863 1864 tcp_clear_retrans_hints_partial(tp); 1865 1866 if (prior_lost == tp->lost_out) 1867 return; 1868 1869 if (tcp_is_reno(tp)) 1870 tcp_limit_reno_sacked(tp); 1871 1872 tcp_verify_left_out(tp); 1873 1874 /* Don't muck with the congestion window here. 1875 * Reason is that we do not increase amount of _data_ 1876 * in network, but units changed and effective 1877 * cwnd/ssthresh really reduced now. 1878 */ 1879 if (icsk->icsk_ca_state != TCP_CA_Loss) { 1880 tp->high_seq = tp->snd_nxt; 1881 tp->snd_ssthresh = tcp_current_ssthresh(sk); 1882 tp->prior_ssthresh = 0; 1883 tp->undo_marker = 0; 1884 tcp_set_ca_state(sk, TCP_CA_Loss); 1885 } 1886 tcp_xmit_retransmit_queue(sk); 1887 } 1888 1889 /* This retransmits one SKB. Policy decisions and retransmit queue 1890 * state updates are done by the caller. Returns non-zero if an 1891 * error occurred which prevented the send. 1892 */ 1893 int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb) 1894 { 1895 struct tcp_sock *tp = tcp_sk(sk); 1896 struct inet_connection_sock *icsk = inet_csk(sk); 1897 unsigned int cur_mss; 1898 int err; 1899 1900 /* Inconslusive MTU probe */ 1901 if (icsk->icsk_mtup.probe_size) { 1902 icsk->icsk_mtup.probe_size = 0; 1903 } 1904 1905 /* Do not sent more than we queued. 1/4 is reserved for possible 1906 * copying overhead: fragmentation, tunneling, mangling etc. 1907 */ 1908 if (atomic_read(&sk->sk_wmem_alloc) > 1909 min(sk->sk_wmem_queued + (sk->sk_wmem_queued >> 2), sk->sk_sndbuf)) 1910 return -EAGAIN; 1911 1912 if (before(TCP_SKB_CB(skb)->seq, tp->snd_una)) { 1913 if (before(TCP_SKB_CB(skb)->end_seq, tp->snd_una)) 1914 BUG(); 1915 if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq)) 1916 return -ENOMEM; 1917 } 1918 1919 if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk)) 1920 return -EHOSTUNREACH; /* Routing failure or similar. */ 1921 1922 cur_mss = tcp_current_mss(sk, 0); 1923 1924 /* If receiver has shrunk his window, and skb is out of 1925 * new window, do not retransmit it. The exception is the 1926 * case, when window is shrunk to zero. In this case 1927 * our retransmit serves as a zero window probe. 1928 */ 1929 if (!before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp)) 1930 && TCP_SKB_CB(skb)->seq != tp->snd_una) 1931 return -EAGAIN; 1932 1933 if (skb->len > cur_mss) { 1934 if (tcp_fragment(sk, skb, cur_mss, cur_mss)) 1935 return -ENOMEM; /* We'll try again later. */ 1936 } 1937 1938 /* Collapse two adjacent packets if worthwhile and we can. */ 1939 if (!(TCP_SKB_CB(skb)->flags & TCPCB_FLAG_SYN) && 1940 (skb->len < (cur_mss >> 1)) && 1941 (!tcp_skb_is_last(sk, skb)) && 1942 (tcp_write_queue_next(sk, skb) != tcp_send_head(sk)) && 1943 (skb_shinfo(skb)->nr_frags == 0 && 1944 skb_shinfo(tcp_write_queue_next(sk, skb))->nr_frags == 0) && 1945 (tcp_skb_pcount(skb) == 1 && 1946 tcp_skb_pcount(tcp_write_queue_next(sk, skb)) == 1) && 1947 (sysctl_tcp_retrans_collapse != 0)) 1948 tcp_retrans_try_collapse(sk, skb, cur_mss); 1949 1950 /* Some Solaris stacks overoptimize and ignore the FIN on a 1951 * retransmit when old data is attached. So strip it off 1952 * since it is cheap to do so and saves bytes on the network. 1953 */ 1954 if (skb->len > 0 && 1955 (TCP_SKB_CB(skb)->flags & TCPCB_FLAG_FIN) && 1956 tp->snd_una == (TCP_SKB_CB(skb)->end_seq - 1)) { 1957 if (!pskb_trim(skb, 0)) { 1958 /* Reuse, even though it does some unnecessary work */ 1959 tcp_init_nondata_skb(skb, TCP_SKB_CB(skb)->end_seq - 1, 1960 TCP_SKB_CB(skb)->flags); 1961 skb->ip_summed = CHECKSUM_NONE; 1962 } 1963 } 1964 1965 /* Make a copy, if the first transmission SKB clone we made 1966 * is still in somebody's hands, else make a clone. 1967 */ 1968 TCP_SKB_CB(skb)->when = tcp_time_stamp; 1969 1970 err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC); 1971 1972 if (err == 0) { 1973 /* Update global TCP statistics. */ 1974 TCP_INC_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS); 1975 1976 tp->total_retrans++; 1977 1978 #if FASTRETRANS_DEBUG > 0 1979 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) { 1980 if (net_ratelimit()) 1981 printk(KERN_DEBUG "retrans_out leaked.\n"); 1982 } 1983 #endif 1984 if (!tp->retrans_out) 1985 tp->lost_retrans_low = tp->snd_nxt; 1986 TCP_SKB_CB(skb)->sacked |= TCPCB_RETRANS; 1987 tp->retrans_out += tcp_skb_pcount(skb); 1988 1989 /* Save stamp of the first retransmit. */ 1990 if (!tp->retrans_stamp) 1991 tp->retrans_stamp = TCP_SKB_CB(skb)->when; 1992 1993 tp->undo_retrans++; 1994 1995 /* snd_nxt is stored to detect loss of retransmitted segment, 1996 * see tcp_input.c tcp_sacktag_write_queue(). 1997 */ 1998 TCP_SKB_CB(skb)->ack_seq = tp->snd_nxt; 1999 } 2000 return err; 2001 } 2002 2003 static int tcp_can_forward_retransmit(struct sock *sk) 2004 { 2005 const struct inet_connection_sock *icsk = inet_csk(sk); 2006 struct tcp_sock *tp = tcp_sk(sk); 2007 2008 /* Forward retransmissions are possible only during Recovery. */ 2009 if (icsk->icsk_ca_state != TCP_CA_Recovery) 2010 return 0; 2011 2012 /* No forward retransmissions in Reno are possible. */ 2013 if (tcp_is_reno(tp)) 2014 return 0; 2015 2016 /* Yeah, we have to make difficult choice between forward transmission 2017 * and retransmission... Both ways have their merits... 2018 * 2019 * For now we do not retransmit anything, while we have some new 2020 * segments to send. In the other cases, follow rule 3 for 2021 * NextSeg() specified in RFC3517. 2022 */ 2023 2024 if (tcp_may_send_now(sk)) 2025 return 0; 2026 2027 return 1; 2028 } 2029 2030 /* This gets called after a retransmit timeout, and the initially 2031 * retransmitted data is acknowledged. It tries to continue 2032 * resending the rest of the retransmit queue, until either 2033 * we've sent it all or the congestion window limit is reached. 2034 * If doing SACK, the first ACK which comes back for a timeout 2035 * based retransmit packet might feed us FACK information again. 2036 * If so, we use it to avoid unnecessarily retransmissions. 2037 */ 2038 void tcp_xmit_retransmit_queue(struct sock *sk) 2039 { 2040 const struct inet_connection_sock *icsk = inet_csk(sk); 2041 struct tcp_sock *tp = tcp_sk(sk); 2042 struct sk_buff *skb; 2043 struct sk_buff *hole = NULL; 2044 u32 last_lost; 2045 int mib_idx; 2046 int fwd_rexmitting = 0; 2047 2048 if (!tp->lost_out) 2049 tp->retransmit_high = tp->snd_una; 2050 2051 if (tp->retransmit_skb_hint) { 2052 skb = tp->retransmit_skb_hint; 2053 last_lost = TCP_SKB_CB(skb)->end_seq; 2054 if (after(last_lost, tp->retransmit_high)) 2055 last_lost = tp->retransmit_high; 2056 } else { 2057 skb = tcp_write_queue_head(sk); 2058 last_lost = tp->snd_una; 2059 } 2060 2061 /* First pass: retransmit lost packets. */ 2062 tcp_for_write_queue_from(skb, sk) { 2063 __u8 sacked = TCP_SKB_CB(skb)->sacked; 2064 2065 if (skb == tcp_send_head(sk)) 2066 break; 2067 /* we could do better than to assign each time */ 2068 if (hole == NULL) 2069 tp->retransmit_skb_hint = skb; 2070 2071 /* Assume this retransmit will generate 2072 * only one packet for congestion window 2073 * calculation purposes. This works because 2074 * tcp_retransmit_skb() will chop up the 2075 * packet to be MSS sized and all the 2076 * packet counting works out. 2077 */ 2078 if (tcp_packets_in_flight(tp) >= tp->snd_cwnd) 2079 return; 2080 2081 if (fwd_rexmitting) { 2082 begin_fwd: 2083 if (!before(TCP_SKB_CB(skb)->seq, tcp_highest_sack_seq(tp))) 2084 break; 2085 mib_idx = LINUX_MIB_TCPFORWARDRETRANS; 2086 2087 } else if (!before(TCP_SKB_CB(skb)->seq, tp->retransmit_high)) { 2088 tp->retransmit_high = last_lost; 2089 if (!tcp_can_forward_retransmit(sk)) 2090 break; 2091 /* Backtrack if necessary to non-L'ed skb */ 2092 if (hole != NULL) { 2093 skb = hole; 2094 hole = NULL; 2095 } 2096 fwd_rexmitting = 1; 2097 goto begin_fwd; 2098 2099 } else if (!(sacked & TCPCB_LOST)) { 2100 if (hole == NULL && !(sacked & TCPCB_SACKED_RETRANS)) 2101 hole = skb; 2102 continue; 2103 2104 } else { 2105 last_lost = TCP_SKB_CB(skb)->end_seq; 2106 if (icsk->icsk_ca_state != TCP_CA_Loss) 2107 mib_idx = LINUX_MIB_TCPFASTRETRANS; 2108 else 2109 mib_idx = LINUX_MIB_TCPSLOWSTARTRETRANS; 2110 } 2111 2112 if (sacked & (TCPCB_SACKED_ACKED|TCPCB_SACKED_RETRANS)) 2113 continue; 2114 2115 if (tcp_retransmit_skb(sk, skb)) 2116 return; 2117 NET_INC_STATS_BH(sock_net(sk), mib_idx); 2118 2119 if (skb == tcp_write_queue_head(sk)) 2120 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, 2121 inet_csk(sk)->icsk_rto, 2122 TCP_RTO_MAX); 2123 } 2124 } 2125 2126 /* Send a fin. The caller locks the socket for us. This cannot be 2127 * allowed to fail queueing a FIN frame under any circumstances. 2128 */ 2129 void tcp_send_fin(struct sock *sk) 2130 { 2131 struct tcp_sock *tp = tcp_sk(sk); 2132 struct sk_buff *skb = tcp_write_queue_tail(sk); 2133 int mss_now; 2134 2135 /* Optimization, tack on the FIN if we have a queue of 2136 * unsent frames. But be careful about outgoing SACKS 2137 * and IP options. 2138 */ 2139 mss_now = tcp_current_mss(sk, 1); 2140 2141 if (tcp_send_head(sk) != NULL) { 2142 TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_FIN; 2143 TCP_SKB_CB(skb)->end_seq++; 2144 tp->write_seq++; 2145 } else { 2146 /* Socket is locked, keep trying until memory is available. */ 2147 for (;;) { 2148 skb = alloc_skb_fclone(MAX_TCP_HEADER, GFP_KERNEL); 2149 if (skb) 2150 break; 2151 yield(); 2152 } 2153 2154 /* Reserve space for headers and prepare control bits. */ 2155 skb_reserve(skb, MAX_TCP_HEADER); 2156 /* FIN eats a sequence byte, write_seq advanced by tcp_queue_skb(). */ 2157 tcp_init_nondata_skb(skb, tp->write_seq, 2158 TCPCB_FLAG_ACK | TCPCB_FLAG_FIN); 2159 tcp_queue_skb(sk, skb); 2160 } 2161 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_OFF); 2162 } 2163 2164 /* We get here when a process closes a file descriptor (either due to 2165 * an explicit close() or as a byproduct of exit()'ing) and there 2166 * was unread data in the receive queue. This behavior is recommended 2167 * by RFC 2525, section 2.17. -DaveM 2168 */ 2169 void tcp_send_active_reset(struct sock *sk, gfp_t priority) 2170 { 2171 struct sk_buff *skb; 2172 2173 /* NOTE: No TCP options attached and we never retransmit this. */ 2174 skb = alloc_skb(MAX_TCP_HEADER, priority); 2175 if (!skb) { 2176 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED); 2177 return; 2178 } 2179 2180 /* Reserve space for headers and prepare control bits. */ 2181 skb_reserve(skb, MAX_TCP_HEADER); 2182 tcp_init_nondata_skb(skb, tcp_acceptable_seq(sk), 2183 TCPCB_FLAG_ACK | TCPCB_FLAG_RST); 2184 /* Send it off. */ 2185 TCP_SKB_CB(skb)->when = tcp_time_stamp; 2186 if (tcp_transmit_skb(sk, skb, 0, priority)) 2187 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED); 2188 2189 TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTRSTS); 2190 } 2191 2192 /* WARNING: This routine must only be called when we have already sent 2193 * a SYN packet that crossed the incoming SYN that caused this routine 2194 * to get called. If this assumption fails then the initial rcv_wnd 2195 * and rcv_wscale values will not be correct. 2196 */ 2197 int tcp_send_synack(struct sock *sk) 2198 { 2199 struct sk_buff *skb; 2200 2201 skb = tcp_write_queue_head(sk); 2202 if (skb == NULL || !(TCP_SKB_CB(skb)->flags & TCPCB_FLAG_SYN)) { 2203 printk(KERN_DEBUG "tcp_send_synack: wrong queue state\n"); 2204 return -EFAULT; 2205 } 2206 if (!(TCP_SKB_CB(skb)->flags & TCPCB_FLAG_ACK)) { 2207 if (skb_cloned(skb)) { 2208 struct sk_buff *nskb = skb_copy(skb, GFP_ATOMIC); 2209 if (nskb == NULL) 2210 return -ENOMEM; 2211 tcp_unlink_write_queue(skb, sk); 2212 skb_header_release(nskb); 2213 __tcp_add_write_queue_head(sk, nskb); 2214 sk_wmem_free_skb(sk, skb); 2215 sk->sk_wmem_queued += nskb->truesize; 2216 sk_mem_charge(sk, nskb->truesize); 2217 skb = nskb; 2218 } 2219 2220 TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_ACK; 2221 TCP_ECN_send_synack(tcp_sk(sk), skb); 2222 } 2223 TCP_SKB_CB(skb)->when = tcp_time_stamp; 2224 return tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC); 2225 } 2226 2227 /* 2228 * Prepare a SYN-ACK. 2229 */ 2230 struct sk_buff *tcp_make_synack(struct sock *sk, struct dst_entry *dst, 2231 struct request_sock *req) 2232 { 2233 struct inet_request_sock *ireq = inet_rsk(req); 2234 struct tcp_sock *tp = tcp_sk(sk); 2235 struct tcphdr *th; 2236 int tcp_header_size; 2237 struct tcp_out_options opts; 2238 struct sk_buff *skb; 2239 struct tcp_md5sig_key *md5; 2240 __u8 *md5_hash_location; 2241 int mss; 2242 2243 skb = sock_wmalloc(sk, MAX_TCP_HEADER + 15, 1, GFP_ATOMIC); 2244 if (skb == NULL) 2245 return NULL; 2246 2247 /* Reserve space for headers. */ 2248 skb_reserve(skb, MAX_TCP_HEADER); 2249 2250 skb->dst = dst_clone(dst); 2251 2252 mss = dst_metric(dst, RTAX_ADVMSS); 2253 if (tp->rx_opt.user_mss && tp->rx_opt.user_mss < mss) 2254 mss = tp->rx_opt.user_mss; 2255 2256 if (req->rcv_wnd == 0) { /* ignored for retransmitted syns */ 2257 __u8 rcv_wscale; 2258 /* Set this up on the first call only */ 2259 req->window_clamp = tp->window_clamp ? : dst_metric(dst, RTAX_WINDOW); 2260 /* tcp_full_space because it is guaranteed to be the first packet */ 2261 tcp_select_initial_window(tcp_full_space(sk), 2262 mss - (ireq->tstamp_ok ? TCPOLEN_TSTAMP_ALIGNED : 0), 2263 &req->rcv_wnd, 2264 &req->window_clamp, 2265 ireq->wscale_ok, 2266 &rcv_wscale); 2267 ireq->rcv_wscale = rcv_wscale; 2268 } 2269 2270 memset(&opts, 0, sizeof(opts)); 2271 TCP_SKB_CB(skb)->when = tcp_time_stamp; 2272 tcp_header_size = tcp_synack_options(sk, req, mss, 2273 skb, &opts, &md5) + 2274 sizeof(struct tcphdr); 2275 2276 skb_push(skb, tcp_header_size); 2277 skb_reset_transport_header(skb); 2278 2279 th = tcp_hdr(skb); 2280 memset(th, 0, sizeof(struct tcphdr)); 2281 th->syn = 1; 2282 th->ack = 1; 2283 TCP_ECN_make_synack(req, th); 2284 th->source = ireq->loc_port; 2285 th->dest = ireq->rmt_port; 2286 /* Setting of flags are superfluous here for callers (and ECE is 2287 * not even correctly set) 2288 */ 2289 tcp_init_nondata_skb(skb, tcp_rsk(req)->snt_isn, 2290 TCPCB_FLAG_SYN | TCPCB_FLAG_ACK); 2291 th->seq = htonl(TCP_SKB_CB(skb)->seq); 2292 th->ack_seq = htonl(tcp_rsk(req)->rcv_isn + 1); 2293 2294 /* RFC1323: The window in SYN & SYN/ACK segments is never scaled. */ 2295 th->window = htons(min(req->rcv_wnd, 65535U)); 2296 #ifdef CONFIG_SYN_COOKIES 2297 if (unlikely(req->cookie_ts)) 2298 TCP_SKB_CB(skb)->when = cookie_init_timestamp(req); 2299 else 2300 #endif 2301 tcp_options_write((__be32 *)(th + 1), tp, &opts, &md5_hash_location); 2302 th->doff = (tcp_header_size >> 2); 2303 TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTSEGS); 2304 2305 #ifdef CONFIG_TCP_MD5SIG 2306 /* Okay, we have all we need - do the md5 hash if needed */ 2307 if (md5) { 2308 tp->af_specific->calc_md5_hash(md5_hash_location, 2309 md5, NULL, req, skb); 2310 } 2311 #endif 2312 2313 return skb; 2314 } 2315 2316 /* 2317 * Do all connect socket setups that can be done AF independent. 2318 */ 2319 static void tcp_connect_init(struct sock *sk) 2320 { 2321 struct dst_entry *dst = __sk_dst_get(sk); 2322 struct tcp_sock *tp = tcp_sk(sk); 2323 __u8 rcv_wscale; 2324 2325 /* We'll fix this up when we get a response from the other end. 2326 * See tcp_input.c:tcp_rcv_state_process case TCP_SYN_SENT. 2327 */ 2328 tp->tcp_header_len = sizeof(struct tcphdr) + 2329 (sysctl_tcp_timestamps ? TCPOLEN_TSTAMP_ALIGNED : 0); 2330 2331 #ifdef CONFIG_TCP_MD5SIG 2332 if (tp->af_specific->md5_lookup(sk, sk) != NULL) 2333 tp->tcp_header_len += TCPOLEN_MD5SIG_ALIGNED; 2334 #endif 2335 2336 /* If user gave his TCP_MAXSEG, record it to clamp */ 2337 if (tp->rx_opt.user_mss) 2338 tp->rx_opt.mss_clamp = tp->rx_opt.user_mss; 2339 tp->max_window = 0; 2340 tcp_mtup_init(sk); 2341 tcp_sync_mss(sk, dst_mtu(dst)); 2342 2343 if (!tp->window_clamp) 2344 tp->window_clamp = dst_metric(dst, RTAX_WINDOW); 2345 tp->advmss = dst_metric(dst, RTAX_ADVMSS); 2346 if (tp->rx_opt.user_mss && tp->rx_opt.user_mss < tp->advmss) 2347 tp->advmss = tp->rx_opt.user_mss; 2348 2349 tcp_initialize_rcv_mss(sk); 2350 2351 tcp_select_initial_window(tcp_full_space(sk), 2352 tp->advmss - (tp->rx_opt.ts_recent_stamp ? tp->tcp_header_len - sizeof(struct tcphdr) : 0), 2353 &tp->rcv_wnd, 2354 &tp->window_clamp, 2355 sysctl_tcp_window_scaling, 2356 &rcv_wscale); 2357 2358 tp->rx_opt.rcv_wscale = rcv_wscale; 2359 tp->rcv_ssthresh = tp->rcv_wnd; 2360 2361 sk->sk_err = 0; 2362 sock_reset_flag(sk, SOCK_DONE); 2363 tp->snd_wnd = 0; 2364 tcp_init_wl(tp, tp->write_seq, 0); 2365 tp->snd_una = tp->write_seq; 2366 tp->snd_sml = tp->write_seq; 2367 tp->snd_up = tp->write_seq; 2368 tp->rcv_nxt = 0; 2369 tp->rcv_wup = 0; 2370 tp->copied_seq = 0; 2371 2372 inet_csk(sk)->icsk_rto = TCP_TIMEOUT_INIT; 2373 inet_csk(sk)->icsk_retransmits = 0; 2374 tcp_clear_retrans(tp); 2375 } 2376 2377 /* 2378 * Build a SYN and send it off. 2379 */ 2380 int tcp_connect(struct sock *sk) 2381 { 2382 struct tcp_sock *tp = tcp_sk(sk); 2383 struct sk_buff *buff; 2384 2385 tcp_connect_init(sk); 2386 2387 buff = alloc_skb_fclone(MAX_TCP_HEADER + 15, sk->sk_allocation); 2388 if (unlikely(buff == NULL)) 2389 return -ENOBUFS; 2390 2391 /* Reserve space for headers. */ 2392 skb_reserve(buff, MAX_TCP_HEADER); 2393 2394 tp->snd_nxt = tp->write_seq; 2395 tcp_init_nondata_skb(buff, tp->write_seq++, TCPCB_FLAG_SYN); 2396 TCP_ECN_send_syn(sk, buff); 2397 2398 /* Send it off. */ 2399 TCP_SKB_CB(buff)->when = tcp_time_stamp; 2400 tp->retrans_stamp = TCP_SKB_CB(buff)->when; 2401 skb_header_release(buff); 2402 __tcp_add_write_queue_tail(sk, buff); 2403 sk->sk_wmem_queued += buff->truesize; 2404 sk_mem_charge(sk, buff->truesize); 2405 tp->packets_out += tcp_skb_pcount(buff); 2406 tcp_transmit_skb(sk, buff, 1, GFP_KERNEL); 2407 2408 /* We change tp->snd_nxt after the tcp_transmit_skb() call 2409 * in order to make this packet get counted in tcpOutSegs. 2410 */ 2411 tp->snd_nxt = tp->write_seq; 2412 tp->pushed_seq = tp->write_seq; 2413 TCP_INC_STATS(sock_net(sk), TCP_MIB_ACTIVEOPENS); 2414 2415 /* Timer for repeating the SYN until an answer. */ 2416 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, 2417 inet_csk(sk)->icsk_rto, TCP_RTO_MAX); 2418 return 0; 2419 } 2420 2421 /* Send out a delayed ack, the caller does the policy checking 2422 * to see if we should even be here. See tcp_input.c:tcp_ack_snd_check() 2423 * for details. 2424 */ 2425 void tcp_send_delayed_ack(struct sock *sk) 2426 { 2427 struct inet_connection_sock *icsk = inet_csk(sk); 2428 int ato = icsk->icsk_ack.ato; 2429 unsigned long timeout; 2430 2431 if (ato > TCP_DELACK_MIN) { 2432 const struct tcp_sock *tp = tcp_sk(sk); 2433 int max_ato = HZ / 2; 2434 2435 if (icsk->icsk_ack.pingpong || 2436 (icsk->icsk_ack.pending & ICSK_ACK_PUSHED)) 2437 max_ato = TCP_DELACK_MAX; 2438 2439 /* Slow path, intersegment interval is "high". */ 2440 2441 /* If some rtt estimate is known, use it to bound delayed ack. 2442 * Do not use inet_csk(sk)->icsk_rto here, use results of rtt measurements 2443 * directly. 2444 */ 2445 if (tp->srtt) { 2446 int rtt = max(tp->srtt >> 3, TCP_DELACK_MIN); 2447 2448 if (rtt < max_ato) 2449 max_ato = rtt; 2450 } 2451 2452 ato = min(ato, max_ato); 2453 } 2454 2455 /* Stay within the limit we were given */ 2456 timeout = jiffies + ato; 2457 2458 /* Use new timeout only if there wasn't a older one earlier. */ 2459 if (icsk->icsk_ack.pending & ICSK_ACK_TIMER) { 2460 /* If delack timer was blocked or is about to expire, 2461 * send ACK now. 2462 */ 2463 if (icsk->icsk_ack.blocked || 2464 time_before_eq(icsk->icsk_ack.timeout, jiffies + (ato >> 2))) { 2465 tcp_send_ack(sk); 2466 return; 2467 } 2468 2469 if (!time_before(timeout, icsk->icsk_ack.timeout)) 2470 timeout = icsk->icsk_ack.timeout; 2471 } 2472 icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER; 2473 icsk->icsk_ack.timeout = timeout; 2474 sk_reset_timer(sk, &icsk->icsk_delack_timer, timeout); 2475 } 2476 2477 /* This routine sends an ack and also updates the window. */ 2478 void tcp_send_ack(struct sock *sk) 2479 { 2480 struct sk_buff *buff; 2481 2482 /* If we have been reset, we may not send again. */ 2483 if (sk->sk_state == TCP_CLOSE) 2484 return; 2485 2486 /* We are not putting this on the write queue, so 2487 * tcp_transmit_skb() will set the ownership to this 2488 * sock. 2489 */ 2490 buff = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC); 2491 if (buff == NULL) { 2492 inet_csk_schedule_ack(sk); 2493 inet_csk(sk)->icsk_ack.ato = TCP_ATO_MIN; 2494 inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK, 2495 TCP_DELACK_MAX, TCP_RTO_MAX); 2496 return; 2497 } 2498 2499 /* Reserve space for headers and prepare control bits. */ 2500 skb_reserve(buff, MAX_TCP_HEADER); 2501 tcp_init_nondata_skb(buff, tcp_acceptable_seq(sk), TCPCB_FLAG_ACK); 2502 2503 /* Send it off, this clears delayed acks for us. */ 2504 TCP_SKB_CB(buff)->when = tcp_time_stamp; 2505 tcp_transmit_skb(sk, buff, 0, GFP_ATOMIC); 2506 } 2507 2508 /* This routine sends a packet with an out of date sequence 2509 * number. It assumes the other end will try to ack it. 2510 * 2511 * Question: what should we make while urgent mode? 2512 * 4.4BSD forces sending single byte of data. We cannot send 2513 * out of window data, because we have SND.NXT==SND.MAX... 2514 * 2515 * Current solution: to send TWO zero-length segments in urgent mode: 2516 * one is with SEG.SEQ=SND.UNA to deliver urgent pointer, another is 2517 * out-of-date with SND.UNA-1 to probe window. 2518 */ 2519 static int tcp_xmit_probe_skb(struct sock *sk, int urgent) 2520 { 2521 struct tcp_sock *tp = tcp_sk(sk); 2522 struct sk_buff *skb; 2523 2524 /* We don't queue it, tcp_transmit_skb() sets ownership. */ 2525 skb = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC); 2526 if (skb == NULL) 2527 return -1; 2528 2529 /* Reserve space for headers and set control bits. */ 2530 skb_reserve(skb, MAX_TCP_HEADER); 2531 /* Use a previous sequence. This should cause the other 2532 * end to send an ack. Don't queue or clone SKB, just 2533 * send it. 2534 */ 2535 tcp_init_nondata_skb(skb, tp->snd_una - !urgent, TCPCB_FLAG_ACK); 2536 TCP_SKB_CB(skb)->when = tcp_time_stamp; 2537 return tcp_transmit_skb(sk, skb, 0, GFP_ATOMIC); 2538 } 2539 2540 int tcp_write_wakeup(struct sock *sk) 2541 { 2542 struct tcp_sock *tp = tcp_sk(sk); 2543 struct sk_buff *skb; 2544 2545 if (sk->sk_state == TCP_CLOSE) 2546 return -1; 2547 2548 if ((skb = tcp_send_head(sk)) != NULL && 2549 before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp))) { 2550 int err; 2551 unsigned int mss = tcp_current_mss(sk, 0); 2552 unsigned int seg_size = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq; 2553 2554 if (before(tp->pushed_seq, TCP_SKB_CB(skb)->end_seq)) 2555 tp->pushed_seq = TCP_SKB_CB(skb)->end_seq; 2556 2557 /* We are probing the opening of a window 2558 * but the window size is != 0 2559 * must have been a result SWS avoidance ( sender ) 2560 */ 2561 if (seg_size < TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq || 2562 skb->len > mss) { 2563 seg_size = min(seg_size, mss); 2564 TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_PSH; 2565 if (tcp_fragment(sk, skb, seg_size, mss)) 2566 return -1; 2567 } else if (!tcp_skb_pcount(skb)) 2568 tcp_set_skb_tso_segs(sk, skb, mss); 2569 2570 TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_PSH; 2571 TCP_SKB_CB(skb)->when = tcp_time_stamp; 2572 err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC); 2573 if (!err) 2574 tcp_event_new_data_sent(sk, skb); 2575 return err; 2576 } else { 2577 if (between(tp->snd_up, tp->snd_una + 1, tp->snd_una + 0xFFFF)) 2578 tcp_xmit_probe_skb(sk, 1); 2579 return tcp_xmit_probe_skb(sk, 0); 2580 } 2581 } 2582 2583 /* A window probe timeout has occurred. If window is not closed send 2584 * a partial packet else a zero probe. 2585 */ 2586 void tcp_send_probe0(struct sock *sk) 2587 { 2588 struct inet_connection_sock *icsk = inet_csk(sk); 2589 struct tcp_sock *tp = tcp_sk(sk); 2590 int err; 2591 2592 err = tcp_write_wakeup(sk); 2593 2594 if (tp->packets_out || !tcp_send_head(sk)) { 2595 /* Cancel probe timer, if it is not required. */ 2596 icsk->icsk_probes_out = 0; 2597 icsk->icsk_backoff = 0; 2598 return; 2599 } 2600 2601 if (err <= 0) { 2602 if (icsk->icsk_backoff < sysctl_tcp_retries2) 2603 icsk->icsk_backoff++; 2604 icsk->icsk_probes_out++; 2605 inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0, 2606 min(icsk->icsk_rto << icsk->icsk_backoff, TCP_RTO_MAX), 2607 TCP_RTO_MAX); 2608 } else { 2609 /* If packet was not sent due to local congestion, 2610 * do not backoff and do not remember icsk_probes_out. 2611 * Let local senders to fight for local resources. 2612 * 2613 * Use accumulated backoff yet. 2614 */ 2615 if (!icsk->icsk_probes_out) 2616 icsk->icsk_probes_out = 1; 2617 inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0, 2618 min(icsk->icsk_rto << icsk->icsk_backoff, 2619 TCP_RESOURCE_PROBE_INTERVAL), 2620 TCP_RTO_MAX); 2621 } 2622 } 2623 2624 EXPORT_SYMBOL(tcp_select_initial_window); 2625 EXPORT_SYMBOL(tcp_connect); 2626 EXPORT_SYMBOL(tcp_make_synack); 2627 EXPORT_SYMBOL(tcp_simple_retransmit); 2628 EXPORT_SYMBOL(tcp_sync_mss); 2629 EXPORT_SYMBOL(tcp_mtup_init); 2630