Lines Matching +full:pull +full:- +full:up +full:- +full:adv
1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
139 * non-ACK.
157 if (V_tcp_hhh[HHOOK_TCP_EST_OUT]->hhh_nhooks > 0) { in hhook_run_tcp_est_out()
165 &tp->t_osd); in hhook_run_tcp_est_out()
178 if (CC_ALGO(tp)->after_idle != NULL) in cc_after_idle()
179 CC_ALGO(tp)->after_idle(&tp->t_ccv); in cc_after_idle()
214 const bool isipv6 = (inp->inp_vflag & INP_IPV6) != 0; in tcp_default_output()
217 const bool hw_tls = tp->t_nic_ktls_xmit != 0; in tcp_default_output()
226 if (tp->t_flags & TF_TOE) in tcp_default_output()
235 if ((tp->t_flags & TF_FASTOPEN) && in tcp_default_output()
236 ((tp->t_state == TCPS_SYN_SENT) || in tcp_default_output()
237 (tp->t_state == TCPS_SYN_RECEIVED)) && in tcp_default_output()
238 SEQ_GT(tp->snd_max, tp->snd_una) && /* SYN or SYN|ACK sent */ in tcp_default_output()
239 (tp->snd_nxt != tp->snd_una)) /* not a retransmit */ in tcp_default_output()
248 idle = (tp->t_flags & TF_LASTIDLE) || (tp->snd_max == tp->snd_una); in tcp_default_output()
249 if (idle && (((ticks - tp->t_rcvtime) >= tp->t_rxtcur) || in tcp_default_output()
250 (tp->t_sndtime && ((ticks - tp->t_sndtime) >= tp->t_rxtcur)))) in tcp_default_output()
252 tp->t_flags &= ~TF_LASTIDLE; in tcp_default_output()
254 if (tp->t_flags & TF_MORETOCOME) { in tcp_default_output()
255 tp->t_flags |= TF_LASTIDLE; in tcp_default_output()
266 if ((tp->t_flags & TF_SACK_PERMIT) && in tcp_default_output()
267 (tp->sackhint.nexthole != NULL) && in tcp_default_output()
268 !IN_FASTRECOVERY(tp->t_flags)) { in tcp_default_output()
274 off = tp->snd_nxt - tp->snd_una; in tcp_default_output()
275 sendwin = min(tp->snd_wnd, tp->snd_cwnd + sendwin); in tcp_default_output()
277 flags = tcp_outflags[tp->t_state]; in tcp_default_output()
279 * Send any SACK-generated retransmissions. If we're explicitly trying in tcp_default_output()
290 if ((tp->t_flags & TF_SACK_PERMIT) && in tcp_default_output()
291 (IN_FASTRECOVERY(tp->t_flags) || in tcp_default_output()
292 (SEQ_LT(tp->snd_nxt, tp->snd_max) && (tp->t_dupacks >= tcprexmtthresh))) && in tcp_default_output()
296 if (IN_FASTRECOVERY(tp->t_flags)) { in tcp_default_output()
297 cwin = imax(sendwin - tcp_compute_pipe(tp), 0); in tcp_default_output()
299 cwin = imax(sendwin - off, 0); in tcp_default_output()
302 if (SEQ_GT(p->end, tp->snd_recover)) { in tcp_default_output()
308 if (SEQ_GEQ(p->rxmit, tp->snd_recover)) { in tcp_default_output()
313 * moves past p->rxmit. in tcp_default_output()
320 len = SEQ_SUB(tp->snd_recover, p->rxmit); in tcp_default_output()
328 len = SEQ_SUB(p->end, p->rxmit); in tcp_default_output()
336 * but sendwin (expected flightsize) - pipe didn't in tcp_default_output()
344 off = SEQ_SUB(p->rxmit, tp->snd_una); in tcp_default_output()
358 if (tp->t_flags & TF_NEEDFIN) in tcp_default_output()
360 if (tp->t_flags & TF_NEEDSYN) in tcp_default_output()
370 if (tp->t_flags & TF_FORCEDATA) { in tcp_default_output()
380 * up sending the packet without recording in tcp_default_output()
388 if (off < sbused(&so->so_snd)) in tcp_default_output()
393 tp->t_rxtshift = 0; in tcp_default_output()
400 * a negative length. This can also occur when TCP opens up in tcp_default_output()
402 * acks after fast-retransmit because TCP will reset snd_nxt in tcp_default_output()
403 * to snd_max after the fast-retransmit. in tcp_default_output()
405 * In the normal retransmit-FIN-only case, however, snd_nxt will in tcp_default_output()
407 * wind up 0. in tcp_default_output()
413 if ((sack_bytes_rxmt == 0) || SEQ_LT(tp->snd_nxt, tp->snd_max)) { in tcp_default_output()
414 len = imin(sbavail(&so->so_snd), sendwin) - off; in tcp_default_output()
422 imin(sbavail(&so->so_snd), sendwin) - in tcp_default_output()
429 * is SYN-SENT state and if segment contains data and if we don't in tcp_default_output()
432 if ((flags & TH_SYN) && SEQ_GT(tp->snd_nxt, tp->snd_una)) { in tcp_default_output()
433 if (tp->t_state != TCPS_SYN_RECEIVED) in tcp_default_output()
439 if ((tp->t_flags & TF_FASTOPEN) && in tcp_default_output()
440 (tp->t_state == TCPS_SYN_RECEIVED)) in tcp_default_output()
442 off--, len++; in tcp_default_output()
450 if ((flags & TH_SYN) && (tp->t_flags & TF_NOOPT)) { in tcp_default_output()
458 * - When retransmitting SYN|ACK on a passively-created socket in tcp_default_output()
460 * - When retransmitting SYN on an actively created socket in tcp_default_output()
462 * - When sending a zero-length cookie (cookie request) on an in tcp_default_output()
465 * - When the socket is in the CLOSED state (RST is being sent) in tcp_default_output()
467 if ((tp->t_flags & TF_FASTOPEN) && in tcp_default_output()
468 (((flags & TH_SYN) && (tp->t_rxtshift > 0)) || in tcp_default_output()
469 ((tp->t_state == TCPS_SYN_SENT) && in tcp_default_output()
470 (tp->t_tfo_client_cookie_len == 0)) || in tcp_default_output()
474 /* Without fast-open there should never be data sent on a SYN. */ in tcp_default_output()
475 if ((flags & TH_SYN) && !(tp->t_flags & TF_FASTOPEN)) { in tcp_default_output()
485 * cancel pending retransmit, pull snd_nxt back in tcp_default_output()
492 * to send, but a 0-byte window. This makes sure in tcp_default_output()
497 if ((sendwin == 0) && (TCPS_HAVEESTABLISHED(tp->t_state)) && in tcp_default_output()
498 (off < (int) sbavail(&so->so_snd)) && in tcp_default_output()
501 tp->t_rxtshift = 0; in tcp_default_output()
502 tp->snd_nxt = tp->snd_una; in tcp_default_output()
518 * presence of TCP-MD5, IP options (IPsec), and possibly SACK in tcp_default_output()
531 * Pre-calculate here as we save another lookup into the darknesses in tcp_default_output()
551 if (inp->inp_options) in tcp_default_output()
552 ipoptlen = inp->inp_options->m_len - in tcp_default_output()
558 if ((tp->t_flags & TF_TSO) && V_tcp_do_tso && len > tp->t_maxseg && in tcp_default_output()
559 (tp->t_port == 0) && in tcp_default_output()
560 ((tp->t_flags & TF_SIGNATURE) == 0) && in tcp_default_output()
563 (tp->t_flags2 & TF2_IPSEC_TSO) != 0)) && in tcp_default_output()
567 if (SEQ_LT((sack_rxmit ? p->rxmit : tp->snd_nxt) + len, in tcp_default_output()
568 tp->snd_una + sbused(&so->so_snd))) { in tcp_default_output()
572 recwin = lmin(lmax(sbspace(&so->so_rcv), 0), in tcp_default_output()
573 (long)TCP_MAXWIN << tp->rcv_scale); in tcp_default_output()
577 * conditions when len is non-zero: in tcp_default_output()
579 * - We have a full segment (or more with TSO) in tcp_default_output()
580 * - This is the last buffer in a write()/send() and we are in tcp_default_output()
582 * - we've timed out (e.g. persist timer) in tcp_default_output()
583 * - we have more then 1/2 the maximum send window's worth of in tcp_default_output()
585 * - we need to retransmit in tcp_default_output()
588 if (len >= tp->t_maxseg) in tcp_default_output()
592 * considered when setting up the initial in tcp_default_output()
598 if ((((tp->t_flags & TF_SIGNATURE) ? in tcp_default_output()
600 ((tp->t_flags & TF_RCVD_TSTMP) ? in tcp_default_output()
602 len) >= tp->t_maxseg) in tcp_default_output()
611 if (!(tp->t_flags & TF_MORETOCOME) && /* normal case */ in tcp_default_output()
612 (idle || (tp->t_flags & TF_NODELAY)) && in tcp_default_output()
613 (uint32_t)len + (uint32_t)off >= sbavail(&so->so_snd) && in tcp_default_output()
614 (tp->t_flags & TF_NOPUSH) == 0) { in tcp_default_output()
617 if (tp->t_flags & TF_FORCEDATA) /* typ. timeout case */ in tcp_default_output()
619 if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0) in tcp_default_output()
621 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) /* retransmit case */ in tcp_default_output()
651 * ACK is pending (it will get piggy-backed on it) or the in tcp_default_output()
652 * remote side already has done a half-close and won't send in tcp_default_output()
655 if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN) && in tcp_default_output()
656 !(tp->t_flags & TF_DELACK) && in tcp_default_output()
657 !TCPS_HAVERCVDFIN(tp->t_state)) { in tcp_default_output()
659 * "adv" is the amount we could increase the window, in tcp_default_output()
661 * TCP_MAXWIN << tp->rcv_scale. in tcp_default_output()
663 int32_t adv; in tcp_default_output() local
666 adv = recwin; in tcp_default_output()
667 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt)) { in tcp_default_output()
668 oldwin = (tp->rcv_adv - tp->rcv_nxt); in tcp_default_output()
669 if (adv > oldwin) in tcp_default_output()
670 adv -= oldwin; in tcp_default_output()
672 adv = 0; in tcp_default_output()
677 * If the new window size ends up being the same as or less in tcp_default_output()
681 if (oldwin >> tp->rcv_scale >= (adv + oldwin) >> tp->rcv_scale) in tcp_default_output()
684 if (adv >= (int32_t)(2 * tp->t_maxseg) && in tcp_default_output()
685 (adv >= (int32_t)(so->so_rcv.sb_hiwat / 4) || in tcp_default_output()
686 recwin <= (so->so_rcv.sb_hiwat / 8) || in tcp_default_output()
687 so->so_rcv.sb_hiwat <= 8 * tp->t_maxseg || in tcp_default_output()
688 adv >= TCP_MAXWIN << tp->rcv_scale)) in tcp_default_output()
690 if (2 * adv >= (int32_t)so->so_rcv.sb_hiwat) in tcp_default_output()
697 * is also a catch-all for the retransmit timer timeout case. in tcp_default_output()
699 if (tp->t_flags & TF_ACKNOW) in tcp_default_output()
702 ((flags & TH_SYN) && (tp->t_flags & TF_NEEDSYN) == 0)) in tcp_default_output()
704 if (SEQ_GT(tp->snd_up, tp->snd_una)) in tcp_default_output()
711 ((tp->t_flags & TF_SENTFIN) == 0 || tp->snd_nxt == tp->snd_una)) in tcp_default_output()
718 if ((tp->t_flags & TF_SACK_PERMIT) && in tcp_default_output()
719 SEQ_GT(tp->snd_max, tp->snd_una) && in tcp_default_output()
735 * (tp->t_flags & TF_FORCEDATA) in tcp_default_output()
747 if (sbavail(&so->so_snd) && !tcp_timer_active(tp, TT_REXMT) && in tcp_default_output()
749 tp->t_rxtshift = 0; in tcp_default_output()
763 if (len >= tp->t_maxseg) in tcp_default_output()
764 tp->t_flags2 |= TF2_PLPMTU_MAXSEGSNT; in tcp_default_output()
766 tp->t_flags2 &= ~TF2_PLPMTU_MAXSEGSNT; in tcp_default_output()
785 tp->snd_nxt = tp->iss; in tcp_default_output()
791 * segments. Options for SYN-ACK segments are handled in TCP in tcp_default_output()
795 if ((tp->t_flags & TF_NOOPT) == 0) { in tcp_default_output()
798 to.to_mss = tcp_mssopt(&inp->inp_inc); in tcp_default_output()
799 if (tp->t_port) in tcp_default_output()
800 to.to_mss -= V_tcp_udp_tunneling_overhead; in tcp_default_output()
810 if ((tp->t_flags & TF_FASTOPEN) && in tcp_default_output()
811 (tp->t_rxtshift == 0)) { in tcp_default_output()
812 if (tp->t_state == TCPS_SYN_RECEIVED) { in tcp_default_output()
815 (u_int8_t *)&tp->t_tfo_cookie.server; in tcp_default_output()
818 } else if (tp->t_state == TCPS_SYN_SENT) { in tcp_default_output()
820 tp->t_tfo_client_cookie_len; in tcp_default_output()
822 tp->t_tfo_cookie.client; in tcp_default_output()
826 * If we wind up having more data to in tcp_default_output()
837 if ((flags & TH_SYN) && (tp->t_flags & TF_REQ_SCALE)) { in tcp_default_output()
838 to.to_wscale = tp->request_r_scale; in tcp_default_output()
842 if ((tp->t_flags & TF_RCVD_TSTMP) || in tcp_default_output()
843 ((flags & TH_SYN) && (tp->t_flags & TF_REQ_TSTMP))) { in tcp_default_output()
845 to.to_tsval = curticks + tp->ts_offset; in tcp_default_output()
846 to.to_tsecr = tp->ts_recent; in tcp_default_output()
848 if (tp->t_rxtshift == 1) in tcp_default_output()
849 tp->t_badrxtwin = curticks; in tcp_default_output()
853 if (tp->rfbuf_ts == 0 && in tcp_default_output()
854 (so->so_rcv.sb_flags & SB_AUTOSIZE)) in tcp_default_output()
855 tp->rfbuf_ts = tcp_ts_getticks(); in tcp_default_output()
858 if (tp->t_flags & TF_SACK_PERMIT) { in tcp_default_output()
861 else if (TCPS_HAVEESTABLISHED(tp->t_state) && in tcp_default_output()
862 tp->rcv_numsacks > 0) { in tcp_default_output()
864 to.to_nsacks = tp->rcv_numsacks; in tcp_default_output()
865 to.to_sacks = (u_char *)tp->sackblks; in tcp_default_output()
869 /* TCP-MD5 (RFC2385). */ in tcp_default_output()
874 if (tp->t_flags & TF_SIGNATURE) in tcp_default_output()
884 if ((tp->t_flags & TF_FASTOPEN) && wanted_cookie && in tcp_default_output()
888 if (tp->t_port) { in tcp_default_output()
902 if (len + optlen + ipoptlen > tp->t_maxseg) { in tcp_default_output()
911 if_hw_tsomax = tp->t_tsomax; in tcp_default_output()
912 if_hw_tsomaxsegcount = tp->t_tsomaxsegcount; in tcp_default_output()
913 if_hw_tsomaxsegsize = tp->t_tsomaxsegsize; in tcp_default_output()
929 max_len = if_hw_tsomax - hdrlen - in tcp_default_output()
930 ipsec_optlen - max_linkhdr; in tcp_default_output()
944 max_len = tp->t_maxseg - optlen - ipsec_optlen; in tcp_default_output()
946 sbavail(&so->so_snd)) { in tcp_default_output()
949 len -= moff; in tcp_default_output()
971 if (tp->t_flags & TF_NEEDFIN) in tcp_default_output()
974 if (optlen + ipoptlen >= tp->t_maxseg) { in tcp_default_output()
988 len = tp->t_maxseg - optlen - ipoptlen; in tcp_default_output()
1024 if ((tp->t_flags & TF_FORCEDATA) && len == 1) { in tcp_default_output()
1027 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) in tcp_default_output()
1028 stats_voi_update_abs_u32(tp->t_stats, in tcp_default_output()
1031 stats_voi_update_abs_u64(tp->t_stats, in tcp_default_output()
1034 } else if (SEQ_LT(tp->snd_nxt, tp->snd_max) || sack_rxmit) { in tcp_default_output()
1035 tp->t_sndrexmitpack++; in tcp_default_output()
1046 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RETXPB, in tcp_default_output()
1053 stats_voi_update_abs_u64(tp->t_stats, VOI_TCP_TXPB, in tcp_default_output()
1071 m->m_data += max_linkhdr; in tcp_default_output()
1072 m->m_len = hdrlen; in tcp_default_output()
1078 mb = sbsndptr_noadv(&so->so_snd, off, &moff); in tcp_default_output()
1079 if (len <= MHLEN - hdrlen - max_linkhdr && !hw_tls) { in tcp_default_output()
1082 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) in tcp_default_output()
1083 sbsndptr_adv(&so->so_snd, mb, len); in tcp_default_output()
1084 m->m_len += len; in tcp_default_output()
1088 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) in tcp_default_output()
1091 msb = &so->so_snd; in tcp_default_output()
1093 m->m_next = tcp_m_copym(mb, moff, in tcp_default_output()
1098 if (len <= (tp->t_maxseg - optlen)) { in tcp_default_output()
1107 if (m->m_next == NULL) { in tcp_default_output()
1122 if (((uint32_t)off + (uint32_t)len == sbused(&so->so_snd)) && in tcp_default_output()
1128 if (tp->t_flags & TF_ACKNOW) in tcp_default_output()
1132 else if (SEQ_GT(tp->snd_up, tp->snd_una)) in tcp_default_output()
1149 m->m_data += max_linkhdr; in tcp_default_output()
1150 m->m_len = hdrlen; in tcp_default_output()
1153 m->m_pkthdr.rcvif = (struct ifnet *)0; in tcp_default_output()
1160 if (tp->t_port) { in tcp_default_output()
1162 udp->uh_sport = htons(V_tcp_udp_tunneling_port); in tcp_default_output()
1163 udp->uh_dport = tp->t_port; in tcp_default_output()
1164 ulen = hdrlen + len - sizeof(struct ip6_hdr); in tcp_default_output()
1165 udp->uh_ulen = htons(ulen); in tcp_default_output()
1170 tcpip_fillheaders(inp, tp->t_port, ip6, th); in tcp_default_output()
1175 if (tp->t_port) { in tcp_default_output()
1177 udp->uh_sport = htons(V_tcp_udp_tunneling_port); in tcp_default_output()
1178 udp->uh_dport = tp->t_port; in tcp_default_output()
1179 ulen = hdrlen + len - sizeof(struct ip); in tcp_default_output()
1180 udp->uh_ulen = htons(ulen); in tcp_default_output()
1184 tcpip_fillheaders(inp, tp->t_port, ip, th); in tcp_default_output()
1192 if (flags & TH_FIN && tp->t_flags & TF_SENTFIN && in tcp_default_output()
1193 tp->snd_nxt == tp->snd_max) in tcp_default_output()
1194 tp->snd_nxt--; in tcp_default_output()
1201 if (tp->t_state == TCPS_SYN_SENT && V_tcp_do_ecn) { in tcp_default_output()
1205 if ((TCPS_HAVERCVDSYN(tp->t_state)) && in tcp_default_output()
1206 (tp->t_flags2 & (TF2_ECN_PERMIT | TF2_ACE_PERMIT))) { in tcp_default_output()
1208 if ((tp->t_state == TCPS_SYN_RECEIVED) && in tcp_default_output()
1209 (tp->t_flags2 & TF2_ECN_SND_ECE)) in tcp_default_output()
1210 tp->t_flags2 &= ~TF2_ECN_SND_ECE; in tcp_default_output()
1213 ip6->ip6_flow &= ~htonl(IPTOS_ECN_MASK << IPV6_FLOWLABEL_LEN); in tcp_default_output()
1214 ip6->ip6_flow |= htonl(ect << IPV6_FLOWLABEL_LEN); in tcp_default_output()
1219 ip->ip_tos &= ~IPTOS_ECN_MASK; in tcp_default_output()
1220 ip->ip_tos |= ect; in tcp_default_output()
1240 th->th_seq = htonl(tp->snd_nxt); in tcp_default_output()
1242 th->th_seq = htonl(tp->snd_max); in tcp_default_output()
1244 th->th_seq = htonl(p->rxmit); in tcp_default_output()
1245 p->rxmit += len; in tcp_default_output()
1252 if ((tp->t_flags & TF_LRD) && SEQ_GEQ(p->rxmit, p->end)) in tcp_default_output()
1253 p->rxmit = tp->snd_recover; in tcp_default_output()
1254 tp->sackhint.sack_bytes_rexmit += len; in tcp_default_output()
1256 if (IN_RECOVERY(tp->t_flags)) { in tcp_default_output()
1262 tp->sackhint.prr_out += len; in tcp_default_output()
1264 th->th_ack = htonl(tp->rcv_nxt); in tcp_default_output()
1267 th->th_off = (sizeof (struct tcphdr) + optlen) >> 2; in tcp_default_output()
1277 if (recwin < (so->so_rcv.sb_hiwat / 4) && in tcp_default_output()
1278 recwin < tp->t_maxseg) in tcp_default_output()
1280 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt) && in tcp_default_output()
1281 recwin < (tp->rcv_adv - tp->rcv_nxt)) in tcp_default_output()
1282 recwin = (tp->rcv_adv - tp->rcv_nxt); in tcp_default_output()
1290 th->th_win = htons((u_short) in tcp_default_output()
1291 (min(sbspace(&so->so_rcv), TCP_MAXWIN))); in tcp_default_output()
1294 recwin = roundup2(recwin, 1 << tp->rcv_scale); in tcp_default_output()
1295 th->th_win = htons((u_short)(recwin >> tp->rcv_scale)); in tcp_default_output()
1299 * Adjust the RXWIN0SENT flag - indicate that we have advertised in tcp_default_output()
1306 if (th->th_win == 0) { in tcp_default_output()
1307 tp->t_sndzerowin++; in tcp_default_output()
1308 tp->t_flags |= TF_RXWIN0SENT; in tcp_default_output()
1310 tp->t_flags &= ~TF_RXWIN0SENT; in tcp_default_output()
1311 if (SEQ_GT(tp->snd_up, tp->snd_nxt)) { in tcp_default_output()
1312 th->th_urp = htons((u_short)(tp->snd_up - tp->snd_nxt)); in tcp_default_output()
1316 * If no urgent pointer to send, then we pull in tcp_default_output()
1321 tp->snd_up = tp->snd_una; /* drag it along */ in tcp_default_output()
1329 m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */ in tcp_default_output()
1340 (u_char *)(th + 1) + (to.to_signature - opt))) != 0) { in tcp_default_output()
1356 if (tp->t_port) { in tcp_default_output()
1357 m->m_pkthdr.csum_flags = CSUM_UDP_IPV6; in tcp_default_output()
1358 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); in tcp_default_output()
1359 udp->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0); in tcp_default_output()
1360 th->th_sum = htons(0); in tcp_default_output()
1363 m->m_pkthdr.csum_flags = CSUM_TCP_IPV6; in tcp_default_output()
1364 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum); in tcp_default_output()
1365 th->th_sum = in6_cksum_pseudo(ip6, in tcp_default_output()
1376 if (tp->t_port) { in tcp_default_output()
1377 m->m_pkthdr.csum_flags = CSUM_UDP; in tcp_default_output()
1378 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); in tcp_default_output()
1379 udp->uh_sum = in_pseudo(ip->ip_src.s_addr, in tcp_default_output()
1380 ip->ip_dst.s_addr, htons(ulen + IPPROTO_UDP)); in tcp_default_output()
1381 th->th_sum = htons(0); in tcp_default_output()
1384 m->m_pkthdr.csum_flags = CSUM_TCP; in tcp_default_output()
1385 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum); in tcp_default_output()
1386 th->th_sum = in_pseudo(ip->ip_src.s_addr, in tcp_default_output()
1387 ip->ip_dst.s_addr, htons(sizeof(struct tcphdr) + in tcp_default_output()
1392 KASSERT(ip->ip_v == IPVERSION, in tcp_default_output()
1393 ("%s: IP version incorrect: %d", __func__, ip->ip_v)); in tcp_default_output()
1402 KASSERT(len > tp->t_maxseg - optlen - ipsec_optlen, in tcp_default_output()
1404 m->m_pkthdr.csum_flags |= CSUM_TSO; in tcp_default_output()
1405 m->m_pkthdr.tso_segsz = tp->t_maxseg - optlen - ipsec_optlen; in tcp_default_output()
1423 lgb = tcp_log_event(tp, th, &so->so_rcv, &so->so_snd, in tcp_default_output()
1436 * m->m_pkthdr.len should have been set before checksum calculation, in tcp_default_output()
1447 ip6->ip6_hlim = in6_selecthlim(inp, NULL); in tcp_default_output()
1454 ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6)); in tcp_default_output()
1456 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) in tcp_default_output()
1457 tp->t_flags2 |= TF2_PLPMTU_PMTUD; in tcp_default_output()
1459 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; in tcp_default_output()
1461 if (tp->t_state == TCPS_SYN_SENT) in tcp_default_output()
1468 error = ip6_output(m, inp->in6p_outputopts, &inp->inp_route6, in tcp_default_output()
1469 ((so->so_options & SO_DONTROUTE) ? IP_ROUTETOIF : 0), in tcp_default_output()
1472 if (error == EMSGSIZE && inp->inp_route6.ro_nh != NULL) in tcp_default_output()
1473 mtu = inp->inp_route6.ro_nh->nh_mtu; in tcp_default_output()
1481 ip->ip_len = htons(m->m_pkthdr.len); in tcp_default_output()
1483 if (inp->inp_vflag & INP_IPV6PROTO) in tcp_default_output()
1484 ip->ip_ttl = in6_selecthlim(inp, NULL); in tcp_default_output()
1494 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) { in tcp_default_output()
1495 tp->t_flags2 |= TF2_PLPMTU_PMTUD; in tcp_default_output()
1496 if (tp->t_port == 0 || len < V_tcp_minmss) { in tcp_default_output()
1497 ip->ip_off |= htons(IP_DF); in tcp_default_output()
1500 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; in tcp_default_output()
1503 if (tp->t_state == TCPS_SYN_SENT) in tcp_default_output()
1508 error = ip_output(m, inp->inp_options, &inp->inp_route, in tcp_default_output()
1509 ((so->so_options & SO_DONTROUTE) ? IP_ROUTETOIF : 0), 0, inp); in tcp_default_output()
1511 if (error == EMSGSIZE && inp->inp_route.ro_nh != NULL) in tcp_default_output()
1512 mtu = inp->inp_route.ro_nh->nh_mtu; in tcp_default_output()
1517 lgb->tlb_errno = error; in tcp_default_output()
1522 tcp_account_for_send(tp, len, (tp->snd_nxt != tp->snd_max), 0, hw_tls); in tcp_default_output()
1531 } else if ((tp->t_flags & TF_FORCEDATA) == 0 || in tcp_default_output()
1533 tcp_seq startseq = tp->snd_nxt; in tcp_default_output()
1540 tp->snd_nxt++; in tcp_default_output()
1542 tp->snd_nxt++; in tcp_default_output()
1543 tp->t_flags |= TF_SENTFIN; in tcp_default_output()
1548 tp->snd_nxt += len; in tcp_default_output()
1549 if (SEQ_GT(tp->snd_nxt, tp->snd_max)) { in tcp_default_output()
1554 if (tp->snd_una == tp->snd_max) in tcp_default_output()
1555 tp->t_acktime = ticks; in tcp_default_output()
1556 tp->snd_max = tp->snd_nxt; in tcp_default_output()
1561 tp->t_sndtime = ticks; in tcp_default_output()
1562 if (tp->t_rtttime == 0) { in tcp_default_output()
1563 tp->t_rtttime = ticks; in tcp_default_output()
1564 tp->t_rtseq = startseq; in tcp_default_output()
1568 if (!(tp->t_flags & TF_GPUTINPROG) && len) { in tcp_default_output()
1569 tp->t_flags |= TF_GPUTINPROG; in tcp_default_output()
1570 tp->gput_seq = startseq; in tcp_default_output()
1571 tp->gput_ack = startseq + in tcp_default_output()
1572 ulmin(sbavail(&so->so_snd) - off, sendwin); in tcp_default_output()
1573 tp->gput_ts = tcp_ts_getticks(); in tcp_default_output()
1580 * and not doing a pure ack or a keep-alive probe. in tcp_default_output()
1582 * round-trip time + 2 * round-trip time variance. in tcp_default_output()
1588 ((sack_rxmit && tp->snd_nxt != tp->snd_max) || in tcp_default_output()
1589 (tp->snd_nxt != tp->snd_una))) { in tcp_default_output()
1592 tp->t_rxtshift = 0; in tcp_default_output()
1595 } else if (len == 0 && sbavail(&so->so_snd) && in tcp_default_output()
1601 * 1) A -> B: packet with enough data to fill the window in tcp_default_output()
1602 * 2) B -> A: ACK for #1 + new data (0 window in tcp_default_output()
1604 * 3) A -> B: ACK for #2, 0 len packet in tcp_default_output()
1612 * So, if you send a 0-length packet, but there is data in tcp_default_output()
1617 tp->t_rxtshift = 0; in tcp_default_output()
1630 tp->t_flags |= TF_SENTFIN; in tcp_default_output()
1632 if (SEQ_GT(tp->snd_nxt + xlen, tp->snd_max)) in tcp_default_output()
1633 tp->snd_max = tp->snd_nxt + xlen; in tcp_default_output()
1636 (tp->rcv_numsacks > 0) && in tcp_default_output()
1637 TCPS_HAVEESTABLISHED(tp->t_state) && in tcp_default_output()
1638 (tp->t_flags & TF_SACK_PERMIT)) { in tcp_default_output()
1639 /* Clean up any DSACK's sent */ in tcp_default_output()
1644 SEQ_LT(tp->snd_nxt, SEQ_MIN(p->rxmit, p->end))) { in tcp_default_output()
1647 * after an RTO, pull snd_nxt along. in tcp_default_output()
1649 tp->snd_nxt = SEQ_MIN(p->rxmit, p->end); in tcp_default_output()
1665 if (((tp->t_flags & TF_FORCEDATA) == 0 || in tcp_default_output()
1670 p->rxmit = SEQ_MIN(p->end, p->rxmit) - len; in tcp_default_output()
1671 tp->sackhint.sack_bytes_rexmit -= len; in tcp_default_output()
1672 KASSERT(tp->sackhint.sack_bytes_rexmit >= 0, in tcp_default_output()
1677 tp->snd_nxt -= len; in tcp_default_output()
1679 tp->snd_nxt--; in tcp_default_output()
1681 if (IN_RECOVERY(tp->t_flags)) in tcp_default_output()
1682 tp->sackhint.prr_out -= len; in tcp_default_output()
1688 tp->t_softerror = error; in tcp_default_output()
1692 tp->snd_cwnd = tcp_maxseg(tp); in tcp_default_output()
1705 tp->t_flags &= ~TF_TSO; in tcp_default_output()
1707 tcp_mss_update(tp, -1, mtu, NULL, NULL); in tcp_default_output()
1715 if (TCPS_HAVERCVDSYN(tp->t_state)) { in tcp_default_output()
1716 tp->t_softerror = error; in tcp_default_output()
1732 if (SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv)) in tcp_default_output()
1733 tp->rcv_adv = tp->rcv_nxt + recwin; in tcp_default_output()
1734 tp->last_ack_sent = tp->rcv_nxt; in tcp_default_output()
1735 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK); in tcp_default_output()
1746 int t = ((tp->t_srtt >> 2) + tp->t_rttvar) >> 1; in tcp_setpersist()
1750 tp->t_flags &= ~TF_PREVVALID; in tcp_setpersist()
1756 if (tp->t_state == TCPS_CLOSED) in tcp_setpersist()
1762 TCPT_RANGESET(tt, t * tcp_backoff[tp->t_rxtshift], in tcp_setpersist()
1764 if (TP_MAXUNACKTIME(tp) && tp->t_acktime) { in tcp_setpersist()
1765 maxunacktime = tp->t_acktime + TP_MAXUNACKTIME(tp) - ticks; in tcp_setpersist()
1772 if (tp->t_rxtshift < V_tcp_retries) in tcp_setpersist()
1773 tp->t_rxtshift++; in tcp_setpersist()
1783 * The optimal order for a SYN/SYN-ACK segment is:
1791 * we only have 10 bytes for SACK options (40 - (12 + 18)).
1799 if ((to->to_flags & mask) != mask) in tcp_addoptions()
1803 switch (to->to_flags & mask) { in tcp_addoptions()
1809 if (TCP_MAXOLEN - optlen < TCPOLEN_MAXSEG) in tcp_addoptions()
1814 to->to_mss = htons(to->to_mss); in tcp_addoptions()
1815 bcopy((u_char *)&to->to_mss, optp, sizeof(to->to_mss)); in tcp_addoptions()
1816 optp += sizeof(to->to_mss); in tcp_addoptions()
1823 if (TCP_MAXOLEN - optlen < TCPOLEN_WINDOW) in tcp_addoptions()
1828 *optp++ = to->to_wscale; in tcp_addoptions()
1835 if (TCP_MAXOLEN - optlen < TCPOLEN_SACK_PERMITTED) in tcp_addoptions()
1846 if (TCP_MAXOLEN - optlen < TCPOLEN_TIMESTAMP) in tcp_addoptions()
1851 to->to_tsval = htonl(to->to_tsval); in tcp_addoptions()
1852 to->to_tsecr = htonl(to->to_tsecr); in tcp_addoptions()
1853 bcopy((u_char *)&to->to_tsval, optp, sizeof(to->to_tsval)); in tcp_addoptions()
1854 optp += sizeof(to->to_tsval); in tcp_addoptions()
1855 bcopy((u_char *)&to->to_tsecr, optp, sizeof(to->to_tsecr)); in tcp_addoptions()
1856 optp += sizeof(to->to_tsecr); in tcp_addoptions()
1860 int siglen = TCPOLEN_SIGNATURE - 2; in tcp_addoptions()
1866 if (TCP_MAXOLEN - optlen < TCPOLEN_SIGNATURE) { in tcp_addoptions()
1867 to->to_flags &= ~TOF_SIGNATURE; in tcp_addoptions()
1873 to->to_signature = optp; in tcp_addoptions()
1874 while (siglen--) in tcp_addoptions()
1881 struct sackblk *sack = (struct sackblk *)to->to_sacks; in tcp_addoptions()
1888 if (TCP_MAXOLEN - optlen < TCPOLEN_SACKHDR + TCPOLEN_SACK) in tcp_addoptions()
1892 sackblks = min(to->to_nsacks, in tcp_addoptions()
1893 (TCP_MAXOLEN - optlen) / TCPOLEN_SACK); in tcp_addoptions()
1895 while (sackblks--) { in tcp_addoptions()
1896 sack_seq = htonl(sack->start); in tcp_addoptions()
1899 sack_seq = htonl(sack->end); in tcp_addoptions()
1913 total_len = TCPOLEN_FAST_OPEN_EMPTY + to->to_tfo_len; in tcp_addoptions()
1914 if (TCP_MAXOLEN - optlen < total_len) { in tcp_addoptions()
1915 to->to_flags &= ~TOF_FASTOPEN; in tcp_addoptions()
1920 if (to->to_tfo_len > 0) { in tcp_addoptions()
1921 bcopy(to->to_tfo_cookie, optp, to->to_tfo_len); in tcp_addoptions()
1922 optp += to->to_tfo_len; in tcp_addoptions()
1940 * "The content of the header beyond the End-of-Option option in tcp_addoptions()
1977 if (off == 0 && m->m_flags & M_PKTHDR) in tcp_m_copym()
1983 if (off < m->m_len) in tcp_m_copym()
1985 off -= m->m_len; in tcp_m_copym()
1986 if ((sb) && (m == sb->sb_sndptr)) { in tcp_m_copym()
1987 sb->sb_sndptroff += m->m_len; in tcp_m_copym()
1988 sb->sb_sndptr = m->m_next; in tcp_m_copym()
1990 m = m->m_next; in tcp_m_copym()
1996 if (hw_tls && (m->m_flags & M_EXTPG)) in tcp_m_copym()
1997 tls = m->m_epg_tls; in tcp_m_copym()
2013 if (m->m_flags & M_EXTPG) in tcp_m_copym()
2014 ntls = m->m_epg_tls; in tcp_m_copym()
2032 mlen = min(len, m->m_len - off); in tcp_m_copym()
2042 if (m->m_flags & M_EXTPG) { in tcp_m_copym()
2065 mlen = (seglimit - frags - 1) * fragsize; in tcp_m_copym()
2074 seglimit -= frags; in tcp_m_copym()
2079 n = m_gethdr(M_NOWAIT, m->m_type); in tcp_m_copym()
2081 n = m_get(M_NOWAIT, m->m_type); in tcp_m_copym()
2089 n->m_pkthdr.len -= off0; in tcp_m_copym()
2091 n->m_pkthdr.len = len; in tcp_m_copym()
2092 pkthdrlen = &n->m_pkthdr.len; in tcp_m_copym()
2095 n->m_len = mlen; in tcp_m_copym()
2096 len_cp += n->m_len; in tcp_m_copym()
2097 if (m->m_flags & (M_EXT | M_EXTPG)) { in tcp_m_copym()
2098 n->m_data = m->m_data + off; in tcp_m_copym()
2102 (u_int)n->m_len); in tcp_m_copym()
2104 if (sb && (sb->sb_sndptr == m) && in tcp_m_copym()
2105 ((n->m_len + off) >= m->m_len) && m->m_next) { in tcp_m_copym()
2106 sb->sb_sndptroff += m->m_len; in tcp_m_copym()
2107 sb->sb_sndptr = m->m_next; in tcp_m_copym()
2111 len -= n->m_len; in tcp_m_copym()
2113 m = m->m_next; in tcp_m_copym()
2114 np = &n->m_next; in tcp_m_copym()
2131 * delay (eg. trans-continental/oceanic links). Setting the in tcp_sndbuf_autoscale()
2135 * The criteria to step up the send buffer one notch are: in tcp_sndbuf_autoscale()
2154 * of available bandwidth (the non-use of it) for wasting some in tcp_sndbuf_autoscale()
2163 if (V_tcp_do_autosndbuf && so->so_snd.sb_flags & SB_AUTOSIZE) { in tcp_sndbuf_autoscale()
2166 lowat = V_tcp_sendbuf_auto_lowat ? so->so_snd.sb_lowat : 0; in tcp_sndbuf_autoscale()
2167 if ((tp->snd_wnd / 4 * 5) >= so->so_snd.sb_hiwat - lowat && in tcp_sndbuf_autoscale()
2168 sbused(&so->so_snd) >= in tcp_sndbuf_autoscale()
2169 (so->so_snd.sb_hiwat / 8 * 7) - lowat && in tcp_sndbuf_autoscale()
2170 sbused(&so->so_snd) < V_tcp_autosndbuf_max && in tcp_sndbuf_autoscale()
2171 sendwin >= (sbused(&so->so_snd) - in tcp_sndbuf_autoscale()
2172 (tp->snd_nxt - tp->snd_una))) { in tcp_sndbuf_autoscale()
2174 min(so->so_snd.sb_hiwat + V_tcp_autosndbuf_inc, in tcp_sndbuf_autoscale()
2176 so->so_snd.sb_flags &= ~SB_AUTOSIZE; in tcp_sndbuf_autoscale()